diff --git a/pkg/project/provider/aws.go b/pkg/project/provider/aws.go index 6b0a0ebacd..2a98117307 100644 --- a/pkg/project/provider/aws.go +++ b/pkg/project/provider/aws.go @@ -92,13 +92,17 @@ func (a *AwsProvider) Init(app string, stage string, args map[string]interface{} if err != nil { return err } - if assumeRole, ok := args["assumeRole"].(map[string]interface{}); ok { - stsclient := sts.NewFromConfig(cfg) - cfg.Credentials = stscreds.NewAssumeRoleProvider(stsclient, assumeRole["roleArn"].(string), func(aro *stscreds.AssumeRoleOptions) { - if sessionName, ok := assumeRole["sessionName"].(string); ok { - aro.RoleSessionName = sessionName + if assumeRoles, ok := args["assumeRoles"].([]interface{}); ok { + for _, role := range assumeRoles { + if roleMap, ok := role.(map[string]interface{}); ok { + stsclient := sts.NewFromConfig(cfg) + cfg.Credentials = stscreds.NewAssumeRoleProvider(stsclient, roleMap["roleArn"].(string), func(aro *stscreds.AssumeRoleOptions) { + if sessionName, ok := roleMap["sessionName"].(string); ok { + aro.RoleSessionName = sessionName + } + }) } - }) + } } _, err = cfg.Credentials.Retrieve(ctx) if err != nil { diff --git a/platform/src/components/aws/fargate.ts b/platform/src/components/aws/fargate.ts index 5e32dc508d..266fb45bdb 100644 --- a/platform/src/components/aws/fargate.ts +++ b/platform/src/components/aws/fargate.ts @@ -1,5 +1,3 @@ -import fs from "fs"; -import path from "path"; import { ComponentResourceOptions, interpolate, secret } from "@pulumi/pulumi"; import { all, output } from "@pulumi/pulumi"; import { Input } from "../input"; @@ -13,7 +11,6 @@ import { ImageArgs, Platform } from "@pulumi/docker-build"; import { Component, Transform, transform } from "../component"; import { cloudwatch, - ecr, ecs, getCallerIdentityOutput, getPartitionOutput, @@ -23,11 +20,11 @@ import { import { Link } from "../link"; import { Permission } from "./permission"; import { bootstrap } from "./helpers/bootstrap"; -import { imageBuilder } from "./helpers/container-builder"; import { toNumber } from "../cpu"; import { toSeconds } from "../duration"; import { Cluster } from "./cluster"; import { physicalName } from "../naming"; +import { Image } from "./image"; export const supportedCpus = { "0.25 vCPU": 256, @@ -1052,86 +1049,20 @@ export function createTaskDefinition( image: (() => { if (typeof container.image === "string") return output(container.image); - const containerImage = container.image; - const contextPath = path.join($cli.paths.root, container.image.context); - const dockerfile = container.image.dockerfile ?? "Dockerfile"; - const dockerfilePath = path.join(contextPath, dockerfile); - const dockerIgnorePath = fs.existsSync( - path.join(contextPath, `${dockerfile}.dockerignore`), - ) - ? path.join(contextPath, `${dockerfile}.dockerignore`) - : path.join(contextPath, ".dockerignore"); - - // add .sst to .dockerignore if not exist - const lines = fs.existsSync(dockerIgnorePath) - ? fs.readFileSync(dockerIgnorePath).toString().split("\n") - : []; - if (!lines.find((line) => line === ".sst")) { - fs.writeFileSync( - dockerIgnorePath, - [...lines, "", "# sst", ".sst"].join("\n"), - ); - } - - // Build image - const image = imageBuilder( - ...transform( - args.transform?.image, - `${name}Image${container.name}`, - { - context: { location: contextPath }, - dockerfile: { location: dockerfilePath }, - buildArgs: containerImage.args, - secrets: all([linkEnvs, containerImage.secrets ?? {}]).apply( - ([link, secrets]) => ({ ...link, ...secrets }), - ), - target: container.image.target, - platforms: [container.image.platform], - tags: [container.name, ...(container.image.tags ?? [])].map( - (tag) => interpolate`${bootstrapData.assetEcrUrl}:${tag}`, - ), - registries: [ - ecr - .getAuthorizationTokenOutput( - { - registryId: bootstrapData.assetEcrRegistryId, - }, - { parent }, - ) - .apply((authToken) => ({ - address: authToken.proxyEndpoint, - password: secret(authToken.password), - username: authToken.userName, - })), - ], - ...(container.image.cache !== false - ? { - cacheFrom: [ - { - registry: { - ref: interpolate`${bootstrapData.assetEcrUrl}:${container.name}-cache`, - }, - }, - ], - cacheTo: [ - { - registry: { - ref: interpolate`${bootstrapData.assetEcrUrl}:${container.name}-cache`, - imageManifest: true, - ociMediaTypes: true, - mode: "max", - }, - }, - ], - } - : {}), - push: true, - }, - { parent }, - ), - ); + const image = new Image(`${name}${container.name}`, { + context: container.image.context, + dockerfile: container.image.dockerfile, + args: container.image.args, + secrets: linkEnvs, + target: container.image.target, + platforms: [container.image.platform], + tags: [container.name, ...(container.image.tags ?? [])], + transform: { + image: args.transform?.image, + }, + }); - return interpolate`${bootstrapData.assetEcrUrl}@${image.digest}`; + return image.uri; })(), cpu: container.cpu ? toNumber(container.cpu) : undefined, memory: container.memory ? toMBs(container.memory) : undefined, diff --git a/platform/src/components/aws/function.ts b/platform/src/components/aws/function.ts index 2bb753ae82..6cd217b23a 100644 --- a/platform/src/components/aws/function.ts +++ b/platform/src/components/aws/function.ts @@ -37,7 +37,7 @@ import { } from "@pulumi/aws"; import { Permission, permission } from "./permission.js"; import { Vpc } from "./vpc.js"; -import { Image } from "@pulumi/docker-build"; +import { Image } from "./image"; import { rpc } from "../rpc/rpc.js"; import { parseRoleArn } from "./helpers/arn.js"; import { RandomBytes } from "@pulumi/random"; @@ -2234,67 +2234,15 @@ export class Function extends Component implements Link.Linkable { // The build artifact directory already exists, with all the user code and // config files. It also has the dockerfile, we need to now just build and push to // the container registry. - return all([isContainer, dev, bundle, containerCache]).apply( - ([ - isContainer, - dev, - bundle, // We need the bundle to be resolved because of implicit dockerfiles even though we don't use it here - containerCache, - ]) => { - if (!isContainer || dev) return; - const authToken = ecr.getAuthorizationTokenOutput({ - registryId: bootstrapData.assetEcrRegistryId, + return all([isContainer, dev, architecture]).apply( + ([isContainer, dev, architecture]) => { + if (!isContainer || dev) return; + return new Image(name, { + context: `.sst/artifacts/${name}-src`, + tags: ['latest'], + platforms: [architecture === "arm64" ? "linux/arm64" : "linux/amd64"], }); - - return new Image( - `${name}Image`, - { - tags: [$interpolate`${bootstrapData.assetEcrUrl}:latest`], - context: { - location: path.join( - $cli.paths.work, - "artifacts", - `${name}-src`, - ), - }, - ...(containerCache !== false - ? { - cacheFrom: [ - { - registry: { - ref: $interpolate`${bootstrapData.assetEcrUrl}:${name}-cache`, - }, - }, - ], - cacheTo: [ - { - registry: { - ref: $interpolate`${bootstrapData.assetEcrUrl}:${name}-cache`, - imageManifest: true, - ociMediaTypes: true, - mode: "max", - }, - }, - ], - } - : {}), - platforms: [ - architecture.apply((v) => - v === "arm64" ? "linux/arm64" : "linux/amd64", - ), - ], - push: true, - registries: [ - authToken.apply((authToken) => ({ - address: authToken.proxyEndpoint, - username: authToken.userName, - password: secret(authToken.password), - })), - ], - }, - { parent }, - ); }, ); } @@ -2513,9 +2461,7 @@ export class Function extends Component implements Link.Linkable { ...(isContainer ? { packageType: "Image", - imageUri: imageAsset!.ref.apply( - (ref) => ref?.replace(":latest", ""), - ), + imageUri: imageAsset!.uri, imageConfig: { commands: [ all([handler, runtime]).apply(([handler, runtime]) => { diff --git a/platform/src/components/aws/helpers/container-builder.ts b/platform/src/components/aws/helpers/container-builder.ts deleted file mode 100644 index b2e1101b7d..0000000000 --- a/platform/src/components/aws/helpers/container-builder.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { all, ComponentResourceOptions } from "@pulumi/pulumi"; -import { Semaphore } from "../../../util/semaphore"; -import { Image, ImageArgs } from "@pulumi/docker-build"; - -const limiter = new Semaphore( - parseInt(process.env.SST_BUILD_CONCURRENCY_CONTAINER || "1"), -); - -export function imageBuilder( - name: string, - args: ImageArgs, - opts?: ComponentResourceOptions, -) { - // Wait for the all args values to be resolved before acquiring the semaphore - return all([args]).apply(async ([args]) => { - await limiter.acquire(name); - const image = new Image( - name, - { - ...(process.env.BUILDX_BUILDER - ? { builder: { name: process.env.BUILDX_BUILDER } } - : {}), - ...args, - }, - opts, - ); - return image.urn.apply(() => { - limiter.release(); - return image; - }); - }); -} diff --git a/platform/src/components/aws/image.ts b/platform/src/components/aws/image.ts new file mode 100644 index 0000000000..afcf63e9ec --- /dev/null +++ b/platform/src/components/aws/image.ts @@ -0,0 +1,317 @@ +import fs from "fs"; +import { all, Input, interpolate, Output, secret } from "@pulumi/pulumi"; +import { Component, Transform, transform } from "../component.js"; +import { Link } from "../link.js"; +import { getRegionOutput } from "@pulumi/aws"; +import { ecr } from "@pulumi/aws"; +import { Semaphore } from "../../util/semaphore.js"; +import { bootstrap } from "./helpers/bootstrap.js"; +import { + Image as PulumiDockerImage, + ImageArgs as PulumiDockerImageArgs, +} from "@pulumi/docker-build"; +import path from "path"; + +// Extracted from `@pulumi/docker-build` `Platform` - unhandled by doc generator. +export type Platform = "darwin/386" | "darwin/amd64" | "darwin/arm" | "darwin/arm64" | "dragonfly/amd64" | "freebsd/386" | "freebsd/amd64" | "freebsd/arm" | "linux/386" | "linux/amd64" | "linux/arm" | "linux/arm64" | "linux/mips64" | "linux/mips64le" | "linux/ppc64le" | "linux/riscv64" | "linux/s390x" | "netbsd/386" | "netbsd/amd64" | "netbsd/arm" | "openbsd/386" | "openbsd/amd64" | "openbsd/arm" | "plan9/386" | "plan9/amd64" | "solaris/amd64" | "windows/386" | "windows/amd64" + +// Extracted from `@pulumi/docker-build` `CompressionType` - unhandled by doc generator. +export type CompressionType = "zstd" | "gzip" | "estargz" + +export interface ImageArgs { + /** + * Key-value pairs of [build args](https://docs.docker.com/build/guide/build-args/) to pass to the Docker build command. + * @example + * ```js + * { + * args: { + * MY_VAR: "value" + * } + * } + * ``` + */ + args?: Input>>; + /** + * The path to the [Docker build context](https://docs.docker.com/build/building/context/#local-context). The path is relative to your project's `sst.config.ts`. + * @default `"."` + * @example + * Specify the folder of the Docker build context: + * ```js + * { + * context: "./app" + * } + * ``` + */ + context?: Input; + /** + * The path to the [Dockerfile](https://docs.docker.com/reference/cli/docker/image/build/#file). + * The path is relative to the build `context`. + * @default `"Dockerfile"` + * @example + * Specify different Dockerfile: + * ```js + * { + * dockerfile: "Dockerfile.prod" + * } + * ``` + */ + dockerfile?: Input; + /** + * Set target platform(s) for the build. Defaults to the host's platform. + * + * Equivalent to Docker's `--platform` flag. + */ + platforms?: Input; + /** + * A mapping of secret names to their corresponding values. + * + * Unlike the Docker CLI, these can be passed by value and do not need to + * exist on-disk or in environment variables. + * + * Build arguments and environment variables are persistent in the final + * image, so you should use this for sensitive values. + * + * Similar to Docker's `--secret` flag. + */ + secrets?: Input>; + /** + * Tags to apply to the Docker image. + * @example + * ```js + * { + * tags: ["v1.0.0", "commit-613c1b2"] + * } + * ``` + */ + tags?: Input; + /** + * The stage to build up to in a [multi-stage Dockerfile](https://docs.docker.com/build/building/multi-stage/#stop-at-a-specific-build-stage). + * @example + * ```js + * { + * target: "stage1" + * } + * ``` + */ + target?: Input; + /** + * Specify compression type and level. + */ + compression?: { + /** + * Specify compression algorithm used: + * - `zstd` - smallest compression. + * - `gzip` - best compatibility. + * - `estargz` - fastest pull. + * + * @default `"zstd"` + */ + type: Input + /** + * Compression level. Different limits for each compression algorithm. + */ + level: Input + } + /** + * [Transform](/docs/components#transform) how this component creates its underlying + * resources. + */ + transform?: { + /** + * Transform the Docker Image resource. + */ + image?: Transform; + }; +} + +const limiter = new Semaphore( + parseInt(process.env.SST_BUILD_CONCURRENCY_CONTAINER || "1"), +); + +/** + * The `Image` component builds docker images and uploads them to [AWS ECR (Elastic Container Registry)](https://aws.amazon.com/ecr/). + * + * @example + * #### Minimal example + * Create `Dockerfile` and `sst.config.ts` in root directory. + * + * ```ts title="sst.config.ts" + * new sst.aws.Image("MyImage", {}); + * ``` + * + * @example + * #### Different dockerfile and context + * [Minimal example](#minimal-example) setup with `./app/Dockerfile.function`. + * + * Note: + * - By default, context is the root directory where `sst.config.ts` is located. + * - By default, dockerfile is `Dockerfile`. + * + * ```ts {2,3} title="sst.config.ts" + * new sst.aws.Image("MyImage", { + * context: './app', + * dockerfile: 'Dockerfile.function' + * }); + * ``` + * + * @example + * #### Smaller compression example + * [Minimal example](#minimal-example) setup, optimised for smaller image. + * This setup reduces both storage and pull speed. + * + * ```ts {2-5} title="sst.config.ts" + * new sst.aws.Image("MyImage", { + * compression: { + * type: 'zstd', + * level: 9 + * } + * }); + * ``` + * + * @example + * #### Faster pull speed example + * [Minimal example](#minimal-example) setup, optimised for faster pulls. + * This setup reduces pull speed by indexing image in chunks. + * + * Alternatively, consider `"gzip"` compression with [SOCI indexing](https://github.com/awslabs/soci-snapshotter) - which can be [automatically created on AWS](https://github.com/awslabs/cfn-ecr-aws-soci-index-builder). + * + * ```ts {2-4} title="sst.config.ts" + * new sst.aws.Image("MyImage", { + * compression: { + * type: 'estargz', + * } + * }); + * ``` + */ +export class Image extends Component implements Link.Linkable { + private _uri: Output; + + constructor(name: string, args: ImageArgs, opts?: any) { + const componentName = `${name}Image` + super(__pulumiType, componentName, args, opts); + + const parent = this; + const region = getRegionOutput({}, opts).name; + const bootstrapData = region.apply((region) => bootstrap.forRegion(region)); + // Empty uri should fail deployment if not set + this._uri = interpolate`` + + all([args, bootstrapData]).apply( + async ([args, bootstrapData]) => { + // Wait for the all args values to be resolved before acquiring the semaphore + await limiter.acquire(componentName); + + const contextPath = path.join($cli.paths.root, args.context ?? "."); + const dockerfile = args.dockerfile ?? "Dockerfile"; + const dockerfilePath = path.join(contextPath, dockerfile); + + // add .sst to .dockerignore if not exist + const dockerIgnorePath = fs.existsSync( + path.join(contextPath, `${dockerfile}.dockerignore`), + ) + ? path.join(contextPath, `${dockerfile}.dockerignore`) + : path.join(contextPath, ".dockerignore"); + const lines = fs.existsSync(dockerIgnorePath) + ? fs.readFileSync(dockerIgnorePath).toString().split("\n") + : []; + if (!lines.find((line) => line === ".sst")) { + fs.writeFileSync( + dockerIgnorePath, + [...lines, "", "# sst", ".sst"].join("\n"), + ); + } + + function normalizeCompressionType() { + if (!args.compression?.type) return 'zstd' + return args.compression.type + } + + const image = new PulumiDockerImage( + ...transform( + args.transform?.image, + componentName, + { + context: { location: contextPath }, + dockerfile: { location: dockerfilePath }, + buildArgs: args.args, + secrets: args.secrets, + target: args.target, + platforms: args.platforms, + tags: [name, ...(args.tags ?? [])].map( + (tag) => interpolate`${bootstrapData.assetEcrUrl}:${tag}`, + ), + registries: [ + ecr + .getAuthorizationTokenOutput( + { + registryId: bootstrapData.assetEcrRegistryId, + }, + { parent }, + ) + .apply((authToken) => ({ + address: authToken.proxyEndpoint, + password: secret(authToken.password), + username: authToken.userName, + })), + ], + cacheFrom: [ + { + registry: { + ref: $interpolate`${bootstrapData.assetEcrUrl}:${name}-cache`, + }, + }, + ], + cacheTo: [ + { + registry: { + ref: $interpolate`${bootstrapData.assetEcrUrl}:${name}-cache`, + imageManifest: true, + ociMediaTypes: true, + mode: "max" as const, + compression: normalizeCompressionType(), + // Normalise compression level + ...(args.compression?.level + ? { compressionLevel: args.compression.level } + : {} + ) + }, + }, + ], + push: true, + ...(process.env.BUILDX_BUILDER + ? { builder: { name: process.env.BUILDX_BUILDER } } + : {}), + }, + { parent }, + ), + ); + this._uri = interpolate`${bootstrapData.assetEcrUrl}@${image.digest}` + + image.urn.apply(() => { + limiter.release(); + }); + return image; + }, + ); + } + + /** + * The uri of the ECR container image. + */ + public get uri() { + return this._uri + } + + /** @internal */ + public getSSTLink() { + return { + properties: { + uri: this.uri, + }, + }; + } +} + +const __pulumiType = "sst:aws:Image"; +// @ts-expect-error +Image.__pulumiType = __pulumiType; diff --git a/platform/src/components/aws/index.ts b/platform/src/components/aws/index.ts index ae22d23ef6..4a4e49bd56 100644 --- a/platform/src/components/aws/index.ts +++ b/platform/src/components/aws/index.ts @@ -17,6 +17,7 @@ export * from "./dynamo.js"; export * from "./efs.js"; export * from "./email.js"; export * from "./function.js"; +export * from "./image.js"; export * from "./kinesis-stream.js"; export * from "./nextjs.js"; export * from "./opencontrol.js"; diff --git a/www/astro.config.mjs b/www/astro.config.mjs index 9d8dbfdf5e..02a457e629 100644 --- a/www/astro.config.mjs +++ b/www/astro.config.mjs @@ -106,6 +106,7 @@ const sidebar = [ "docs/component/aws/dynamo", "docs/component/aws/realtime", "docs/component/aws/sns-topic", + "docs/component/aws/image", "docs/component/aws/function", "docs/component/aws/postgres", "docs/component/aws/app-sync", diff --git a/www/generate.ts b/www/generate.ts index ceca76d1f3..7e33a96d13 100644 --- a/www/generate.ts +++ b/www/generate.ts @@ -2171,6 +2171,7 @@ async function buildComponents() { "../platform/src/components/aws/dynamo-lambda-subscriber.ts", "../platform/src/components/aws/efs.ts", "../platform/src/components/aws/email.ts", + "../platform/src/components/aws/image.ts", "../platform/src/components/aws/function.ts", "../platform/src/components/aws/mysql.ts", "../platform/src/components/aws/postgres.ts",