diff --git a/README.md b/README.md index b5b189f..c370701 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,21 @@ Alternatively, a given path can be specified via the `--working-directory` flag: lambda-builder build --working-directory path/to/app ``` +In addition to the `lambda.yml`, a docker image can be produced from the generated artifact by specifying the `--build-image` flag. This also allows for multiple `--label` flags as well as specifying a single image tag via either `-t` or `--tag`: + +```shell +# will write a lambda.zip in the specified path +# and generate a docker image named `lambda-builder:$APP:latest` +# where $APP is the last portion of the working directory +lambda-builder build --build-image + +# adds the labels com.example/key=value and com.example/another-key=value +lambda-builder build --build-image --label com.example/key=value --label com.example/another-key=value + +# tags the image as app/awesome:1234 +lambda-builder build --build-image --tag app/awesome:1234 +``` + ### How does it work Internally, `lambda-builder` detects a given language and builds the app according to the script specified by the detected builder within a disposablecontainer environment emulating AWS Lambda. If a builder is not detected, the build will fail. The following languages are supported: @@ -80,16 +95,22 @@ Internally, `lambda-builder` detects a given language and builds the app accordi When the app is built, a `lambda.zip` will be produced in the specified working directory. The resulting `lambda.zip` can be uploaded to S3 and used within a Lambda function. -Both the builder and the build image environment can be overriden in an optional `lambda.yml` file in the specified working directory. An example of this file is as follows: +Both the builder and the build image environment can be overriden in an optional `lambda.yml` file in the specified working directory. + +### `lambda.yml` + +The following a short description of the `lambda.yml` format. ```yaml --- build_image: mlupin/docker-lambda:dotnetcore3.1-build builder: dotnet +run_image: mlupin/docker-lambda:dotnetcore3.1 ``` - `build_image`: A docker image that is accessible by the docker daemon. The `build_image` _should_ be based on an existing Lambda image - builders may fail if they cannot run within the specified `build_image`. The build will fail if the image is inaccessible by the docker daemon. - `builder`: The name of a builder. This may be used if multiple builders match and a specific builder is desired. If an invalid builder is specified, the build will fail. +- `run_image`: A docker image that is accessible by the docker daemon. The `run_image` _should_ be based on an existing Lambda image - built images may fail to start if they are not compatible with the produced artifact. The generation of the `run` iage will fail if the image is inaccessible by the docker daemon. ### Deploying diff --git a/builders/dotnet.go b/builders/dotnet.go index bee5d6c..a884ccc 100644 --- a/builders/dotnet.go +++ b/builders/dotnet.go @@ -8,7 +8,12 @@ type DotnetBuilder struct { func NewDotnetBuilder(config Config) (DotnetBuilder, error) { var err error - config.BuildImage, err = getBuilder(config, "mlupin/docker-lambda:dotnet6-build") + config.BuilderBuildImage, err = getBuildImage(config, "mlupin/docker-lambda:dotnet6-build") + if err != nil { + return DotnetBuilder{}, err + } + + config.BuilderRunImage, err = getRunImage(config, "mlupin/docker-lambda:dotnet6") if err != nil { return DotnetBuilder{}, err } @@ -19,11 +24,7 @@ func NewDotnetBuilder(config Config) (DotnetBuilder, error) { } func (b DotnetBuilder) BuildImage() string { - return b.Config.BuildImage -} - -func (b DotnetBuilder) GetConfig() Config { - return b.Config + return b.Config.BuilderBuildImage } func (b DotnetBuilder) Detect() bool { @@ -35,7 +36,15 @@ func (b DotnetBuilder) Detect() bool { } func (b DotnetBuilder) Execute() error { - return executeBuilder(b.script(), b.Config) + return executeBuilder(b.script(), b.GetTaskBuildDir(), b.Config) +} + +func (b DotnetBuilder) GetConfig() Config { + return b.Config +} + +func (b DotnetBuilder) GetTaskBuildDir() string { + return "/var/task" } func (b DotnetBuilder) Name() string { diff --git a/builders/go.go b/builders/go.go index 9673923..cadab78 100644 --- a/builders/go.go +++ b/builders/go.go @@ -8,7 +8,12 @@ type GoBuilder struct { func NewGoBuilder(config Config) (GoBuilder, error) { var err error - config.BuildImage, err = getBuilder(config, "lambci/lambda:build-go1.x") + config.BuilderBuildImage, err = getBuildImage(config, "lambci/lambda:build-go1.x") + if err != nil { + return GoBuilder{}, err + } + + config.BuilderRunImage, err = getRunImage(config, "mlupin/docker-lambda:provided.al2") if err != nil { return GoBuilder{}, err } @@ -19,11 +24,7 @@ func NewGoBuilder(config Config) (GoBuilder, error) { } func (b GoBuilder) BuildImage() string { - return b.Config.BuildImage -} - -func (b GoBuilder) GetConfig() Config { - return b.Config + return b.Config.BuilderBuildImage } func (b GoBuilder) Detect() bool { @@ -35,7 +36,15 @@ func (b GoBuilder) Detect() bool { } func (b GoBuilder) Execute() error { - return executeBuilder(b.script(), b.Config) + return executeBuilder(b.script(), b.GetTaskBuildDir(), b.Config) +} + +func (b GoBuilder) GetConfig() Config { + return b.Config +} + +func (b GoBuilder) GetTaskBuildDir() string { + return "/go/src/handler" } func (b GoBuilder) Name() string { diff --git a/builders/main.go b/builders/main.go index 1b8736b..ece3398 100644 --- a/builders/main.go +++ b/builders/main.go @@ -2,6 +2,7 @@ package builders import ( "fmt" + "html/template" "io/ioutil" "os" "path/filepath" @@ -14,6 +15,7 @@ import ( type Builder interface { GetConfig() Config + GetTaskBuildDir() string Detect() bool Execute() error BuildImage() string @@ -21,18 +23,62 @@ type Builder interface { } type Config struct { - BuildImage string - Identifier string - RunQuiet bool - WorkingDirectory string + BuildImage bool + BuilderBuildImage string + BuilderRunImage string + Identifier string + ImageLabels []string + ImageTag string + RunQuiet bool + WorkingDirectory string +} + +func (c Config) GetImageTag() string { + if c.ImageTag != "" { + return c.ImageTag + } + + appName := filepath.Base(c.WorkingDirectory) + return fmt.Sprintf("lambda-builder/%s:latest", appName) } type LambdaYML struct { Builder string `yaml:"builder"` BuildImage string `yaml:"build_image"` + RunImage string `yaml:"run_image"` +} + +func executeBuilder(script string, taskBuildDir string, config Config) error { + tmp, err := os.MkdirTemp("", "lambda-builder") + defer func() { + os.RemoveAll(tmp) + }() + + if err != nil { + return fmt.Errorf("error preparing temporary build directory: %s", err.Error()) + } + + if err := executeBuildContainer(tmp, script, taskBuildDir, config); err != nil { + return err + } + + if config.BuildImage { + fmt.Printf("=====> Building image\n") + fmt.Printf(" Generating temporary Dockerfile\n") + if err := generateDockerfile(tmp, config); err != nil { + return err + } + + fmt.Printf(" Executing build of %s\n", config.GetImageTag()) + if err := buildDockerImage(tmp, config); err != nil { + return err + } + } + + return nil } -func executeBuilder(script string, config Config) error { +func executeBuildContainer(tmp string, script string, taskBuildDir string, config Config) error { args := []string{ "container", "run", @@ -41,7 +87,8 @@ func executeBuilder(script string, config Config) error { "--label", "com.dokku.lambda-builder/executor=true", "--name", fmt.Sprintf("lambda-builder-executor-%s", config.Identifier), "--volume", fmt.Sprintf("%s:/tmp/task", config.WorkingDirectory), - config.BuildImage, + "--volume", fmt.Sprintf("%s:%s", tmp, taskBuildDir), + config.BuilderBuildImage, "/bin/bash", "-c", script, } @@ -54,11 +101,71 @@ func executeBuilder(script string, config Config) error { res, err := cmd.Execute() if err != nil { - return fmt.Errorf("failed to execute builder: %s", err.Error()) + return fmt.Errorf("error executing builder: %s", err.Error()) } if res.ExitCode != 0 { - return fmt.Errorf("failed to execute builder, exit code %d", res.ExitCode) + return fmt.Errorf("error executing builder, exit code %d", res.ExitCode) + } + + return nil +} + +func generateDockerfile(tmp string, config Config) error { + dockerfileName := filepath.Join(tmp, fmt.Sprintf("%s.Dockerfile", config.Identifier)) + f, err := os.Create(dockerfileName) + if err != nil { + return fmt.Errorf("error creating Dockerfile: %s", err) + } + + tpl, err := template.New("t1").Parse(` +FROM {{ .run_image }} +COPY . /var/task +`) + if err != nil { + return fmt.Errorf("error generating template: %s", err) + } + + data := map[string]string{ + "run_image": config.BuilderRunImage, + } + + if err := tpl.Execute(f, data); err != nil { + return fmt.Errorf("error writing Dockerfile: %s", err) + } + + return nil +} + +func buildDockerImage(tmp string, config Config) error { + args := []string{ + "image", + "build", + "--file", filepath.Join(tmp, fmt.Sprintf("%s.Dockerfile", config.Identifier)), + "--progress", "plain", + "--tag", config.GetImageTag(), + } + + for _, label := range config.ImageLabels { + args = append(args, "--label", label) + } + + args = append(args, tmp) + + cmd := execute.ExecTask{ + Args: args, + Command: "docker", + Cwd: config.WorkingDirectory, + StreamStdio: !config.RunQuiet, + } + + res, err := cmd.Execute() + if err != nil { + return fmt.Errorf("error building image: %s", err.Error()) + } + + if res.ExitCode != 0 { + return fmt.Errorf("error building image, exit code %d", res.ExitCode) } return nil @@ -88,9 +195,9 @@ func ParseLambdaYML(config Config) (LambdaYML, error) { return lambdaYML, nil } -func getBuilder(config Config, defaultImage string) (string, error) { - if config.BuildImage != "" { - return config.BuildImage, nil +func getBuildImage(config Config, defaultImage string) (string, error) { + if config.BuilderBuildImage != "" { + return config.BuilderBuildImage, nil } lambdaYML, err := ParseLambdaYML(config) @@ -104,3 +211,20 @@ func getBuilder(config Config, defaultImage string) (string, error) { return lambdaYML.BuildImage, nil } + +func getRunImage(config Config, defaultImage string) (string, error) { + if config.BuilderRunImage != "" { + return config.BuilderRunImage, nil + } + + lambdaYML, err := ParseLambdaYML(config) + if err != nil { + return "", err + } + + if lambdaYML.RunImage == "" { + return defaultImage, nil + } + + return lambdaYML.RunImage, nil +} diff --git a/builders/nodejs.go b/builders/nodejs.go index 589efe8..aa0f9f9 100644 --- a/builders/nodejs.go +++ b/builders/nodejs.go @@ -8,7 +8,12 @@ type NodejsBuilder struct { func NewNodejsBuilder(config Config) (NodejsBuilder, error) { var err error - config.BuildImage, err = getBuilder(config, "mlupin/docker-lambda:nodejs14.x-build") + config.BuilderBuildImage, err = getBuildImage(config, "mlupin/docker-lambda:nodejs14.x-build") + if err != nil { + return NodejsBuilder{}, err + } + + config.BuilderRunImage, err = getRunImage(config, "mlupin/docker-lambda:nodejs14.x") if err != nil { return NodejsBuilder{}, err } @@ -19,11 +24,7 @@ func NewNodejsBuilder(config Config) (NodejsBuilder, error) { } func (b NodejsBuilder) BuildImage() string { - return b.Config.BuildImage -} - -func (b NodejsBuilder) GetConfig() Config { - return b.Config + return b.Config.BuilderBuildImage } func (b NodejsBuilder) Detect() bool { @@ -35,7 +36,15 @@ func (b NodejsBuilder) Detect() bool { } func (b NodejsBuilder) Execute() error { - return executeBuilder(b.script(), b.Config) + return executeBuilder(b.script(), b.GetTaskBuildDir(), b.Config) +} + +func (b NodejsBuilder) GetConfig() Config { + return b.Config +} + +func (b NodejsBuilder) GetTaskBuildDir() string { + return "/var/task" } func (b NodejsBuilder) Name() string { diff --git a/builders/python.go b/builders/python.go index c05db2b..1273b22 100644 --- a/builders/python.go +++ b/builders/python.go @@ -21,14 +21,19 @@ type PythonBuilder struct { func NewPythonBuilder(config Config) (PythonBuilder, error) { var err error version := "3.9" - if config.BuildImage == "" { + if config.BuilderBuildImage == "" || config.BuilderRunImage == "" { version, err = parsePythonVersion(config.WorkingDirectory, []string{"3.8", "3.9"}) if err != nil { return PythonBuilder{}, err } } - config.BuildImage, err = getBuilder(config, fmt.Sprintf("mlupin/docker-lambda:python%s-build", version)) + config.BuilderBuildImage, err = getBuildImage(config, fmt.Sprintf("mlupin/docker-lambda:python%s-build", version)) + if err != nil { + return PythonBuilder{}, err + } + + config.BuilderRunImage, err = getRunImage(config, fmt.Sprintf("mlupin/docker-lambda:python%s", version)) if err != nil { return PythonBuilder{}, err } @@ -39,11 +44,7 @@ func NewPythonBuilder(config Config) (PythonBuilder, error) { } func (b PythonBuilder) BuildImage() string { - return b.Config.BuildImage -} - -func (b PythonBuilder) GetConfig() Config { - return b.Config + return b.Config.BuilderBuildImage } func (b PythonBuilder) Detect() bool { @@ -63,7 +64,15 @@ func (b PythonBuilder) Detect() bool { } func (b PythonBuilder) Execute() error { - return executeBuilder(b.script(), b.Config) + return executeBuilder(b.script(), b.GetTaskBuildDir(), b.Config) +} + +func (b PythonBuilder) GetConfig() Config { + return b.Config +} + +func (b PythonBuilder) GetTaskBuildDir() string { + return "/var/task" } func (b PythonBuilder) Name() string { diff --git a/builders/ruby.go b/builders/ruby.go index 1ffbd65..dd76f8a 100644 --- a/builders/ruby.go +++ b/builders/ruby.go @@ -8,7 +8,12 @@ type RubyBuilder struct { func NewRubyBuilder(config Config) (RubyBuilder, error) { var err error - config.BuildImage, err = getBuilder(config, "mlupin/docker-lambda:ruby2.7-build") + config.BuilderBuildImage, err = getBuildImage(config, "mlupin/docker-lambda:ruby2.7-build") + if err != nil { + return RubyBuilder{}, err + } + + config.BuilderRunImage, err = getRunImage(config, "mlupin/docker-lambda:ruby2.7") if err != nil { return RubyBuilder{}, err } @@ -19,11 +24,7 @@ func NewRubyBuilder(config Config) (RubyBuilder, error) { } func (b RubyBuilder) BuildImage() string { - return b.Config.BuildImage -} - -func (b RubyBuilder) GetConfig() Config { - return b.Config + return b.Config.BuilderBuildImage } func (b RubyBuilder) Detect() bool { @@ -34,8 +35,16 @@ func (b RubyBuilder) Detect() bool { return false } +func (b RubyBuilder) GetConfig() Config { + return b.Config +} + +func (b RubyBuilder) GetTaskBuildDir() string { + return "/var/task" +} + func (b RubyBuilder) Execute() error { - return executeBuilder(b.script(), b.Config) + return executeBuilder(b.script(), b.GetTaskBuildDir(), b.Config) } func (b RubyBuilder) Name() string { diff --git a/commands/build.go b/commands/build.go index dd132b4..71894df 100644 --- a/commands/build.go +++ b/commands/build.go @@ -19,6 +19,9 @@ import ( type BuildCommand struct { command.Meta + buildImage bool + imageTag string + labels []string quiet bool workingDirectory string } @@ -63,7 +66,10 @@ func (c *BuildCommand) FlagSet() *flag.FlagSet { f := c.Meta.FlagSet(c.Name(), command.FlagSetClient) f.BoolVar(&c.quiet, "quiet", false, "run builder in quiet mode") + f.BoolVar(&c.buildImage, "build-image", false, "build a docker image") + f.StringVarP(&c.imageTag, "tag", "t", "", "name and optionally a tag in the 'name:tag' format") f.StringVar(&c.workingDirectory, "working-directory", workingDirectory, "working directory") + f.StringArrayVar(&c.labels, "label", []string{}, " set metadata for an image") return f } @@ -71,8 +77,8 @@ func (c *BuildCommand) AutocompleteFlags() complete.Flags { return command.MergeAutocompleteFlags( c.Meta.AutocompleteFlags(command.FlagSetClient), complete.Flags{ - "--count": complete.PredictNothing, - "--quiet": complete.PredictNothing, + "--build-image": complete.PredictNothing, + "--quiet": complete.PredictNothing, }, ) } @@ -111,7 +117,10 @@ func (c *BuildCommand) Run(args []string) int { identifier := uuid.New().String() config := builders.Config{ + BuildImage: c.buildImage, Identifier: identifier, + ImageLabels: c.labels, + ImageTag: c.imageTag, RunQuiet: c.quiet, WorkingDirectory: c.workingDirectory, }