diff --git a/.gitignore b/.gitignore index b031dced1..50664d49e 100644 --- a/.gitignore +++ b/.gitignore @@ -389,5 +389,6 @@ $RECYCLE.BIN/ /Dockerfile tests/integration/workflows/go_dep/data/src/*/vendor/* +tests/integration/workflows/go_dep/data/pkg/* # End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 952e5f9a3..b8725094b 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "1.6.0" +__version__ = "1.8.1" RPC_PROTOCOL_VERSION = "0.3" diff --git a/aws_lambda_builders/__main__.py b/aws_lambda_builders/__main__.py index d29839893..e6cfc4318 100644 --- a/aws_lambda_builders/__main__.py +++ b/aws_lambda_builders/__main__.py @@ -12,6 +12,7 @@ import logging import re +from aws_lambda_builders.architecture import X86_64 from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version @@ -126,6 +127,7 @@ def main(): # pylint: disable=too-many-statements mode=params.get("mode", None), download_dependencies=params.get("download_dependencies", True), dependencies_dir=params.get("dependencies_dir", None), + architecture=params.get("architecture", X86_64), ) # Return a success response diff --git a/aws_lambda_builders/architecture.py b/aws_lambda_builders/architecture.py new file mode 100644 index 000000000..703b67408 --- /dev/null +++ b/aws_lambda_builders/architecture.py @@ -0,0 +1,5 @@ +""" +Enum for determining type of architectures for Lambda Function. +""" +X86_64 = "x86_64" +ARM64 = "arm64" diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index f11187f2b..474c82f8e 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -6,6 +6,7 @@ import os import logging +from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY from aws_lambda_builders.workflow import Capability @@ -66,6 +67,7 @@ def build( mode=None, download_dependencies=True, dependencies_dir=None, + architecture=X86_64, ): """ Actually build the code by running workflows @@ -108,13 +110,17 @@ def build( :param mode: Optional, Mode the build should produce - :type mode: bool + :type download_dependencies: bool :param download_dependencies: Optional, Should download dependencies when building - :type mode: str + :type dependencies_dir: str :param dependencies_dir: Optional, Path to folder the dependencies should be downloaded to + + :type architecture: str + :param architecture: + Type of architecture x86_64 and arm64 for Lambda Function """ if not os.path.exists(scratch_dir): @@ -132,6 +138,7 @@ def build( mode=mode, download_dependencies=download_dependencies, dependencies_dir=dependencies_dir, + architecture=architecture, ) return workflow.run() diff --git a/aws_lambda_builders/exceptions.py b/aws_lambda_builders/exceptions.py index 646b914dc..2ed3f3f8e 100644 --- a/aws_lambda_builders/exceptions.py +++ b/aws_lambda_builders/exceptions.py @@ -24,6 +24,30 @@ class MisMatchRuntimeError(LambdaBuilderError): ) +class RuntimeValidatorError(LambdaBuilderError): + """ + Raise when runtime is not supported or when runtime is not compatible with architecture + """ + + MESSAGE = "Runtime validation error for {runtime}" + + +class UnsupportedRuntimeError(RuntimeValidatorError): + """ + Raise when runtime is not supported + """ + + MESSAGE = "Runtime {runtime} is not supported" + + +class UnsupportedArchitectureError(RuntimeValidatorError): + """ + Raise when runtime does not support architecture + """ + + MESSAGE = "Architecture {architecture} is not supported for runtime {runtime}" + + class WorkflowNotFoundError(LambdaBuilderError): """ Raised when a workflow matching the given capabilities was not found diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index 1ccd6dce5..9bb215637 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -7,6 +7,7 @@ import os import logging +from aws_lambda_builders.architecture import X86_64, ARM64 LOG = logging.getLogger(__name__) @@ -148,3 +149,18 @@ def _access_check(fn, mode): if _access_check(name, mode): paths.append(name) return paths + + +def get_goarch(architecture): + """ + Parameters + ---------- + architecture : str + name of the type of architecture + + Returns + ------- + str + returns a valid GO Architecture value + """ + return "arm64" if architecture == ARM64 else "amd64" diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py index 4e014c103..ac0c99b6d 100644 --- a/aws_lambda_builders/validator.py +++ b/aws_lambda_builders/validator.py @@ -4,14 +4,74 @@ import logging +from aws_lambda_builders.architecture import ARM64, X86_64 +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError + + LOG = logging.getLogger(__name__) +SUPPORTED_RUNTIMES = { + "nodejs10.x": [X86_64], + "nodejs12.x": [ARM64, X86_64], + "nodejs14.x": [ARM64, X86_64], + "python2.7": [X86_64], + "python3.6": [X86_64], + "python3.7": [X86_64], + "python3.8": [ARM64, X86_64], + "python3.9": [ARM64, X86_64], + "ruby2.5": [X86_64], + "ruby2.7": [ARM64, X86_64], + "java8": [ARM64, X86_64], + "java11": [ARM64, X86_64], + "go1.x": [ARM64, X86_64], + "dotnetcore2.1": [X86_64], + "dotnetcore3.1": [ARM64, X86_64], + "provided": [ARM64, X86_64], +} + class RuntimeValidator(object): - def __init__(self, runtime): + def __init__(self, runtime, architecture): + """ + + Parameters + ---------- + runtime : str + name of the AWS Lambda runtime that you are building for. This is sent to the builder for + informational purposes. + architecture : str + Architecture for which the build will be based on in AWS lambda + """ self.runtime = runtime self._runtime_path = None + self.architecture = architecture def validate(self, runtime_path): + """ + Parameters + ---------- + runtime_path : str + runtime to check eg: /usr/bin/runtime + + Returns + ------- + str + runtime to check eg: /usr/bin/runtime + + Raises + ------ + UnsupportedRuntimeError + Raised when runtime provided is not support. + + UnsupportedArchitectureError + Raised when runtime is not compatible with architecture + """ + runtime_architectures = SUPPORTED_RUNTIMES.get(self.runtime, None) + + if not runtime_architectures: + raise UnsupportedRuntimeError(runtime=self.runtime) + if self.architecture not in runtime_architectures: + raise UnsupportedArchitectureError(runtime=self.runtime, architecture=self.architecture) + self._runtime_path = runtime_path return runtime_path diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index 83e9aa765..414b95b71 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -12,8 +12,15 @@ from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.registry import DEFAULT_REGISTRY -from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError +from aws_lambda_builders.exceptions import ( + WorkflowFailedError, + WorkflowUnknownError, + MisMatchRuntimeError, + RuntimeValidatorError, +) from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.architecture import X86_64 + LOG = logging.getLogger(__name__) @@ -32,16 +39,17 @@ class BuildMode(object): # TODO: Move sanitize out to its own class. -def sanitize(func): +def sanitize(func): # pylint: disable=too-many-statements """ sanitize the executable path of the runtime specified by validating it. :param func: Workflow's run method is sanitized """ @functools.wraps(func) - def wrapper(self, *args, **kwargs): + def wrapper(self, *args, **kwargs): # pylint: disable=too-many-statements valid_paths = {} invalid_paths = {} + validation_errors = [] # NOTE: we need to access binaries to get paths and resolvers, before validating. for binary, binary_checker in self.binaries.items(): invalid_paths[binary] = [] @@ -61,18 +69,30 @@ def wrapper(self, *args, **kwargs): except MisMatchRuntimeError as ex: LOG.debug("Invalid executable for %s at %s", binary, executable_path, exc_info=str(ex)) invalid_paths[binary].append(executable_path) + + except RuntimeValidatorError as ex: + LOG.debug("Runtime validation error for %s", binary, exc_info=str(ex)) + if str(ex) not in validation_errors: + validation_errors.append(str(ex)) + if valid_paths.get(binary, None): binary_checker.binary_path = valid_paths[binary] break + if validation_errors: + raise WorkflowFailedError( + workflow_name=self.NAME, action_name="Validation", reason="\n".join(validation_errors) + ) + if len(self.binaries) != len(valid_paths): validation_failed_binaries = set(self.binaries.keys()).difference(valid_paths.keys()) - messages = [] for validation_failed_binary in validation_failed_binaries: message = "Binary validation failed for {0}, searched for {0} in following locations : {1} which did not satisfy constraints for runtime: {2}. Do you have {0} for runtime: {2} on your PATH?".format( validation_failed_binary, invalid_paths[validation_failed_binary], self.runtime ) - messages.append(message) - raise WorkflowFailedError(workflow_name=self.NAME, action_name="Validation", reason="\n".join(messages)) + validation_errors.append(message) + raise WorkflowFailedError( + workflow_name=self.NAME, action_name="Validation", reason="\n".join(validation_errors) + ) func(self, *args, **kwargs) return wrapper @@ -142,56 +162,40 @@ def __init__( mode=BuildMode.RELEASE, download_dependencies=True, dependencies_dir=None, + architecture=X86_64, ): """ Initialize the builder with given arguments. These arguments together form the "public API" that each build action must support at the minimum. - :type source_dir: str - :param source_dir: + Parameters + ---------- + source_dir : str Path to a folder containing the source code - - :type artifacts_dir: str - :param artifacts_dir: + artifacts_dir : str Path to a folder where the built artifacts should be placed - - :type scratch_dir: str - :param scratch_dir: + scratch_dir : str Path to a directory that the workflow can use as scratch space. Workflows are expected to use this directory to write temporary files instead of ``/tmp`` or other OS-specific temp directories. - - :type manifest_path: str - :param manifest_path: + manifest_path : str Path to the dependency manifest - - :type runtime: str - :param runtime: - Optional, name of the AWS Lambda runtime that you are building for. This is sent to the builder for - informational purposes. - - :type optimizations: dict - :param optimizations: - Optional dictionary of optimization flags to pass to the build action. **Not supported**. - - :type options: dict - :param options: - Optional dictionary of options ot pass to build action. **Not supported**. - - :type executable_search_paths: list - :param executable_search_paths: - Optional, Additional list of paths to search for executables required by the workflow. - - :type mode: str - :param mode: - Optional, Mode the build should produce - - :type mode: bool - :param download_dependencies: - Optional, Should download dependencies when building - - :type mode: str - :param dependencies_dir: - Optional, Path to folder the dependencies should be downloaded to + runtime : str, optional + name of the AWS Lambda runtime that you are building for. This is sent to the builder for + informational purposes, by default None + executable_search_paths : list, optional + Additional list of paths to search for executables required by the workflow, by default None + optimizations : dict, optional + dictionary of optimization flags to pass to the build action. **Not supported**, by default None + options : dict, optional + dictionary of options ot pass to build action. **Not supported**., by default None + mode : str, optional + Mode the build should produce, by default BuildMode.RELEASE + download_dependencies: bool, optional + Should download dependencies when building + dependencies_dir : str, optional + Path to folder the dependencies should be downloaded to + architecture : str, optional + Architecture type either arm64 or x86_64 for which the build will be based on in AWS lambda, by default X86_64 """ self.source_dir = source_dir @@ -205,6 +209,7 @@ def __init__( self.mode = mode self.download_dependencies = download_dependencies self.dependencies_dir = dependencies_dir + self.architecture = architecture # Actions are registered by the subclasses as they seem fit self.actions = [] @@ -237,7 +242,7 @@ def get_validators(self): """ No-op validator that does not validate the runtime_path. """ - return [RuntimeValidator(runtime=self.runtime)] + return [RuntimeValidator(runtime=self.runtime, architecture=self.architecture)] @property def binaries(self): diff --git a/aws_lambda_builders/workflows/custom_make/validator.py b/aws_lambda_builders/workflows/custom_make/validator.py new file mode 100644 index 000000000..ed1216dc8 --- /dev/null +++ b/aws_lambda_builders/workflows/custom_make/validator.py @@ -0,0 +1,19 @@ +""" +Custom Make Runtime Validation +""" + +from aws_lambda_builders.validator import RuntimeValidator + + +class CustomMakeRuntimeValidator(RuntimeValidator): + """ + Default runtime validator for CustomMake workflow + """ + + def __init__(self, runtime, architecture): + super(CustomMakeRuntimeValidator, self).__init__(runtime, architecture) + self._valid_runtime_path = None + + def validate(self, runtime_path): + self._runtime_path = runtime_path + return runtime_path diff --git a/aws_lambda_builders/workflows/custom_make/workflow.py b/aws_lambda_builders/workflows/custom_make/workflow.py index e1f66e0ba..04c680237 100644 --- a/aws_lambda_builders/workflows/custom_make/workflow.py +++ b/aws_lambda_builders/workflows/custom_make/workflow.py @@ -1,6 +1,7 @@ """ ProvidedMakeWorkflow """ +from aws_lambda_builders.workflows.custom_make.validator import CustomMakeRuntimeValidator from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.path_resolver import PathResolver @@ -56,3 +57,6 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim def get_resolvers(self): return [PathResolver(runtime="provided", binary="make", executable_search_paths=self.executable_search_paths)] + + def get_validators(self): + return [CustomMakeRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py index de7386d04..4a28770d6 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py @@ -8,6 +8,7 @@ from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError from aws_lambda_builders.workflow import BuildMode +from aws_lambda_builders.architecture import ARM64 from .utils import OSUtils from .dotnetcli import DotnetCLIExecutionError @@ -66,13 +67,14 @@ class RunPackageAction(BaseAction): DESCRIPTION = "Execute the `dotnet lambda package` command." PURPOSE = Purpose.COMPILE_SOURCE - def __init__(self, source_dir, subprocess_dotnet, artifacts_dir, options, mode, os_utils=None): + def __init__(self, source_dir, subprocess_dotnet, artifacts_dir, options, mode, architecture=None, os_utils=None): super(RunPackageAction, self).__init__() self.source_dir = source_dir self.subprocess_dotnet = subprocess_dotnet self.artifacts_dir = artifacts_dir self.options = options self.mode = mode + self.architecture = architecture self.os_utils = os_utils if os_utils else OSUtils() def execute(self): @@ -82,7 +84,15 @@ def execute(self): zipfilename = os.path.basename(os.path.normpath(self.source_dir)) + ".zip" zipfullpath = os.path.join(self.artifacts_dir, zipfilename) - arguments = ["lambda", "package", "--output-package", zipfullpath] + arguments = [ + "lambda", + "package", + "--output-package", + zipfullpath, + # Specify the architecture with the --runtime MSBuild parameter + "--msbuild-parameters", + "--runtime " + self._get_runtime(), + ] if self.mode and self.mode.lower() == BuildMode.DEBUG: LOG.debug("Debug build requested: Setting configuration to Debug") @@ -102,3 +112,14 @@ def execute(self): except DotnetCLIExecutionError as ex: raise ActionFailedError(str(ex)) + + def _get_runtime(self): + """ + Returns the msbuild runtime for the action architecture + + Returns + ------- + str + linux-arm64 if ARM64, linux-x64 otherwise + """ + return "linux-arm64" if self.architecture == ARM64 else "linux-x64" diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py b/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py index 6756beffc..cd79dc8c1 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py @@ -30,7 +30,12 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim dotnetcli_install = GlobalToolInstallAction(subprocess_dotnet=subprocess_dotnetcli) dotnetcli_deployment = RunPackageAction( - source_dir, subprocess_dotnet=subprocess_dotnetcli, artifacts_dir=artifacts_dir, options=options, mode=mode + source_dir, + subprocess_dotnet=subprocess_dotnetcli, + artifacts_dir=artifacts_dir, + options=options, + mode=mode, + architecture=self.architecture, ) self.actions = [dotnetcli_install, dotnetcli_deployment] diff --git a/aws_lambda_builders/workflows/go_dep/DESIGN.md b/aws_lambda_builders/workflows/go_dep/DESIGN.md index e5cf63d00..f797465f9 100644 --- a/aws_lambda_builders/workflows/go_dep/DESIGN.md +++ b/aws_lambda_builders/workflows/go_dep/DESIGN.md @@ -4,10 +4,18 @@ Building Go projects using the dep tool (https://github.com/golang/dep) is rather simple, if you was to do this by hand, you would perform these commands: +For x86 architecture + - `dep ensure` - `GOOS=linux GOARCH=amd64 go build -o handler main.go` - `zip -r source.zip` +Or for ARM architecture + + - `dep ensure` + - `GOOS=linux GOARCH=arm64 go build -o handler main.go` + - `zip -r source.zip` + The scope of the Go dep builder is to create a macro for these commands to ensure that spelling and paths are correct. We don't have to care about versioning of the tooling of either Go or dep since Lambda doesn't have to care, and so it becomes user preference. @@ -15,9 +23,16 @@ user preference. ## Implementation The go-dep builder runs the above commands with some minor tweaks, the commands ran on behalf of the user are: +For x86 architecture: + 1. dep ensure 2. GOOS=linux GOARCH=amd64 go build -o $ARTIFACT_DIR/$HANDLER_NAME $SOURCE_DIR +For ARM architecture: + + 1. dep ensure + 2. GOOS=linux GOARCH=arm64 go build -o $ARTIFACT_DIR/$HANDLER_NAME $SOURCE_DIR + The main difference being we want to capture the compiled binary to package later, so the binary has the output path as the artifact dir set by the caller. diff --git a/aws_lambda_builders/workflows/go_dep/actions.py b/aws_lambda_builders/workflows/go_dep/actions.py index f424e7c0b..82c96f65d 100644 --- a/aws_lambda_builders/workflows/go_dep/actions.py +++ b/aws_lambda_builders/workflows/go_dep/actions.py @@ -6,6 +6,9 @@ import os from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError +from aws_lambda_builders.architecture import X86_64, ARM64 +from aws_lambda_builders.utils import get_goarch + from .subproc_exec import ExecutionError @@ -46,7 +49,7 @@ class GoBuildAction(BaseAction): DESCRIPTION = "Builds final binary" PURPOSE = Purpose.COMPILE_SOURCE - def __init__(self, base_dir, source_path, output_path, subprocess_go, env=None): + def __init__(self, base_dir, source_path, output_path, subprocess_go, architecture=X86_64, env=None): super(GoBuildAction, self).__init__() self.base_dir = base_dir @@ -54,11 +57,12 @@ def __init__(self, base_dir, source_path, output_path, subprocess_go, env=None): self.output_path = output_path self.subprocess_go = subprocess_go + self.goarch = get_goarch(architecture) self.env = env if not env is None else {} def execute(self): env = self.env - env.update({"GOOS": "linux", "GOARCH": "amd64"}) + env.update({"GOOS": "linux", "GOARCH": self.goarch}) try: self.subprocess_go.run(["build", "-o", self.output_path, self.source_path], cwd=self.source_path, env=env) diff --git a/aws_lambda_builders/workflows/go_dep/workflow.py b/aws_lambda_builders/workflows/go_dep/workflow.py index b357418fa..e9c309e31 100644 --- a/aws_lambda_builders/workflows/go_dep/workflow.py +++ b/aws_lambda_builders/workflows/go_dep/workflow.py @@ -7,7 +7,6 @@ from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflow import BaseWorkflow, Capability - from .actions import DepEnsureAction, GoBuildAction from .utils import OSUtils from .subproc_exec import SubprocessExec @@ -48,5 +47,12 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim self.actions = [ DepEnsureAction(base_dir, subprocess_dep), - GoBuildAction(base_dir, osutils.abspath(source_dir), output_path, subprocess_go, env=osutils.environ), + GoBuildAction( + base_dir, + osutils.abspath(source_dir), + output_path, + subprocess_go, + self.architecture, + env=osutils.environ, + ), ] diff --git a/aws_lambda_builders/workflows/go_modules/DESIGN.md b/aws_lambda_builders/workflows/go_modules/DESIGN.md index 22b0bf00c..8c13a3a18 100644 --- a/aws_lambda_builders/workflows/go_modules/DESIGN.md +++ b/aws_lambda_builders/workflows/go_modules/DESIGN.md @@ -31,7 +31,13 @@ def build(self, source_dir_path, artifacts_dir_path, executable_name): The general algorithm for preparing a Go package for use on AWS Lambda is very simple. It's as follows: -Pass in GOOS=linux and GOARCH=amd64 to the `go build` command to target the +Depending on the architecture pass in either: + + - `GOOS=linux and GOARCH=arm64` for ARM architecture or + + - `GOOS=linux and GOARCH=amd64` for an X86 architecture + +to the `go build` command to target the OS and architecture used on AWS Lambda. Let go tooling handle the cross-compilation, regardless of the build environment. Move the resulting static binary to the artifacts folder to be shipped as a single-file zip diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index ab3e36758..fb9247c46 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -4,6 +4,8 @@ import logging from aws_lambda_builders.workflow import BuildMode +from aws_lambda_builders.architecture import X86_64, ARM64 +from aws_lambda_builders.utils import get_goarch LOG = logging.getLogger(__name__) @@ -19,7 +21,7 @@ class GoModulesBuilder(object): LANGUAGE = "go" - def __init__(self, osutils, binaries, mode=BuildMode.RELEASE): + def __init__(self, osutils, binaries, mode=BuildMode.RELEASE, architecture=X86_64): """Initialize a GoModulesBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -28,10 +30,14 @@ def __init__(self, osutils, binaries, mode=BuildMode.RELEASE): :type binaries: dict :param binaries: A dict of language binaries + + :type architecture: str + :param architecture: name of the type of architecture """ self.osutils = osutils self.binaries = binaries self.mode = mode + self.goarch = get_goarch(architecture) def build(self, source_dir_path, output_path): """Builds a go project onto an output path. @@ -44,7 +50,7 @@ def build(self, source_dir_path, output_path): """ env = {} env.update(self.osutils.environ) - env.update({"GOOS": "linux", "GOARCH": "amd64"}) + env.update({"GOOS": "linux", "GOARCH": self.goarch}) runtime_path = self.binaries[self.LANGUAGE].binary_path cmd = [runtime_path, "build"] if self.mode and self.mode.lower() == BuildMode.DEBUG: diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py index 6ac325b68..1e0949be0 100644 --- a/aws_lambda_builders/workflows/go_modules/validator.py +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -7,28 +7,20 @@ import os import subprocess +from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.exceptions import MisMatchRuntimeError LOG = logging.getLogger(__name__) -class GoRuntimeValidator(object): +class GoRuntimeValidator(RuntimeValidator): LANGUAGE = "go" - SUPPORTED_RUNTIMES = {"go1.x"} GO_VERSION_REGEX = re.compile("go(\\d)\\.(x|\\d+)") - def __init__(self, runtime): - self.runtime = runtime + def __init__(self, runtime, architecture): + super(GoRuntimeValidator, self).__init__(runtime, architecture) self._valid_runtime_path = None - def has_runtime(self): - """ - Checks if the runtime is supported. - :param string runtime: Runtime to check - :return bool: True, if the runtime is supported. - """ - return self.runtime in self.SUPPORTED_RUNTIMES - @staticmethod def get_go_versions(version_string): parts = GoRuntimeValidator.GO_VERSION_REGEX.findall(version_string) @@ -41,12 +33,24 @@ def get_go_versions(version_string): def validate(self, runtime_path): """ Checks if the language supplied matches the required lambda runtime - :param string runtime_path: runtime to check eg: /usr/bin/go - :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime + + Parameters + ---------- + runtime_path : str + runtime to check eg: /usr/bin/go1.x + + Returns + ------- + str + runtime_path, runtime to check eg: /usr/bin/go1.x + + Raises + ------ + MisMatchRuntimeError + Raise runtime is not support or runtime does not support architecture. """ - if not self.has_runtime(): - LOG.warning("'%s' runtime is not " "a supported runtime", self.runtime) - return None + + runtime_path = super(GoRuntimeValidator, self).validate(runtime_path) expected_major_version = int(self.runtime.replace(self.LANGUAGE, "").split(".")[0]) min_expected_minor_version = 11 if expected_major_version == 1 else 0 diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py index 0932e81f5..ceafdbd7a 100644 --- a/aws_lambda_builders/workflows/go_modules/workflow.py +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -31,8 +31,8 @@ def __init__( output_path = osutils.joinpath(artifacts_dir, handler) - builder = GoModulesBuilder(osutils, binaries=self.binaries, mode=mode) + builder = GoModulesBuilder(osutils, binaries=self.binaries, mode=mode, architecture=self.architecture) self.actions = [GoModulesBuildAction(source_dir, output_path, builder)] def get_validators(self): - return [GoRuntimeValidator(runtime=self.runtime)] + return [GoRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/aws_lambda_builders/workflows/java_gradle/gradle_validator.py b/aws_lambda_builders/workflows/java_gradle/gradle_validator.py index 951b5554c..6d839ef3c 100644 --- a/aws_lambda_builders/workflows/java_gradle/gradle_validator.py +++ b/aws_lambda_builders/workflows/java_gradle/gradle_validator.py @@ -6,11 +6,13 @@ import re from aws_lambda_builders.workflows.java.utils import OSUtils +from aws_lambda_builders.validator import RuntimeValidator + LOG = logging.getLogger(__name__) -class GradleValidator(object): +class GradleValidator(RuntimeValidator): VERSION_STRING_WARNING = ( "%s failed to return a version string using the '-v' option. The workflow is unable to " "check that the version of the JVM used is compatible with AWS Lambda." @@ -22,17 +24,31 @@ class GradleValidator(object): "been configured to be compatible with Java %s using 'targetCompatibility' in Gradle." ) - def __init__(self, runtime, os_utils=None, log=None): + def __init__(self, runtime, architecture, os_utils=None, log=None): + super(GradleValidator, self).__init__(runtime, architecture) self.language = "java" self._valid_binary_path = None - self._runtime = runtime self.os_utils = OSUtils() if not os_utils else os_utils self.log = LOG if not log else log - def validate(self, gradle_path): + def validate(self, runtime_path): + """ + Parameters + ---------- + runtime_path : str + gradle path to check eg: /usr/bin/java8 + + Returns + ------- + str + runtime to check for the java binaries eg: /usr/bin/java8 + """ + + gradle_path = super(GradleValidator, self).validate(runtime_path) + jvm_mv = self._get_major_version(gradle_path) - language_version = self._runtime.replace("java", "") + language_version = self.runtime.replace("java", "") if jvm_mv: if int(jvm_mv) > int(language_version): diff --git a/aws_lambda_builders/workflows/java_gradle/workflow.py b/aws_lambda_builders/workflows/java_gradle/workflow.py index 17ff12579..67ec861df 100644 --- a/aws_lambda_builders/workflows/java_gradle/workflow.py +++ b/aws_lambda_builders/workflows/java_gradle/workflow.py @@ -6,6 +6,7 @@ from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.workflows.java.actions import JavaCopyDependenciesAction from aws_lambda_builders.workflows.java.utils import OSUtils + from .actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction from .gradle import SubprocessGradle from .gradle_resolver import GradleResolver @@ -28,6 +29,7 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwar self.os_utils = OSUtils() self.build_dir = None + subprocess_gradle = SubprocessGradle(gradle_binary=self.binaries["gradle"], os_utils=self.os_utils) self.actions = [ @@ -41,7 +43,7 @@ def get_resolvers(self): return [GradleResolver(executable_search_paths=self.executable_search_paths)] def get_validators(self): - return [GradleValidator(self.runtime, self.os_utils)] + return [GradleValidator(self.runtime, self.architecture, self.os_utils)] @property def build_output_dir(self): diff --git a/aws_lambda_builders/workflows/java_maven/maven_validator.py b/aws_lambda_builders/workflows/java_maven/maven_validator.py index ba26f953d..995123690 100644 --- a/aws_lambda_builders/workflows/java_maven/maven_validator.py +++ b/aws_lambda_builders/workflows/java_maven/maven_validator.py @@ -6,11 +6,12 @@ import re from aws_lambda_builders.workflows.java.utils import OSUtils +from aws_lambda_builders.validator import RuntimeValidator LOG = logging.getLogger(__name__) -class MavenValidator(object): +class MavenValidator(RuntimeValidator): VERSION_STRING_WARNING = ( "%s failed to return a version string using the '-v' option. The workflow is unable to " "check that the version of the JVM used is compatible with AWS Lambda." @@ -22,17 +23,30 @@ class MavenValidator(object): "been configured to be compatible with Java %s using 'maven.compiler.target' in Maven." ) - def __init__(self, runtime, os_utils=None, log=None): + def __init__(self, runtime, architecture, os_utils=None, log=None): + super(MavenValidator, self).__init__(runtime, architecture) self.language = "java" self._valid_binary_path = None - self._runtime = runtime self.os_utils = OSUtils() if not os_utils else os_utils self.log = LOG if not log else log - def validate(self, maven_path): + def validate(self, runtime_path): + """ + Parameters + ---------- + runtime_path : str + maven_path to check eg: /usr/bin/java8 + + Returns + ------- + str + runtime to check for the java binaries eg: /usr/bin/java8 + """ + + maven_path = super(MavenValidator, self).validate(runtime_path) jvm_mv = self._get_major_version(maven_path) - language_version = self._runtime.replace("java", "") + language_version = self.runtime.replace("java", "") if jvm_mv: if int(jvm_mv) > int(language_version): diff --git a/aws_lambda_builders/workflows/java_maven/workflow.py b/aws_lambda_builders/workflows/java_maven/workflow.py index 9339be09b..84210d13a 100644 --- a/aws_lambda_builders/workflows/java_maven/workflow.py +++ b/aws_lambda_builders/workflows/java_maven/workflow.py @@ -5,6 +5,7 @@ from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflows.java.actions import JavaCopyDependenciesAction from aws_lambda_builders.workflows.java.utils import OSUtils + from .actions import JavaMavenBuildAction, JavaMavenCopyDependencyAction, JavaMavenCopyArtifactsAction from .maven import SubprocessMaven from .maven_resolver import MavenResolver @@ -43,4 +44,4 @@ def get_resolvers(self): return [MavenResolver(executable_search_paths=self.executable_search_paths)] def get_validators(self): - return [MavenValidator(self.runtime, self.os_utils)] + return [MavenValidator(self.runtime, self.architecture, self.os_utils)] diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index 961489df7..370774bf8 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -40,6 +40,11 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim tar_dest_dir = osutils.joinpath(scratch_dir, "unpacked") tar_package_dir = osutils.joinpath(tar_dest_dir, "package") + if not osutils.file_exists(manifest_path): + LOG.warning("package.json file not found. Continuing the build without dependencies.") + self.actions = [CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)] + return + npm_pack = NodejsNpmPackAction( tar_dest_dir, scratch_dir, manifest_path, osutils=osutils, subprocess_npm=subprocess_npm ) diff --git a/aws_lambda_builders/workflows/python_pip/DESIGN.md b/aws_lambda_builders/workflows/python_pip/DESIGN.md index d619fa645..1d9da131a 100644 --- a/aws_lambda_builders/workflows/python_pip/DESIGN.md +++ b/aws_lambda_builders/workflows/python_pip/DESIGN.md @@ -89,10 +89,10 @@ Sort the downloaded packages into three categories: Pip will give us a wheel when it can, but some distributions do not ship with wheels at all in which case we will have an sdist for it. In some cases a platform specific wheel file may be availble so pip will have downloaded that, -if our platform does not match the platform lambda runs on -(linux_x86_64/manylinux) then the downloaded wheel file may not be compatible -with lambda. Pure python wheels still will be compatible because they have no -platform specific dependencies. +if our platform does not match the platform defined for the lambda function +(linux/manylinux x86_64 or aarch64) then the downloaded wheel file may not be +compatible with lambda. Pure python wheels still will be compatible because +they have no platform specific dependencies. #### Step 3: Try to download a compatible wheel for each incompatible package @@ -100,7 +100,7 @@ Next we need to go through the downloaded packages and pick out any dependencies that do not have a compatible wheel file downloaded. For these packages we need to explicitly try to download a compatible wheel file. A compatible wheel file means one that is explicitly for marked as supporting the -linux_x86_64 or manylinux1. +corresponding architecture for the function. #### Step 4: Try to compile wheel files ourselves diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index eb4f6cf57..62128c85d 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -3,6 +3,7 @@ """ from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError +from aws_lambda_builders.architecture import X86_64 from aws_lambda_builders.workflows.python_pip.utils import OSUtils from .exceptions import MissingPipError from .packager import PythonPipDependencyBuilder, PackagerError, DependencyBuilder, SubprocessPip, PipRunner @@ -15,13 +16,16 @@ class PythonPipBuildAction(BaseAction): PURPOSE = Purpose.RESOLVE_DEPENDENCIES LANGUAGE = "python" - def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, dependencies_dir, binaries): + def __init__( + self, artifacts_dir, scratch_dir, manifest_path, runtime, dependencies_dir, binaries, architecture=X86_64 + ): self.artifacts_dir = artifacts_dir self.manifest_path = manifest_path self.scratch_dir = scratch_dir self.runtime = runtime self.dependencies_dir = dependencies_dir self.binaries = binaries + self.architecture = architecture def execute(self): os_utils = OSUtils() @@ -31,7 +35,9 @@ def execute(self): except MissingPipError as ex: raise ActionFailedError(str(ex)) pip_runner = PipRunner(python_exe=python_path, pip=pip) - dependency_builder = DependencyBuilder(osutils=os_utils, pip_runner=pip_runner, runtime=self.runtime) + dependency_builder = DependencyBuilder( + osutils=os_utils, pip_runner=pip_runner, runtime=self.runtime, architecture=self.architecture + ) package_builder = PythonPipDependencyBuilder( osutils=os_utils, runtime=self.runtime, dependency_builder=dependency_builder diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 0e884c8da..1f7864d47 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -8,7 +8,7 @@ import logging from email.parser import FeedParser - +from aws_lambda_builders.architecture import ARM64, X86_64 from .compat import pip_import_string from .compat import pip_no_compile_c_env_vars from .compat import pip_no_compile_c_shim @@ -95,7 +95,7 @@ def get_lambda_abi(runtime): class PythonPipDependencyBuilder(object): - def __init__(self, runtime, osutils=None, dependency_builder=None): + def __init__(self, runtime, osutils=None, dependency_builder=None, architecture=X86_64): """Initialize a PythonPipDependencyBuilder. :type runtime: str @@ -110,13 +110,17 @@ def __init__(self, runtime, osutils=None, dependency_builder=None): :type dependency_builder: :class:`DependencyBuilder` :param dependency_builder: This class will be used to build the dependencies of the project. + + :type architecture: str + :param description: Architecture used to build dependencies for. This can + be either arm64 or x86_64. The default value is x86_64 if it's not provided. """ self.osutils = osutils if osutils is None: self.osutils = OSUtils() if dependency_builder is None: - dependency_builder = DependencyBuilder(self.osutils, runtime) + dependency_builder = DependencyBuilder(self.osutils, runtime, architecture=architecture) self._dependency_builder = dependency_builder def build_dependencies(self, artifacts_dir_path, scratch_dir_path, requirements_path, ui=None, config=None): @@ -166,12 +170,32 @@ class DependencyBuilder(object): packager. """ - _ADDITIONAL_COMPATIBLE_PLATFORM = {"any", "linux_x86_64"} + _COMPATIBLE_PLATFORM_ARM64 = { + "any", + "manylinux2014_aarch64", + } + + _COMPATIBLE_PLATFORM_X86_64 = { + "any", + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2010_x86_64", + "manylinux2014_x86_64", + } + + _COMPATIBLE_PLATFORMS = { + ARM64: _COMPATIBLE_PLATFORM_ARM64, + X86_64: _COMPATIBLE_PLATFORM_X86_64, + } + _MANYLINUX_LEGACY_MAP = { "manylinux1_x86_64": "manylinux_2_5_x86_64", "manylinux2010_x86_64": "manylinux_2_12_x86_64", "manylinux2014_x86_64": "manylinux_2_17_x86_64", } + + _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"} + # Mapping of abi to glibc version in Lambda runtime. _RUNTIME_GLIBC = { "cp27mu": (2, 17), @@ -184,9 +208,8 @@ class DependencyBuilder(object): # not in _RUNTIME_GLIBC. # Unlikely to hit this case. _DEFAULT_GLIBC = (2, 17) - _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"} - def __init__(self, osutils, runtime, pip_runner=None): + def __init__(self, osutils, runtime, pip_runner=None, architecture=X86_64): """Initialize a DependencyBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -199,12 +222,16 @@ def __init__(self, osutils, runtime, pip_runner=None): :type pip_runner: :class:`PipRunner` :param pip_runner: This class is responsible for executing our pip on our behalf. + + :type architecture: str + :param architecture: Architecture to build for. """ self._osutils = osutils if pip_runner is None: pip_runner = PipRunner(python_exe=None, pip=SubprocessPip(osutils)) self._pip = pip_runner self.runtime = runtime + self.architecture = architecture def build_site_packages(self, requirements_filepath, target_directory, scratch_directory): """Build site-packages directory for a set of requiremetns. @@ -262,9 +289,10 @@ def _download_dependencies(self, directory, requirements_filename): # ship with wheels at all in which case we will have an sdist for it. # In some cases a platform specific wheel file may be availble so pip # will have downloaded that, if our platform does not match the - # platform lambda runs on (linux_x86_64/manylinux) then the downloaded - # wheel file may not be compatible with lambda. Pure python wheels - # still will be compatible because they have no platform dependencies. + # platform that the function will run on (x86_64 or arm64) then the + # downloaded wheel file may not be compatible with Lambda. Pure python + # wheels still will be compatible because they have no platform + # dependencies. compatible_wheels = set() incompatible_wheels = set() sdists = set() @@ -343,7 +371,8 @@ def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. LOG.debug("Downloading missing wheels: %s", packages) lambda_abi = get_lambda_abi(self.runtime) - self._pip.download_manylinux_wheels([pkg.identifier for pkg in packages], directory, lambda_abi) + platform = "manylinux2014_aarch64" if self.architecture == ARM64 else "manylinux2014_x86_64" + self._pip.download_manylinux_wheels([pkg.identifier for pkg in packages], directory, lambda_abi, platform) def _build_sdists(self, sdists, directory, compile_c=True): LOG.debug("Build missing wheels from sdists " "(C compiling %s): %s", compile_c, sdists) @@ -399,29 +428,43 @@ def _is_compatible_platform_tag(self, expected_abi, platform): In addition to checking the tag pattern, we also need to verify the glibc version """ - if platform in self._ADDITIONAL_COMPATIBLE_PLATFORM: + if platform in self._COMPATIBLE_PLATFORMS[self.architecture]: return True - elif platform.startswith("manylinux"): - perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform) - m = re.match("manylinux_([0-9]+)_([0-9]+)_(.*)", perennial_tag) - if m is None: - return False - tag_major, tag_minor = [int(x) for x in m.groups()[:2]] - runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC) - if (tag_major, tag_minor) <= (runtime_major, runtime_minor): - # glibc version is compatible with Lambda Runtime - return True - return False + + arch = "aarch64" if self.architecture == ARM64 else "x86_64" + + # Verify the tag pattern + # Try to get the matching value for legacy values or keep the current + perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform) + + match = re.match("manylinux_([0-9]+)_([0-9]+)_" + arch, perennial_tag) + if match is None: + return False + + # Get the glibc major and minor versions and compare them with the expected ABI + # platform: manylinux_2_17_aarch64 -> 2 and 17 + # expected_abi: cp37m -> compat glibc -> 2 and 17 + # -> Compatible + tag_major, tag_minor = [int(x) for x in match.groups()[:2]] + runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC) + + return (tag_major, tag_minor) <= (runtime_major, runtime_minor) def _iter_all_compatibility_tags(self, wheel): """ Generates all possible combination of tag sets as described in PEP 425 https://www.python.org/dev/peps/pep-0425/#compressed-tag-sets """ + # ex: wheel = numpy-1.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64 implementation_tag, abi_tag, platform_tag = wheel.split("-")[-3:] + # cp38, cp38, manylinux_2_17_aarch64.manylinux2014_aarch64 for implementation in implementation_tag.split("."): + # cp38 for abi in abi_tag.split("."): + # cp38 for platform in platform_tag.split("."): + # manylinux_2_17_aarch64 + # manylinux2014_aarch64 yield (implementation, abi, platform) def _apply_wheel_allowlist(self, compatible_wheels, incompatible_wheels): @@ -693,22 +736,22 @@ def download_all_dependencies(self, requirements_filename, directory): # complain at deployment time. self.build_wheel(wheel_package_path, directory) - def download_manylinux_wheels(self, packages, directory, lambda_abi): + def download_manylinux_wheels(self, packages, directory, lambda_abi, platform="manylinux2014_x86_64"): """Download wheel files for manylinux for all the given packages.""" # If any one of these dependencies fails pip will bail out. Since we # are only interested in all the ones we can download, we need to feed # each package to pip individually. The return code of pip doesn't # matter here since we will inspect the working directory to see which # wheels were downloaded. We are only interested in wheel files - # compatible with lambda, which means manylinux1_x86_64 platform and - # cpython implementation. The compatible abi depends on the python + # compatible with Lambda, which depends on the function architecture, + # and cpython implementation. The compatible abi depends on the python # version and is checked later. for package in packages: arguments = [ "--only-binary=:all:", "--no-deps", "--platform", - "manylinux2014_x86_64", + platform, "--implementation", "cp", "--abi", diff --git a/aws_lambda_builders/workflows/python_pip/validator.py b/aws_lambda_builders/workflows/python_pip/validator.py index d31a929bc..f3cd35808 100644 --- a/aws_lambda_builders/workflows/python_pip/validator.py +++ b/aws_lambda_builders/workflows/python_pip/validator.py @@ -6,37 +6,40 @@ import os import subprocess +from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.exceptions import MisMatchRuntimeError from .utils import OSUtils LOG = logging.getLogger(__name__) -class PythonRuntimeValidator(object): - SUPPORTED_RUNTIMES = {"python2.7", "python3.6", "python3.7", "python3.8", "python3.9"} - - def __init__(self, runtime): +class PythonRuntimeValidator(RuntimeValidator): + def __init__(self, runtime, architecture): + super(PythonRuntimeValidator, self).__init__(runtime, architecture) self.language = "python" - self.runtime = runtime self._valid_runtime_path = None - def has_runtime(self): - """ - Checks if the runtime is supported. - :param string runtime: Runtime to check - :return bool: True, if the runtime is supported. - """ - return self.runtime in self.SUPPORTED_RUNTIMES - def validate(self, runtime_path): """ Checks if the language supplied matches the required lambda runtime - :param string runtime_path: runtime to check eg: /usr/bin/python3.6 - :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime + + Parameters + ---------- + runtime_path : str + runtime to check eg: /usr/bin/go + + Returns + ------- + str + runtime_path, runtime to check eg: /usr/bin/python3.6 + + Raises + ------ + MisMatchRuntimeError + Raise runtime is not support or runtime does not support architecture. """ - if not self.has_runtime(): - LOG.warning("'%s' runtime is not " "a supported runtime", self.runtime) - return + + runtime_path = super(PythonRuntimeValidator, self).validate(runtime_path) cmd = self._validate_python_cmd(runtime_path) diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index c6ce6861d..57a3e66bf 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -91,6 +91,7 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim runtime, self.dependencies_dir, binaries=self.binaries, + architecture=self.architecture, ) ) # if dependencies folder is provided, copy dependencies from dependencies folder to build folder @@ -105,4 +106,4 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) def get_validators(self): - return [PythonRuntimeValidator(runtime=self.runtime)] + return [PythonRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/requirements/dev.txt b/requirements/dev.txt index c4ca79eef..750cd6f5c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -17,6 +17,8 @@ mock==4.0.2; python_version >= '3.6' parameterized==0.7.4 pathlib2==2.3.2; python_version<"3.4" futures==3.2.0; python_version<"3.2.3" +pyelftools~=0.27 # Used to verify the generated Go binary architecture in integration tests (utils.py) + # tempfile backport for < 3.6 backports.tempfile==1.0; python_version<"3.7" diff --git a/tests/functional/test_builder.py b/tests/functional/test_builder.py index 9561ef442..edc31a348 100644 --- a/tests/functional/test_builder.py +++ b/tests/functional/test_builder.py @@ -51,6 +51,7 @@ def test_run_hello_workflow_with_exec_paths(self): self.artifacts_dir, self.scratch_dir, "/ignored", + "python3.8", executable_search_paths=[str(pathlib.Path(sys.executable).parent)], ) diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index 6f2b4fe5b..881cd9ca5 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -74,11 +74,12 @@ def test_run_hello_workflow_with_backcompat(self, flavor, protocol_version): "artifacts_dir": self.artifacts_dir, "scratch_dir": self.scratch_dir, "manifest_path": "/ignored", - "runtime": "ignored", + "runtime": "python3.8", "optimizations": {}, "options": {}, "download_dependencies": False, "dependencies_dir": "/ignored-dep", + "architecture": "x86_64", }, } diff --git a/tests/functional/test_utils.py b/tests/functional/test_utils.py index f19604dff..bb0cea0ae 100644 --- a/tests/functional/test_utils.py +++ b/tests/functional/test_utils.py @@ -4,7 +4,7 @@ from unittest import TestCase -from aws_lambda_builders.utils import copytree +from aws_lambda_builders.utils import copytree, get_goarch class TestCopyTree(TestCase): @@ -40,6 +40,11 @@ def test_must_respect_excludes_list(self): self.assertEqual(set(os.listdir(os.path.join(self.dest, "a"))), {"c"}) self.assertEqual(set(os.listdir(os.path.join(self.dest, "a"))), {"c"}) + def test_must_return_valid_go_architecture(self): + self.assertEqual(get_goarch("arm64"), "arm64") + self.assertEqual(get_goarch("x86_64"), "amd64") + self.assertEqual(get_goarch(""), "amd64") + def file(*args): path = os.path.join(*args) diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 2b627fac0..643b08ad3 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -8,6 +8,7 @@ import pytest import mock +from aws_lambda_builders.architecture import ARM64 from aws_lambda_builders.workflows.python_pip.packager import PipRunner, UnsupportedPackageError from aws_lambda_builders.workflows.python_pip.packager import DependencyBuilder from aws_lambda_builders.workflows.python_pip.packager import Package @@ -200,10 +201,10 @@ def _write_requirements_txt(self, packages, directory): with open(filepath, "w") as f: f.write(contents) - def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner): + def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner, **kwargs): appdir = str(_create_app_structure(tmpdir)) self._write_requirements_txt(reqs, appdir) - builder = DependencyBuilder(OSUtils(), "python3.6", runner) + builder = DependencyBuilder(OSUtils(), "python3.6", runner, **kwargs) return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): @@ -514,6 +515,29 @@ def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner): for req in reqs: assert req in installed_packages + def test_can_get_arm64_whls(self, tmpdir, osutils, pip_runner): + reqs = ["foo", "bar", "baz"] + pip, runner = pip_runner + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner, architecture=ARM64) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.packages_to_download( + expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], + packages=[ + "foo-1.0-cp36-none-any.whl", + "bar-1.2-cp36-none-manylinux2014_aarch64.whl", + "baz-1.5-cp36-cp36m-manylinux2014_aarch64.whl", + ], + ) + + site_packages = os.path.join(appdir, ".chalice.", "site-packages") + with osutils.tempdir() as scratch_dir: + builder.build_site_packages(requirements_file, site_packages, scratch_dir) + installed_packages = os.listdir(site_packages) + + pip.validate() + for req in reqs: + assert req in installed_packages + def test_does_fail_on_invalid_local_package(self, tmpdir, osutils, pip_runner): reqs = ["../foo"] pip, runner = pip_runner diff --git a/tests/integration/workflows/dotnet_clipackage/test_dotnet.py b/tests/integration/workflows/dotnet_clipackage/test_dotnet.py index 2565aae42..eb6e6cf46 100644 --- a/tests/integration/workflows/dotnet_clipackage/test_dotnet.py +++ b/tests/integration/workflows/dotnet_clipackage/test_dotnet.py @@ -1,30 +1,58 @@ import os import shutil import tempfile +import json +try: + import pathlib +except ImportError: + import pathlib2 as pathlib from unittest import TestCase from aws_lambda_builders.builder import LambdaBuilder +from aws_lambda_builders.architecture import ARM64, X86_64 -class TestDotnetDep(TestCase): +class TestDotnetBase(TestCase): + """ + Base class for dotnetcore tests + """ + TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata") def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() - self.builder = LambdaBuilder(language="dotnet", dependency_manager="cli-package", application_framework=None) - - self.runtime = "dotnetcore2.1" + self.runtime = "dotnetcore2.1" # default to 2.1 def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) + def verify_architecture(self, deps_file_name, expected_architecture): + deps_file = pathlib.Path(self.artifacts_dir, deps_file_name) + + if not deps_file.exists(): + self.fail("Failed verifying architecture, {} file not found".format(deps_file_name)) + + with open(str(deps_file)) as f: + deps_json = json.loads(f.read()) + version = self.runtime[-3:] + target_name = ".NETCoreApp,Version=v{}/{}".format(version, expected_architecture) + target = deps_json.get("runtimeTarget").get("name") + + self.assertEqual(target, target_name) + + +class TestDotnet21(TestDotnetBase): + """ + Tests for dotnetcore 2.1 + """ + def test_with_defaults_file(self): - source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile") + source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile2.1") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime) @@ -41,6 +69,7 @@ def test_with_defaults_file(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64") def test_require_parameters(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "RequireParameters") @@ -67,3 +96,78 @@ def test_require_parameters(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + self.verify_architecture("RequireParameters.deps.json", "linux-x64") + + +class TestDotnet31(TestDotnetBase): + """ + Tests for dotnetcore 3.1 + """ + + def setUp(self): + super(TestDotnet31, self).setUp() + self.runtime = "dotnetcore3.1" + + def test_with_defaults_file(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1") + + self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime) + + expected_files = { + "Amazon.Lambda.Core.dll", + "Amazon.Lambda.Serialization.Json.dll", + "Newtonsoft.Json.dll", + "WithDefaultsFile.deps.json", + "WithDefaultsFile.dll", + "WithDefaultsFile.pdb", + "WithDefaultsFile.runtimeconfig.json", + } + + output_files = set(os.listdir(self.artifacts_dir)) + + self.assertEqual(expected_files, output_files) + self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64") + + def test_with_defaults_file_x86(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1") + + self.builder.build( + source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=X86_64 + ) + + expected_files = { + "Amazon.Lambda.Core.dll", + "Amazon.Lambda.Serialization.Json.dll", + "Newtonsoft.Json.dll", + "WithDefaultsFile.deps.json", + "WithDefaultsFile.dll", + "WithDefaultsFile.pdb", + "WithDefaultsFile.runtimeconfig.json", + } + + output_files = set(os.listdir(self.artifacts_dir)) + + self.assertEqual(expected_files, output_files) + self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64") + + def test_with_defaults_file_arm64(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1") + + self.builder.build( + source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=ARM64 + ) + + expected_files = { + "Amazon.Lambda.Core.dll", + "Amazon.Lambda.Serialization.Json.dll", + "Newtonsoft.Json.dll", + "WithDefaultsFile.deps.json", + "WithDefaultsFile.dll", + "WithDefaultsFile.pdb", + "WithDefaultsFile.runtimeconfig.json", + } + + output_files = set(os.listdir(self.artifacts_dir)) + + self.assertEqual(expected_files, output_files) + self.verify_architecture("WithDefaultsFile.deps.json", "linux-arm64") diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/Function.cs b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/Function.cs similarity index 100% rename from tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/Function.cs rename to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/Function.cs diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/WithDefaultsFile.csproj b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/WithDefaultsFile.csproj similarity index 100% rename from tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/WithDefaultsFile.csproj rename to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/WithDefaultsFile.csproj diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/aws-lambda-tools-defaults.json b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/aws-lambda-tools-defaults.json similarity index 100% rename from tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/aws-lambda-tools-defaults.json rename to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/aws-lambda-tools-defaults.json diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/Function.cs b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/Function.cs new file mode 100644 index 000000000..23fc86994 --- /dev/null +++ b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/Function.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +using Amazon.Lambda.Core; + +// Assembly attribute to enable the Lambda function's JSON input to be converted into a .NET class. +[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.Json.JsonSerializer))] + +namespace WithDefaultsFile +{ + public class Function + { + + /// + /// A simple function that takes a string and does a ToUpper + /// + /// + /// + /// + public string FunctionHandler(string input, ILambdaContext context) + { + return input?.ToUpper(); + } + } +} diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/WithDefaultsFile.csproj b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/WithDefaultsFile.csproj new file mode 100644 index 000000000..fc380827b --- /dev/null +++ b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/WithDefaultsFile.csproj @@ -0,0 +1,11 @@ + + + netcoreapp3.1 + true + Lambda + + + + + + \ No newline at end of file diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/aws-lambda-tools-defaults.json b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/aws-lambda-tools-defaults.json new file mode 100644 index 000000000..56786ebd2 --- /dev/null +++ b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/aws-lambda-tools-defaults.json @@ -0,0 +1,16 @@ +{ + "Information": [ + "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", + "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", + "dotnet lambda help", + "All the command line options for the Lambda command can be specified in this file." + ], + "profile": "", + "region": "", + "configuration": "Release", + "framework": "netcoreapp3.1", + "function-runtime": "dotnetcore3.1", + "function-memory-size": 256, + "function-timeout": 30, + "function-handler": "WithDefaultsFile::WithDefaultsFile.Function::FunctionHandler" +} \ No newline at end of file diff --git a/tests/integration/workflows/go_modules/__init__.py b/tests/integration/workflows/go_modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/go_modules/test_go.py b/tests/integration/workflows/go_modules/test_go.py index 3fd79415e..17f0aad46 100644 --- a/tests/integration/workflows/go_modules/test_go.py +++ b/tests/integration/workflows/go_modules/test_go.py @@ -2,11 +2,18 @@ import shutil import tempfile +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + from unittest import TestCase from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError +from tests.integration.workflows.go_modules.utils import get_executable_arch + class TestGoWorkflow(TestCase): """ @@ -66,3 +73,45 @@ def test_fails_if_modules_cannot_resolve_dependencies(self): options={"artifact_executable_name": "failed"}, ) self.assertIn("GoModulesBuilder:Build - Builder Failed: ", str(ctx.exception)) + + def test_build_defaults_to_x86_architecture(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_x86_architecture = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-amd64"}, + ) + pathname = Path(self.artifacts_dir, "no-deps-main-amd64") + self.assertEqual(get_executable_arch(pathname), "x64") + + def test_builds_x86_architecture(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_x86_architecture = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-amd64"}, + architecture="x86_64", + ) + pathname = Path(self.artifacts_dir, "no-deps-main-amd64") + self.assertEqual(get_executable_arch(pathname), "x64") + + def test_builds_arm64_architecture(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_arm_architecture = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-arm64"}, + architecture="arm64", + ) + + pathname = Path(self.artifacts_dir, "no-deps-main-arm64") + self.assertEqual(get_executable_arch(pathname), "AArch64") diff --git a/tests/integration/workflows/go_modules/utils.py b/tests/integration/workflows/go_modules/utils.py new file mode 100644 index 000000000..a5caae61e --- /dev/null +++ b/tests/integration/workflows/go_modules/utils.py @@ -0,0 +1,20 @@ +from elftools.elf.elffile import ELFFile + + +def get_executable_arch(path): + """ + Returns the architecture of an executable binary + + Parameters + ---------- + path : str + path to the Go binaries generated + + Returns + ------- + str + Architecture type of the generated binaries + """ + with open(str(path), "rb") as f: + e = ELFFile(f) + return e.get_machine_arch() diff --git a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py index 4cdb41e62..37910007d 100644 --- a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py +++ b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py @@ -1,4 +1,5 @@ import logging +import mock import os import shutil import tempfile @@ -9,7 +10,7 @@ from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError -workflow_logger = logging.getLogger("aws_lambda_builders.workflows.nodejs_npm.workflow") +logger = logging.getLogger("aws_lambda_builders.workflows.nodejs_npm.workflow") class TestNodejsNpmWorkflow(TestCase): @@ -27,7 +28,7 @@ def setUp(self): self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps") self.builder = LambdaBuilder(language="nodejs", dependency_manager="npm", application_framework=None) - self.runtime = "nodejs8.10" + self.runtime = "nodejs12.x" def tearDown(self): shutil.rmtree(self.artifacts_dir) @@ -49,6 +50,23 @@ def test_builds_project_without_dependencies(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + def test_builds_project_without_manifest(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-manifest") + + with mock.patch.object(logger, "warning") as mock_warning: + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + ) + + expected_files = {"app.js"} + output_files = set(os.listdir(self.artifacts_dir)) + mock_warning.assert_called_once_with("package.json file not found. Continuing the build without dependencies.") + self.assertEqual(expected_files, output_files) + def test_builds_project_and_excludes_hidden_aws_sam(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "excluded-files") @@ -182,7 +200,7 @@ def test_builds_project_with_remote_dependencies_without_download_dependencies_a def test_builds_project_with_remote_dependencies_without_download_dependencies_without_dependencies_dir(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps") - with mock.patch.object(workflow_logger, "info") as mock_info: + with mock.patch.object(logger, "info") as mock_info: self.builder.build( source_dir, self.artifacts_dir, diff --git a/tests/integration/workflows/nodejs_npm/testdata/no-manifest/app.js b/tests/integration/workflows/nodejs_npm/testdata/no-manifest/app.js new file mode 100644 index 000000000..67021f79c --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/no-manifest/app.js @@ -0,0 +1,2 @@ +const HELLO_WORLD = "Hello world!" +console.log(HELLO_WORLD) \ No newline at end of file diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 44963c028..fa21ecfc6 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -55,6 +55,25 @@ def tearDown(self): shutil.rmtree(self.scratch_dir) shutil.rmtree(self.dependencies_dir) + def check_architecture_in(self, library, architectures): + wheel_architectures = [] + with open(os.path.join(self.artifacts_dir, library, "WHEEL")) as wheel: + for line in wheel: + if line.startswith("Tag:"): + wheel_architecture = line.rstrip().split("-")[-1] + if wheel_architecture in architectures: + return # Success + wheel_architectures.append(wheel_architecture) + self.fail( + "Wheel architectures [{}] not found in [{}]".format( + ", ".join(wheel_architectures), ", ".join(architectures) + ) + ) + + # Temporarily skipping this test in Windows + # Fails and we are not sure why: pip version/multiple Python versions in path/os/pypa issue? + # TODO: Revisit when we deprecate Python2 + @skipIf(IS_WINDOWS, "Skip in windows tests") def test_must_build_python_project(self): self.builder.build( self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime=self.runtime @@ -63,12 +82,37 @@ def test_must_build_python_project(self): if self.runtime == "python2.7": expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) elif self.runtime == "python3.6": + self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) else: + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) + output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + def test_must_build_python_project_with_arm_architecture(self): + if self.runtime != "python3.8": + self.skipTest("{} is not supported on ARM architecture".format(self.runtime)) + ### Check the wheels + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime, + architecture="arm64", + ) + expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.20.3.dist-info"}) + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"]) + + # Temporarily skipping this test in Windows + # Fails and we are not sure why: pip version/multiple Python versions in path/os/pypa issue? + # TODO: Revisit when we deprecate Python2 + @skipIf(IS_WINDOWS, "Skip in windows tests") def test_mismatch_runtime_python_project(self): # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python2.7 # is still accessible within a python 3 virtualenv. diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 2c040b7f3..4a490852b 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -135,6 +135,7 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, "artifacts_dir", "scratch_dir", "manifest_path", + architecture="arm64", runtime="runtime", optimizations="optimizations", options="options", @@ -149,6 +150,7 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, "artifacts_dir", "scratch_dir", "manifest_path", + architecture="arm64", runtime="runtime", optimizations="optimizations", options="options", diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py index e1cc8844a..710edd8ab 100644 --- a/tests/unit/test_validator.py +++ b/tests/unit/test_validator.py @@ -1,14 +1,29 @@ from unittest import TestCase from aws_lambda_builders.validator import RuntimeValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError class TestRuntimeValidator(TestCase): def setUp(self): - self.validator = RuntimeValidator(runtime="chitti2.0") + self.validator = RuntimeValidator(runtime="python3.8", architecture="arm64") def test_inits(self): - self.assertEqual(self.validator.runtime, "chitti2.0") + self.assertEqual(self.validator.runtime, "python3.8") + self.assertEqual(self.validator.architecture, "arm64") def test_validate_runtime(self): - self.validator.validate("/usr/bin/chitti") + self.validator.validate("/usr/bin/python3.8") + self.assertEqual(self.validator._runtime_path, "/usr/bin/python3.8") + + def test_validate_with_unsupported_runtime(self): + validator = RuntimeValidator(runtime="unknown_runtime", architecture="x86_64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate("/usr/bin/unknown_runtime") + + def test_validate_with_runtime_and_incompatible_architecture(self): + runtime_list = ["dotnetcore2.1", "nodejs10.x", "ruby2.5", "python3.6", "python3.7", "python2.7"] + for runtime in runtime_list: + validator = RuntimeValidator(runtime=runtime, architecture="arm64") + with self.assertRaises(UnsupportedArchitectureError): + validator.validate("/usr/bin/{}".format(runtime)) diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index 149a82fe7..47013c0ed 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -13,7 +13,13 @@ from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY -from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError +from aws_lambda_builders.exceptions import ( + WorkflowFailedError, + WorkflowUnknownError, + MisMatchRuntimeError, + UnsupportedRuntimeError, + UnsupportedArchitectureError, +) from aws_lambda_builders.actions import ActionFailedError @@ -112,6 +118,7 @@ def test_must_initialize_variables(self): self.assertEqual(self.work.executable_search_paths, [str(sys.executable)]) self.assertEqual(self.work.optimizations, {"a": "b"}) self.assertEqual(self.work.options, {"c": "d"}) + self.assertEqual(self.work.architecture, "x86_64") class TestBaseWorkflow_is_supported(TestCase): @@ -132,6 +139,7 @@ def setUp(self): executable_search_paths=[], optimizations={"a": "b"}, options={"c": "d"}, + architecture="arm64", ) def test_must_ignore_manifest_if_not_provided(self): @@ -150,6 +158,9 @@ def test_must_match_manifest_name_from_path(self): self.assertTrue(self.work.is_supported()) + def test_must_match_architecture_type(self): + self.assertEqual(self.work.architecture, "arm64") + def test_must_fail_if_manifest_not_in_list(self): self.work.SUPPORTED_MANIFESTS = ["someother_manifest"] @@ -269,7 +280,7 @@ def test_must_raise_if_action_crashed(self): self.assertIn("somevalueerror", str(ctx.exception)) def test_supply_executable_path(self): - # Run workflow with supplied executable path to search for executables. + # Run workflow with supplied executable path to search for executables action_mock = Mock() self.work = self.MyWorkflow( @@ -287,6 +298,59 @@ def test_supply_executable_path(self): self.work.run() + def test_must_raise_for_unknown_runtime(self): + action_mock = Mock() + validator_mock = Mock() + validator_mock.validate = Mock() + validator_mock.validate = MagicMock(side_effect=UnsupportedRuntimeError(runtime="runtime")) + + resolver_mock = Mock() + resolver_mock.exec_paths = ["/usr/bin/binary"] + binaries_mock = Mock() + binaries_mock.return_value = [] + + self.work.get_validators = lambda: validator_mock + self.work.get_resolvers = lambda: resolver_mock + self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] + self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} + with self.assertRaises(WorkflowFailedError) as ex: + self.work.run() + + self.assertIn("Runtime runtime is not supported", str(ex.exception)) + + def test_must_raise_for_incompatible_runtime_and_architecture(self): + self.work = self.MyWorkflow( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="python2.7", + executable_search_paths=[str(pathlib.Path(os.getcwd()).parent)], + optimizations={"a": "b"}, + options={"c": "d"}, + ) + action_mock = Mock() + validator_mock = Mock() + validator_mock.validate = Mock() + validator_mock.validate = MagicMock( + side_effect=UnsupportedArchitectureError(runtime="python2.7", architecture="arm64") + ) + + resolver_mock = Mock() + resolver_mock.exec_paths = ["/usr/bin/binary"] + binaries_mock = Mock() + binaries_mock.return_value = [] + + self.work.architecture = "arm64" + self.work.get_validators = lambda: validator_mock + self.work.get_resolvers = lambda: resolver_mock + self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] + self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} + with self.assertRaises(WorkflowFailedError) as ex: + self.work.run() + + self.assertIn("Architecture arm64 is not supported for runtime python2.7", str(ex.exception)) + class TestBaseWorkflow_repr(TestCase): class MyWorkflow(BaseWorkflow): diff --git a/tests/unit/workflows/custom_make/test_workflow.py b/tests/unit/workflows/custom_make/test_workflow.py index 856f39c76..8775c11b0 100644 --- a/tests/unit/workflows/custom_make/test_workflow.py +++ b/tests/unit/workflows/custom_make/test_workflow.py @@ -1,5 +1,6 @@ from unittest import TestCase +from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.exceptions import WorkflowFailedError from aws_lambda_builders.workflows.custom_make.workflow import CustomMakeWorkflow @@ -29,3 +30,19 @@ def test_workflow_sets_up_make_actions_no_options(self): with self.assertRaises(WorkflowFailedError): CustomMakeWorkflow("source", "artifacts", "scratch_dir", "manifest") + + def test_must_validate_architecture(self): + workflow = CustomMakeWorkflow( + "source", "artifacts", "scratch_dir", "manifest", options={"build_logical_id": "hello"} + ) + workflow_with_arm = CustomMakeWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + options={"build_logical_id": "hello"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/dotnet_clipackage/test_actions.py b/tests/unit/workflows/dotnet_clipackage/test_actions.py index 1df2694a3..4b27d9b21 100644 --- a/tests/unit/workflows/dotnet_clipackage/test_actions.py +++ b/tests/unit/workflows/dotnet_clipackage/test_actions.py @@ -1,11 +1,12 @@ from unittest import TestCase -from concurrent.futures import ThreadPoolExecutor -from mock import patch import os import platform +from concurrent.futures import ThreadPoolExecutor +from mock import patch from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.workflows.dotnet_clipackage.dotnetcli import DotnetCLIExecutionError from aws_lambda_builders.workflows.dotnet_clipackage.actions import GlobalToolInstallAction, RunPackageAction @@ -66,46 +67,89 @@ class TestRunPackageAction(TestCase): @patch("aws_lambda_builders.workflows.dotnet_clipackage.utils.OSUtils") def setUp(self, MockSubprocessDotnetCLI, MockOSUtils): self.subprocess_dotnet = MockSubprocessDotnetCLI.return_value - self.os_utils = MockOSUtils - self.source_dir = os.path.join("/source_dir") - self.artifacts_dir = os.path.join("/artifacts_dir") - self.scratch_dir = os.path.join("/scratch_dir") + self.os_utils = MockOSUtils.return_value + self.source_dir = "/source_dir" + self.artifacts_dir = "/artifacts_dir" + self.scratch_dir = "/scratch_dir" def tearDown(self): self.subprocess_dotnet.reset_mock() + self.os_utils.reset_mock() def test_build_package(self): mode = "Release" options = {} action = RunPackageAction( - self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, os_utils=self.os_utils + ) + + action.execute() + + zip_path = os.path.join(self.artifacts_dir, "source_dir.zip") + + self.subprocess_dotnet.run.assert_called_once_with( + ["lambda", "package", "--output-package", zip_path, "--msbuild-parameters", "--runtime linux-x64"], + cwd="/source_dir", + ) + + def test_build_package_x86(self): + mode = "Release" + + options = {} + action = RunPackageAction( + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, X86_64, os_utils=self.os_utils + ) + + action.execute() + + zip_path = os.path.join(self.artifacts_dir, "source_dir.zip") + + self.subprocess_dotnet.run.assert_called_once_with( + ["lambda", "package", "--output-package", zip_path, "--msbuild-parameters", "--runtime linux-x64"], + cwd="/source_dir", + ) + + def test_build_package_arm64(self): + mode = "Release" + + options = {} + action = RunPackageAction( + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, ARM64, os_utils=self.os_utils ) action.execute() - zipFilePath = os.path.join("/", "artifacts_dir", "source_dir.zip") + zip_path = os.path.join(self.artifacts_dir, "source_dir.zip") self.subprocess_dotnet.run.assert_called_once_with( - ["lambda", "package", "--output-package", zipFilePath], cwd="/source_dir" + ["lambda", "package", "--output-package", zip_path, "--msbuild-parameters", "--runtime linux-arm64"], + cwd="/source_dir", ) def test_build_package_arguments(self): mode = "Release" options = {"--framework": "netcoreapp2.1"} action = RunPackageAction( - self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, os_utils=self.os_utils ) action.execute() - if platform.system().lower() == "windows": - zipFilePath = "/artifacts_dir\\source_dir.zip" - else: - zipFilePath = "/artifacts_dir/source_dir.zip" + zip_path = self.artifacts_dir + ("\\" if platform.system().lower() == "windows" else "/") + "source_dir.zip" self.subprocess_dotnet.run.assert_called_once_with( - ["lambda", "package", "--output-package", zipFilePath, "--framework", "netcoreapp2.1"], cwd="/source_dir" + [ + "lambda", + "package", + "--output-package", + zip_path, + "--msbuild-parameters", + "--runtime linux-x64", + "--framework", + "netcoreapp2.1", + ], + cwd="/source_dir", ) def test_build_error(self): @@ -114,7 +158,7 @@ def test_build_error(self): self.subprocess_dotnet.run.side_effect = DotnetCLIExecutionError(message="Failed Package") options = {} action = RunPackageAction( - self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, os_utils=self.os_utils ) self.assertRaises(ActionFailedError, action.execute) @@ -123,13 +167,23 @@ def test_debug_configuration_set(self): mode = "Debug" options = None action = RunPackageAction( - self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, os_utils=self.os_utils ) - zipFilePath = os.path.join("/", "artifacts_dir", "source_dir.zip") + zip_path = os.path.join("/", "artifacts_dir", "source_dir.zip") action.execute() self.subprocess_dotnet.run.assert_called_once_with( - ["lambda", "package", "--output-package", zipFilePath, "--configuration", "Debug"], cwd="/source_dir" + [ + "lambda", + "package", + "--output-package", + zip_path, + "--msbuild-parameters", + "--runtime linux-x64", + "--configuration", + "Debug", + ], + cwd="/source_dir", ) diff --git a/tests/unit/workflows/dotnet_clipackage/test_workflow.py b/tests/unit/workflows/dotnet_clipackage/test_workflow.py index f520c40dd..c3cb9a9f3 100644 --- a/tests/unit/workflows/dotnet_clipackage/test_workflow.py +++ b/tests/unit/workflows/dotnet_clipackage/test_workflow.py @@ -1,5 +1,6 @@ from unittest import TestCase +from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.workflows.dotnet_clipackage.workflow import DotnetCliPackageWorkflow from aws_lambda_builders.workflows.dotnet_clipackage.actions import GlobalToolInstallAction, RunPackageAction @@ -11,3 +12,23 @@ def test_actions(self): self.assertIsInstance(workflow.actions[0], GlobalToolInstallAction) self.assertIsInstance(workflow.actions[1], RunPackageAction) + + def test_must_validate_architecture(self): + workflow = DotnetCliPackageWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = DotnetCliPackageWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/go_dep/test_actions.py b/tests/unit/workflows/go_dep/test_actions.py index 33978182a..e0f73fd16 100644 --- a/tests/unit/workflows/go_dep/test_actions.py +++ b/tests/unit/workflows/go_dep/test_actions.py @@ -2,6 +2,7 @@ from mock import patch from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.workflows.go_dep.actions import DepEnsureAction, GoBuildAction from aws_lambda_builders.workflows.go_dep.subproc_exec import ExecutionError @@ -67,3 +68,33 @@ def test_fails_go_build(self, SubProcMock): action.execute() self.assertEqual(raised.exception.args[0], "Exec Failed: boom!") + + @patch("aws_lambda_builders.workflows.go_dep.subproc_exec.SubprocessExec") + def test_runs_go_build_with_arm_architecture(self, SubProcMock): + """ + tests the happy path of running `dep ensure` + """ + + sub_proc_go = SubProcMock.return_value + action = GoBuildAction("base", "source", "output", sub_proc_go, ARM64, env={}) + + action.execute() + + sub_proc_go.run.assert_called_with( + ["build", "-o", "output", "source"], cwd="source", env={"GOOS": "linux", "GOARCH": "arm64"} + ) + + @patch("aws_lambda_builders.workflows.go_dep.subproc_exec.SubprocessExec") + def test_fails_go_build_with_arm_architecture(self, SubProcMock): + """ + tests failure, something being returned on stderr + """ + + sub_proc_go = SubProcMock.return_value + sub_proc_go.run.side_effect = ExecutionError(message="boom!") + action = GoBuildAction("base", "source", "output", sub_proc_go, "unknown_architecture", env={}) + + with self.assertRaises(ActionFailedError) as raised: + action.execute() + + self.assertEqual(raised.exception.args[0], "Exec Failed: boom!") diff --git a/tests/unit/workflows/go_dep/test_workflow.py b/tests/unit/workflows/go_dep/test_workflow.py index d4c14518a..76aa4eefc 100644 --- a/tests/unit/workflows/go_dep/test_workflow.py +++ b/tests/unit/workflows/go_dep/test_workflow.py @@ -1,5 +1,7 @@ from unittest import TestCase +from aws_lambda_builders.architecture import X86_64, ARM64 + from aws_lambda_builders.workflows.go_dep.workflow import GoDepWorkflow from aws_lambda_builders.workflows.go_dep.actions import DepEnsureAction, GoBuildAction @@ -14,6 +16,27 @@ def test_workflow_sets_up_workflow(self): workflow = GoDepWorkflow( "source", "artifacts", "scratch", "manifest", options={"artifact_executable_name": "foo"} ) + self.assertEqual(len(workflow.actions), 2) self.assertIsInstance(workflow.actions[0], DepEnsureAction) self.assertIsInstance(workflow.actions[1], GoBuildAction) + + def test_must_validate_architecture(self): + workflow = GoDepWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = GoDepWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/go_modules/test_builder.py b/tests/unit/workflows/go_modules/test_builder.py index 1c205af46..bc3d24bbb 100644 --- a/tests/unit/workflows/go_modules/test_builder.py +++ b/tests/unit/workflows/go_modules/test_builder.py @@ -60,3 +60,14 @@ def test_debug_configuration_set(self): stderr="PIPE", stdout="PIPE", ) + + def test_debug_configuration_set_with_arm_architecture(self): + self.under_test = GoModulesBuilder(self.osutils, self.binaries, "Debug", "arm64") + self.under_test.build("source_dir", "output_path") + self.osutils.popen.assert_called_with( + ["/path/to/go", "build", "-gcflags", "all=-N -l", "-o", "output_path", "source_dir"], + cwd="source_dir", + env={"GOOS": "linux", "GOARCH": "arm64"}, + stderr="PIPE", + stdout="PIPE", + ) diff --git a/tests/unit/workflows/go_modules/test_validator.py b/tests/unit/workflows/go_modules/test_validator.py index 221fea415..14acd8698 100644 --- a/tests/unit/workflows/go_modules/test_validator.py +++ b/tests/unit/workflows/go_modules/test_validator.py @@ -5,6 +5,7 @@ from aws_lambda_builders.exceptions import MisMatchRuntimeError from aws_lambda_builders.workflows.go_modules.validator import GoRuntimeValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError class MockSubProcess(object): @@ -19,16 +20,12 @@ def communicate(self): class TestGoRuntimeValidator(TestCase): def setUp(self): - self.validator = GoRuntimeValidator(runtime="go1.x") - - @parameterized.expand(["go1.x"]) - def test_supported_runtimes(self, runtime): - validator = GoRuntimeValidator(runtime=runtime) - self.assertTrue(validator.has_runtime()) + self.validator = GoRuntimeValidator(runtime="go1.x", architecture="arm64") def test_runtime_validate_unsupported_language_fail_open(self): - validator = GoRuntimeValidator(runtime="go2.x") - validator.validate(runtime_path="/usr/bin/go2") + validator = GoRuntimeValidator(runtime="go2.x", architecture="arm64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate(runtime_path="/usr/bin/go2") @parameterized.expand( [ diff --git a/tests/unit/workflows/go_modules/test_workflow.py b/tests/unit/workflows/go_modules/test_workflow.py index ddab1449d..86e42e306 100644 --- a/tests/unit/workflows/go_modules/test_workflow.py +++ b/tests/unit/workflows/go_modules/test_workflow.py @@ -2,6 +2,7 @@ from aws_lambda_builders.workflows.go_modules.workflow import GoModulesWorkflow from aws_lambda_builders.workflows.go_modules.actions import GoModulesBuildAction +from aws_lambda_builders.architecture import X86_64, ARM64 class TestGoModulesWorkflow(TestCase): @@ -19,5 +20,26 @@ def test_workflow_sets_up_builder_actions(self): runtime="go1.x", options={"artifact_executable_name": "main"}, ) + self.assertEqual(len(workflow.actions), 1) self.assertIsInstance(workflow.actions[0], GoModulesBuildAction) + + def test_must_validate_architecture(self): + workflow = GoModulesWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = GoModulesWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/java_gradle/test_gradle_validator.py b/tests/unit/workflows/java_gradle/test_gradle_validator.py index 93d4e3069..6b3e2d63b 100644 --- a/tests/unit/workflows/java_gradle/test_gradle_validator.py +++ b/tests/unit/workflows/java_gradle/test_gradle_validator.py @@ -3,6 +3,7 @@ from mock import patch, Mock from parameterized import parameterized from aws_lambda_builders.workflows.java_gradle.gradle_validator import GradleValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError class FakePopen(object): @@ -24,51 +25,69 @@ class TestGradleBinaryValidator(TestCase): def setUp(self, MockOSUtils): self.mock_os_utils = MockOSUtils.return_value self.mock_log = Mock() - self.gradle_path = "/path/to/gradle" + self.runtime_path = "/path/to/gradle" self.runtime = "java8" + self.architecture = "x86_64" @parameterized.expand(["1.7.0", "1.8.9", "11.0.0", "12 (Fluff)", "12"]) def test_accepts_any_jvm_mv(self, version): version_string = ("JVM: %s" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils) - self.assertTrue(validator.validate(gradle_path=self.gradle_path)) - self.assertEqual(validator.validated_binary_path, self.gradle_path) + validator = GradleValidator(runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) def test_emits_warning_when_jvm_mv_greater_than_8(self): version_string = "JVM: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(gradle_path=self.gradle_path)) - self.assertEqual(validator.validated_binary_path, self.gradle_path) - self.mock_log.warning.assert_called_with(GradleValidator.MAJOR_VERSION_WARNING, self.gradle_path, "9", "8", "8") + validator = GradleValidator( + runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) + self.mock_log.warning.assert_called_with( + GradleValidator.MAJOR_VERSION_WARNING, self.runtime_path, "9", "8", "8" + ) @parameterized.expand(["1.6.0", "1.7.0", "1.8.9"]) def test_does_not_emit_warning_when_jvm_mv_8_or_less(self, version): version_string = ("JVM: %s" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(gradle_path=self.gradle_path)) - self.assertEqual(validator.validated_binary_path, self.gradle_path) + validator = GradleValidator( + runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) self.mock_log.warning.assert_not_called() def test_emits_warning_when_gradle_excutable_fails(self): version_string = "JVM: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=1)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - validator.validate(gradle_path=self.gradle_path) - self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.gradle_path) + validator = GradleValidator( + runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + validator.validate(runtime_path=self.runtime_path) + self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.runtime_path) def test_emits_warning_when_version_string_not_found(self): version_string = "The Java Version: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=0)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - validator.validate(gradle_path=self.gradle_path) - self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.gradle_path) + validator = GradleValidator( + runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + validator.validate(runtime_path=self.runtime_path) + self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.runtime_path) def test_no_warning_when_jvm_mv_11_and_java11_runtime(self): version_string = "JVM: 11.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = GradleValidator(runtime="java11", os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(gradle_path=self.gradle_path)) - self.assertEqual(validator.validated_binary_path, self.gradle_path) + validator = GradleValidator( + runtime="java11", architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) + + def test_runtime_validate_unsupported_language_fail_open(self): + validator = GradleValidator(runtime="java2.0", architecture="arm64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate(runtime_path="/usr/bin/java2.0") diff --git a/tests/unit/workflows/java_gradle/test_workflow.py b/tests/unit/workflows/java_gradle/test_workflow.py index 889f01ce7..ed6bd13bb 100644 --- a/tests/unit/workflows/java_gradle/test_workflow.py +++ b/tests/unit/workflows/java_gradle/test_workflow.py @@ -6,6 +6,7 @@ from aws_lambda_builders.workflows.java_gradle.actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction from aws_lambda_builders.workflows.java_gradle.gradle_resolver import GradleResolver from aws_lambda_builders.workflows.java_gradle.gradle_validator import GradleValidator +from aws_lambda_builders.architecture import ARM64 class TestJavaGradleWorkflow(TestCase): @@ -16,35 +17,45 @@ class TestJavaGradleWorkflow(TestCase): def test_workflow_sets_up_gradle_actions(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") - self.assertEqual(len(workflow.actions), 2) - self.assertIsInstance(workflow.actions[0], JavaGradleBuildAction) - self.assertIsInstance(workflow.actions[1], JavaGradleCopyArtifactsAction) def test_workflow_sets_up_resolvers(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") - resolvers = workflow.get_resolvers() self.assertEqual(len(resolvers), 1) - self.assertIsInstance(resolvers[0], GradleResolver) def test_workflow_sets_up_validators(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") - validators = workflow.get_validators() self.assertEqual(len(validators), 1) - self.assertIsInstance(validators[0], GradleValidator) def test_computes_correct_build_dir(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") - sha1 = hashlib.sha1() sha1.update(os.path.abspath(workflow.source_dir).encode("utf8")) - expected_build_dir = os.path.join(workflow.scratch_dir, sha1.hexdigest()) - self.assertEqual(expected_build_dir, workflow.build_output_dir) + + def test_must_validate_architecture(self): + workflow = JavaGradleWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = JavaGradleWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/java_maven/test_maven_validator.py b/tests/unit/workflows/java_maven/test_maven_validator.py index fa88481dc..702d789b2 100644 --- a/tests/unit/workflows/java_maven/test_maven_validator.py +++ b/tests/unit/workflows/java_maven/test_maven_validator.py @@ -3,6 +3,7 @@ from mock import patch, Mock from parameterized import parameterized from aws_lambda_builders.workflows.java_maven.maven_validator import MavenValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError class FakePopen(object): @@ -24,52 +25,60 @@ class TestMavenBinaryValidator(TestCase): def setUp(self, MockOSUtils): self.mock_os_utils = MockOSUtils.return_value self.mock_log = Mock() - self.maven_path = "/path/to/maven" + self.runtime_path = "/path/to/maven" self.runtime = "java8" + self.architecture = "x86_64" @parameterized.expand(["1.7.0", "1.8.9", "11.0.0"]) def test_accepts_any_jvm_mv(self, version): version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils) - self.assertTrue(validator.validate(maven_path=self.maven_path)) - self.assertEqual(validator.validated_binary_path, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) @parameterized.expand(["12"]) def test_accepts_major_version_only_jvm_mv(self, version): version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils) - self.assertTrue(validator.validate(maven_path=self.maven_path)) - self.assertEqual(validator.validated_binary_path, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) def test_emits_warning_when_jvm_mv_greater_than_8(self): version_string = "Java version: 10.0.1, vendor: Oracle Corporation".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(maven_path=self.maven_path)) - self.assertEqual(validator.validated_binary_path, self.maven_path) - self.mock_log.warning.assert_called_with(MavenValidator.MAJOR_VERSION_WARNING, self.maven_path, "10", "8", "8") + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils, log=self.mock_log) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) + self.mock_log.warning.assert_called_with( + MavenValidator.MAJOR_VERSION_WARNING, self.runtime_path, "10", "8", "8" + ) @parameterized.expand(["1.6.0", "1.7.0", "1.8.9"]) def test_does_not_emit_warning_when_jvm_mv_8_or_less(self, version): version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(maven_path=self.maven_path)) - self.assertEqual(validator.validated_binary_path, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils, log=self.mock_log) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) self.mock_log.warning.assert_not_called() def test_emits_warning_when_maven_excutable_fails(self): version_string = "Java version: %s, vendor: Oracle Corporation".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=1)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - validator.validate(maven_path=self.maven_path) - self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils, log=self.mock_log) + validator.validate(runtime_path=self.runtime_path) + self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.runtime_path) def test_emits_warning_when_version_string_not_found(self): version_string = "Blah: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=0)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - validator.validate(maven_path=self.maven_path) - self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils, log=self.mock_log) + validator.validate(runtime_path=self.runtime_path) + self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.runtime_path) + + def test_runtime_validate_unsupported_language_fail_open(self): + validator = MavenValidator(runtime="java2.0", architecture="arm64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate(runtime_path="/usr/bin/java2.0") diff --git a/tests/unit/workflows/java_maven/test_workflow.py b/tests/unit/workflows/java_maven/test_workflow.py index 7c830b605..670d04950 100644 --- a/tests/unit/workflows/java_maven/test_workflow.py +++ b/tests/unit/workflows/java_maven/test_workflow.py @@ -9,6 +9,7 @@ from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflows.java_maven.maven_resolver import MavenResolver from aws_lambda_builders.workflows.java_maven.maven_validator import MavenValidator +from aws_lambda_builders.architecture import ARM64 class TestJavaMavenWorkflow(TestCase): @@ -19,38 +20,46 @@ class TestJavaMavenWorkflow(TestCase): def test_workflow_sets_up_maven_actions(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") - self.assertEqual(len(workflow.actions), 4) - self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertIsInstance(workflow.actions[1], JavaMavenBuildAction) - self.assertIsInstance(workflow.actions[2], JavaMavenCopyDependencyAction) - self.assertIsInstance(workflow.actions[3], JavaMavenCopyArtifactsAction) def test_workflow_sets_up_resolvers(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") - resolvers = workflow.get_resolvers() self.assertEqual(len(resolvers), 1) - self.assertIsInstance(resolvers[0], MavenResolver) def test_workflow_sets_up_validators(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") - validators = workflow.get_validators() self.assertEqual(len(validators), 1) - self.assertIsInstance(validators[0], MavenValidator) def test_workflow_excluded_files(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") - self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertEqual(".aws-sam", workflow.actions[0].excludes[0]) - self.assertEqual(".git", workflow.actions[0].excludes[1]) + + def test_must_validate_architecture(self): + workflow = JavaMavenWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = JavaMavenWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/nodejs_npm/test_workflow.py b/tests/unit/workflows/nodejs_npm/test_workflow.py index 1222d6107..6401d76c1 100644 --- a/tests/unit/workflows/nodejs_npm/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm/test_workflow.py @@ -1,6 +1,9 @@ +import mock + from unittest import TestCase from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction +from aws_lambda_builders.architecture import ARM64 from aws_lambda_builders.workflows.nodejs_npm.workflow import NodejsNpmWorkflow from aws_lambda_builders.workflows.nodejs_npm.actions import ( NodejsNpmPackAction, @@ -8,6 +11,7 @@ NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction, ) +from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils class TestNodejsNpmWorkflow(TestCase): @@ -17,26 +21,34 @@ class TestNodejsNpmWorkflow(TestCase): this is just a quick wiring test to provide fast feedback if things are badly broken """ + def setUp(self): + self.osutils_mock = mock.Mock(spec=OSUtils()) + def test_workflow_sets_up_npm_actions_with_download_dependencies_without_dependencies_dir(self): - workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest") + self.osutils_mock.file_exists.return_value = True - self.assertEqual(len(workflow.actions), 5) + workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils_mock) + self.assertEqual(len(workflow.actions), 5) self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) - self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) - self.assertIsInstance(workflow.actions[2], CopySourceAction) - self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) - self.assertIsInstance(workflow.actions[4], NodejsNpmrcCleanUpAction) def test_workflow_sets_up_npm_actions_without_download_dependencies_with_dependencies_dir(self): + self.osutils_mock.file_exists.return_value = True + workflow = NodejsNpmWorkflow( - "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=False + "source", + "artifacts", + "scratch_dir", + "manifest", + dependencies_dir="dep", + download_dependencies=False, + osutils=self.osutils_mock, ) self.assertEqual(len(workflow.actions), 5) @@ -49,8 +61,16 @@ def test_workflow_sets_up_npm_actions_without_download_dependencies_with_depende def test_workflow_sets_up_npm_actions_with_download_dependencies_and_dependencies_dir(self): + self.osutils_mock.file_exists.return_value = True + workflow = NodejsNpmWorkflow( - "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=True + "source", + "artifacts", + "scratch_dir", + "manifest", + dependencies_dir="dep", + download_dependencies=True, + osutils=self.osutils_mock, ) self.assertEqual(len(workflow.actions), 6) @@ -64,8 +84,16 @@ def test_workflow_sets_up_npm_actions_with_download_dependencies_and_dependencie def test_workflow_sets_up_npm_actions_without_download_dependencies_and_without_dependencies_dir(self): + self.osutils_mock.file_exists.return_value = True + workflow = NodejsNpmWorkflow( - "source", "artifacts", "scratch_dir", "manifest", dependencies_dir=None, download_dependencies=False + "source", + "artifacts", + "scratch_dir", + "manifest", + dependencies_dir=None, + download_dependencies=False, + osutils=self.osutils_mock, ) self.assertEqual(len(workflow.actions), 4) @@ -74,3 +102,35 @@ def test_workflow_sets_up_npm_actions_without_download_dependencies_and_without_ self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) self.assertIsInstance(workflow.actions[2], CopySourceAction) self.assertIsInstance(workflow.actions[3], NodejsNpmrcCleanUpAction) + + def test_workflow_only_copy_action(self): + self.osutils_mock.file_exists.return_value = False + + workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils_mock) + + self.assertEqual(len(workflow.actions), 1) + + self.assertIsInstance(workflow.actions[0], CopySourceAction) + + def test_must_validate_architecture(self): + self.osutils_mock.file_exists.return_value = True + workflow = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + osutils=self.osutils_mock, + ) + workflow_with_arm = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + osutils=self.osutils_mock, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index a2589db97..85a2c033d 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -1,9 +1,10 @@ import sys from unittest import TestCase -from mock import patch, Mock +from mock import patch, Mock, ANY from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction @@ -13,7 +14,8 @@ class TestPythonPipBuildAction(TestCase): @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") - def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): + @patch("aws_lambda_builders.workflows.python_pip.actions.DependencyBuilder") + def test_action_must_call_builder(self, DependencyBuilderMock, PythonPipDependencyBuilderMock): builder_instance = PythonPipDependencyBuilderMock.return_value action = PythonPipBuildAction( @@ -26,6 +28,30 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): ) action.execute() + DependencyBuilderMock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=X86_64) + + builder_instance.build_dependencies.assert_called_with( + artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest" + ) + + @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") + @patch("aws_lambda_builders.workflows.python_pip.actions.DependencyBuilder") + def test_action_must_call_builder_with_architecture(self, DependencyBuilderMock, PythonPipDependencyBuilderMock): + builder_instance = PythonPipDependencyBuilderMock.return_value + + action = PythonPipBuildAction( + "artifacts", + "scratch_dir", + "manifest", + "runtime", + None, + {"python": BinaryPath(resolver=Mock(), validator=Mock(), binary="python", binary_path=sys.executable)}, + ARM64, + ) + action.execute() + + DependencyBuilderMock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=ARM64) + builder_instance.build_dependencies.assert_called_with( artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest" ) diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 6fa61a7de..ddc3e4c46 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -5,6 +5,7 @@ import mock import pytest +from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.workflows.python_pip.utils import OSUtils from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim @@ -117,6 +118,18 @@ def test_can_call_dependency_builder(self, osutils): "path/to/requirements.txt", "artifacts/path/", "scratch_dir/path/" ) + @mock.patch("aws_lambda_builders.workflows.python_pip.packager.DependencyBuilder") + def test_can_create_new_dependency_builder(self, DependencyBuilderMock, osutils): + osutils_mock = mock.Mock(spec=osutils) + builder = PythonPipDependencyBuilder(osutils=osutils_mock, runtime="runtime") + DependencyBuilderMock.assert_called_with(osutils_mock, "runtime", architecture=X86_64) + + @mock.patch("aws_lambda_builders.workflows.python_pip.packager.DependencyBuilder") + def test_can_call_dependency_builder_with_architecture(self, DependencyBuilderMock, osutils): + osutils_mock = mock.Mock(spec=osutils) + builder = PythonPipDependencyBuilder(osutils=osutils_mock, runtime="runtime", architecture=ARM64) + DependencyBuilderMock.assert_called_with(osutils_mock, "runtime", architecture=ARM64) + class TestPackage(object): def test_can_create_package_with_custom_osutils(self, osutils): diff --git a/tests/unit/workflows/python_pip/test_validator.py b/tests/unit/workflows/python_pip/test_validator.py index d1db8a17a..95caf6773 100644 --- a/tests/unit/workflows/python_pip/test_validator.py +++ b/tests/unit/workflows/python_pip/test_validator.py @@ -5,6 +5,7 @@ from aws_lambda_builders.exceptions import MisMatchRuntimeError from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError class MockSubProcess(object): @@ -17,16 +18,12 @@ def communicate(self): class TestPythonRuntimeValidator(TestCase): def setUp(self): - self.validator = PythonRuntimeValidator(runtime="python3.7") - - @parameterized.expand(["python2.7", "python3.6", "python3.7", "python3.8", "python3.9"]) - def test_supported_runtimes(self, runtime): - validator = PythonRuntimeValidator(runtime=runtime) - self.assertTrue(validator.has_runtime()) + self.validator = PythonRuntimeValidator(runtime="python3.7", architecture="x86_64") def test_runtime_validate_unsupported_language_fail_open(self): - validator = PythonRuntimeValidator(runtime="python2.6") - validator.validate(runtime_path="/usr/bin/python2.6") + validator = PythonRuntimeValidator(runtime="python2.6", architecture="arm64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate(runtime_path="/usr/bin/python2.6") def test_runtime_validate_supported_version_runtime(self): with mock.patch("subprocess.Popen") as mock_subprocess: @@ -46,3 +43,15 @@ def test_python_command(self): version_strings = ["sys.version_info.major == 3", "sys.version_info.minor == 7"] for version_string in version_strings: self.assertTrue(all([part for part in cmd if version_string in part])) + + @parameterized.expand( + [ + ("python2.7", "arm64"), + ("python3.6", "arm64"), + ("python3.7", "arm64"), + ] + ) + def test_runtime_validate_with_incompatible_architecture(self, runtime, architecture): + validator = PythonRuntimeValidator(runtime=runtime, architecture=architecture) + with self.assertRaises(UnsupportedArchitectureError): + validator.validate(runtime_path="/usr/bin/python") diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py index b07085702..124917c47 100644 --- a/tests/unit/workflows/python_pip/test_workflow.py +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -1,5 +1,4 @@ -import mock - +from mock import patch, ANY, Mock from unittest import TestCase from aws_lambda_builders.actions import CopySourceAction @@ -11,33 +10,31 @@ class TestPythonPipWorkflow(TestCase): def setUp(self): self.osutils = OSUtils() - - def test_workflow_sets_up_actions(self): - osutils_mock = mock.Mock(spec=self.osutils) - osutils_mock.file_exists.return_value = True + self.osutils_mock = Mock(spec=self.osutils) + self.osutils_mock.file_exists.return_value = True self.workflow = PythonPipWorkflow( - "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=osutils_mock + "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=self.osutils_mock ) + + def test_workflow_sets_up_actions(self): self.assertEqual(len(self.workflow.actions), 2) self.assertIsInstance(self.workflow.actions[0], PythonPipBuildAction) self.assertIsInstance(self.workflow.actions[1], CopySourceAction) def test_workflow_sets_up_actions_without_requirements(self): - osutils_mock = mock.Mock(spec=self.osutils) - osutils_mock.file_exists.return_value = False + self.osutils_mock.file_exists.return_value = False self.workflow = PythonPipWorkflow( - "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=osutils_mock + "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=self.osutils_mock ) self.assertEqual(len(self.workflow.actions), 1) self.assertIsInstance(self.workflow.actions[0], CopySourceAction) def test_workflow_validator(self): - self.workflow = PythonPipWorkflow("source", "artifacts", "scratch_dir", "manifest", runtime="python3.7") for validator in self.workflow.get_validators(): self.assertTrue(isinstance(validator, PythonRuntimeValidator)) def test_workflow_sets_up_actions_without_download_dependencies_with_dependencies_dir(self): - osutils_mock = mock.Mock(spec=self.osutils) + osutils_mock = Mock(spec=self.osutils) osutils_mock.file_exists.return_value = True self.workflow = PythonPipWorkflow( "source", @@ -54,7 +51,7 @@ def test_workflow_sets_up_actions_without_download_dependencies_with_dependencie self.assertIsInstance(self.workflow.actions[1], CopySourceAction) def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_dir(self): - osutils_mock = mock.Mock(spec=self.osutils) + osutils_mock = Mock(spec=self.osutils) osutils_mock.file_exists.return_value = True self.workflow = PythonPipWorkflow( "source", @@ -72,7 +69,7 @@ def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_di self.assertIsInstance(self.workflow.actions[2], CopySourceAction) def test_workflow_sets_up_actions_without_download_dependencies_without_dependencies_dir(self): - osutils_mock = mock.Mock(spec=self.osutils) + osutils_mock = Mock(spec=self.osutils) osutils_mock.file_exists.return_value = True self.workflow = PythonPipWorkflow( "source", @@ -86,3 +83,25 @@ def test_workflow_sets_up_actions_without_download_dependencies_without_dependen ) self.assertEqual(len(self.workflow.actions), 1) self.assertIsInstance(self.workflow.actions[0], CopySourceAction) + + @patch("aws_lambda_builders.workflows.python_pip.workflow.PythonPipBuildAction") + def test_must_build_with_architecture(self, PythonPipBuildActionMock): + self.workflow = PythonPipWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + runtime="python3.7", + architecture="ARM64", + osutils=self.osutils_mock, + ) + PythonPipBuildActionMock.assert_called_with( + "artifacts", + "scratch_dir", + "manifest", + "python3.7", + None, + binaries=ANY, + architecture="ARM64", + ) + self.assertEqual(2, len(self.workflow.actions)) diff --git a/tests/unit/workflows/ruby_bundler/test_workflow.py b/tests/unit/workflows/ruby_bundler/test_workflow.py index 180991fa1..5b233f228 100644 --- a/tests/unit/workflows/ruby_bundler/test_workflow.py +++ b/tests/unit/workflows/ruby_bundler/test_workflow.py @@ -1,6 +1,7 @@ from unittest import TestCase from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction +from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.workflows.ruby_bundler.workflow import RubyBundlerWorkflow from aws_lambda_builders.workflows.ruby_bundler.actions import RubyBundlerInstallAction, RubyBundlerVendorAction @@ -11,15 +12,6 @@ class TestRubyBundlerWorkflow(TestCase): this is just a quick wiring test to provide fast feedback if things are badly broken """ - def test_workflow_sets_up_bundler_actions_with_download_dependencies_without_dependencies_dir(self): - workflow = RubyBundlerWorkflow( - "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=False - ) - self.assertEqual(len(workflow.actions), 3) - self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertIsInstance(workflow.actions[1], RubyBundlerInstallAction) - self.assertIsInstance(workflow.actions[2], RubyBundlerVendorAction) - def test_workflow_sets_up_npm_actions_without_download_dependencies_with_dependencies_dir(self): workflow = RubyBundlerWorkflow( "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=False @@ -50,3 +42,23 @@ def test_workflow_sets_up_bundler_actions_without_download_dependencies_without_ self.assertEqual(len(workflow.actions), 1) self.assertIsInstance(workflow.actions[0], CopySourceAction) + + def test_must_validate_architecture(self): + workflow = RubyBundlerWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = RubyBundlerWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64")