diff --git a/aws_lambda_builders/actions.py b/aws_lambda_builders/actions.py index 89ae3801a..c3fb9672e 100644 --- a/aws_lambda_builders/actions.py +++ b/aws_lambda_builders/actions.py @@ -3,6 +3,7 @@ """ import logging +import os import shutil import six @@ -30,6 +31,9 @@ class Purpose(object): # Action is copying source code COPY_SOURCE = "COPY_SOURCE" + # Action is copying dependencies + COPY_DEPENDENCIES = "COPY_DEPENDENCIES" + # Action is compiling source code COMPILE_SOURCE = "COMPILE_SOURCE" @@ -99,3 +103,32 @@ def __init__(self, source_dir, dest_dir, excludes=None): def execute(self): copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes)) + + +class CopyDependenciesAction(BaseAction): + + NAME = "CopyDependencies" + + DESCRIPTION = "Copying dependencies while skipping source file" + + PURPOSE = Purpose.COPY_DEPENDENCIES + + def __init__(self, source_dir, artifact_dir, destination_dir, excludes=None): + self.source_dir = source_dir + self.artifact_dir = artifact_dir + self.dest_dir = destination_dir + self.excludes = excludes or [] + + def execute(self): + source = set(os.listdir(self.source_dir)) + artifact = set(os.listdir(self.artifact_dir)) + dependencies = artifact - source + + for name in dependencies: + dependencies_source = os.path.join(self.artifact_dir, name) + new_destination = os.path.join(self.dest_dir, name) + + if os.path.isdir(dependencies_source): + copytree(dependencies_source, new_destination) + else: + shutil.copy2(dependencies_source, new_destination) diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index b48ea4430..961489df7 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -1,13 +1,17 @@ """ NodeJS NPM Workflow """ +import logging + from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.workflow import BaseWorkflow, Capability -from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction from .actions import NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction from .utils import OSUtils from .npm import SubprocessNpm +LOG = logging.getLogger(__name__) + class NodejsNpmWorkflow(BaseWorkflow): @@ -40,18 +44,34 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim tar_dest_dir, scratch_dir, manifest_path, osutils=osutils, subprocess_npm=subprocess_npm ) - npm_install = NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm) - npm_copy_npmrc = NodejsNpmrcCopyAction(tar_package_dir, source_dir, osutils=osutils) self.actions = [ npm_pack, npm_copy_npmrc, CopySourceAction(tar_package_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), - npm_install, - NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils), ] + if self.download_dependencies: + # installed the dependencies into artifact folder + self.actions.append(NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm)) + + # if dependencies folder exists, copy dependencies into dependencies into dependencies folder + if self.dependencies_dir: + self.actions.append(CopyDependenciesAction(source_dir, artifacts_dir, self.dependencies_dir)) + else: + # if dependencies folder exists and not download dependencies, simply copy the dependencies from the + # dependencies folder to artifact folder + if self.dependencies_dir: + self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) + else: + LOG.info( + "download_dependencies is False and dependencies_dir is None. Copying the source files into the " + "artifacts directory. " + ) + + self.actions.append(NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils)) + def get_resolvers(self): """ specialized path resolver that just returns the list of executable for the runtime on the path. diff --git a/aws_lambda_builders/workflows/ruby_bundler/workflow.py b/aws_lambda_builders/workflows/ruby_bundler/workflow.py index f57f6cf60..04e051f65 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/workflow.py +++ b/aws_lambda_builders/workflows/ruby_bundler/workflow.py @@ -1,13 +1,16 @@ """ Ruby Bundler Workflow """ +import logging from aws_lambda_builders.workflow import BaseWorkflow, Capability -from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction from .actions import RubyBundlerInstallAction, RubyBundlerVendorAction from .utils import OSUtils from .bundler import SubprocessBundler +LOG = logging.getLogger(__name__) + class RubyBundlerWorkflow(BaseWorkflow): @@ -31,12 +34,26 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim if osutils is None: osutils = OSUtils() - subprocess_bundler = SubprocessBundler(osutils) - bundle_install = RubyBundlerInstallAction(artifacts_dir, subprocess_bundler=subprocess_bundler) - - bundle_deployment = RubyBundlerVendorAction(artifacts_dir, subprocess_bundler=subprocess_bundler) - self.actions = [ - CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), - bundle_install, - bundle_deployment, - ] + self.actions = [CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)] + + if self.download_dependencies: + # installed the dependencies into artifact folder + subprocess_bundler = SubprocessBundler(osutils) + bundle_install = RubyBundlerInstallAction(artifacts_dir, subprocess_bundler=subprocess_bundler) + bundle_deployment = RubyBundlerVendorAction(artifacts_dir, subprocess_bundler=subprocess_bundler) + self.actions.append(bundle_install) + self.actions.append(bundle_deployment) + + # if dependencies folder exists, copy dependencies into dependencies into dependencies folder + if self.dependencies_dir: + self.actions.append(CopyDependenciesAction(source_dir, artifacts_dir, self.dependencies_dir)) + else: + # if dependencies folder exists and not download dependencies, simply copy the dependencies from the + # dependencies folder to artifact folder + if self.dependencies_dir: + self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) + else: + LOG.info( + "download_dependencies is False and dependencies_dir is None. Copying the source files into the " + "artifacts directory. " + ) diff --git a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py index 83a8de6e0..4cdb41e62 100644 --- a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py +++ b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py @@ -1,12 +1,16 @@ +import logging import os import shutil import tempfile from unittest import TestCase +import mock from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError +workflow_logger = logging.getLogger("aws_lambda_builders.workflows.nodejs_npm.workflow") + class TestNodejsNpmWorkflow(TestCase): """ @@ -18,6 +22,7 @@ class TestNodejsNpmWorkflow(TestCase): def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() + self.dependencies_dir = tempfile.mkdtemp() self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps") @@ -27,6 +32,7 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) + shutil.rmtree(self.dependencies_dir) def test_builds_project_without_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") @@ -126,3 +132,72 @@ def test_fails_if_package_json_is_broken(self): ) self.assertIn("Unexpected end of JSON input", str(ctx.exception)) + + def test_builds_project_with_remote_dependencies_without_download_dependencies_with_dependencies_dir(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps") + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + dependencies_dir=self.dependencies_dir, + download_dependencies=False, + ) + + expected_files = {"package.json", "included.js"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + def test_builds_project_with_remote_dependencies_without_download_dependencies_and_dependencies_dir(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps") + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + dependencies_dir=self.dependencies_dir, + download_dependencies=True, + ) + + expected_files = {"package.json", "included.js", "node_modules"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + expected_modules = {"minimal-request-promise"} + output_modules = set(os.listdir(os.path.join(self.artifacts_dir, "node_modules"))) + self.assertEqual(expected_modules, output_modules) + + expected_modules = {"minimal-request-promise"} + output_modules = set(os.listdir(os.path.join(self.dependencies_dir, "node_modules"))) + self.assertEqual(expected_modules, output_modules) + + expected_dependencies_files = {"node_modules"} + output_dependencies_files = set(os.listdir(os.path.join(self.dependencies_dir))) + self.assertNotIn(expected_dependencies_files, output_dependencies_files) + + def test_builds_project_with_remote_dependencies_without_download_dependencies_without_dependencies_dir(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps") + + with mock.patch.object(workflow_logger, "info") as mock_info: + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + dependencies_dir=None, + download_dependencies=False, + ) + + expected_files = {"package.json", "included.js"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + mock_info.assert_called_with( + "download_dependencies is False and dependencies_dir is None. Copying the source files into the " + "artifacts directory. " + ) diff --git a/tests/integration/workflows/ruby_bundler/test_ruby.py b/tests/integration/workflows/ruby_bundler/test_ruby.py index d17f6c380..c1379dbc6 100644 --- a/tests/integration/workflows/ruby_bundler/test_ruby.py +++ b/tests/integration/workflows/ruby_bundler/test_ruby.py @@ -11,6 +11,7 @@ import logging logger = logging.getLogger("aws_lambda_builders.workflows.ruby_bundler.bundler") +workflow_logger = logging.getLogger("aws_lambda_builders.workflows.ruby_bundler.workflow") class TestRubyWorkflow(TestCase): @@ -23,6 +24,7 @@ class TestRubyWorkflow(TestCase): def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() + self.dependencies_dir = tempfile.mkdtemp() self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps") self.builder = LambdaBuilder(language="ruby", dependency_manager="bundler", application_framework=None) self.runtime = "ruby2.5" @@ -30,6 +32,7 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) + shutil.rmtree(self.dependencies_dir) def test_builds_project_without_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") @@ -84,3 +87,58 @@ def test_must_log_warning_if_gemfile_not_found(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) mock_warning.assert_called_with("Gemfile not found. Continuing the build without dependencies.") + + def test_builds_project_without_downloaded_dependencies_with_dependencies_dir(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps") + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "Gemfile"), + runtime=self.runtime, + dependencies_dir=self.dependencies_dir, + download_dependencies=False, + ) + expected_files = {"handler.rb", "Gemfile"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + def test_builds_project_with_downloaded_dependencies_and_dependencies_dir(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps") + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "Gemfile"), + runtime=self.runtime, + dependencies_dir=self.dependencies_dir, + download_dependencies=True, + ) + expected_files = {"handler.rb", "Gemfile", "Gemfile.lock", ".bundle", "vendor"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + expected_dependencies_files = {"Gemfile.lock", ".bundle", "vendor"} + output_dependencies_files = set(os.listdir(self.dependencies_dir)) + self.assertEqual(output_dependencies_files, expected_dependencies_files) + + def test_builds_project_without_downloaded_dependencies_without_dependencies_dir(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps") + with mock.patch.object(workflow_logger, "info") as mock_info: + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "Gemfile"), + runtime=self.runtime, + dependencies_dir=None, + download_dependencies=False, + ) + expected_files = {"handler.rb", "Gemfile"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + mock_info.assert_called_with( + "download_dependencies is False and dependencies_dir is None. Copying the source files into the " + "artifacts directory. " + ) diff --git a/tests/unit/test_actions.py b/tests/unit/test_actions.py index dc1fa38c8..82ef32954 100644 --- a/tests/unit/test_actions.py +++ b/tests/unit/test_actions.py @@ -1,7 +1,7 @@ from unittest import TestCase from mock import patch, ANY -from aws_lambda_builders.actions import BaseAction, CopySourceAction, Purpose +from aws_lambda_builders.actions import BaseAction, CopySourceAction, Purpose, CopyDependenciesAction class TestBaseActionInheritance(TestCase): @@ -52,3 +52,26 @@ def test_must_copy(self, copytree_mock): action.execute() copytree_mock.assert_called_with(source_dir, dest_dir, ignore=ANY) + + +class TestCopyDependenciesAction_execute(TestCase): + @patch("aws_lambda_builders.actions.shutil.copy2") + @patch("aws_lambda_builders.actions.copytree") + @patch("aws_lambda_builders.actions.os.path.isdir") + @patch("aws_lambda_builders.actions.os.listdir") + @patch("aws_lambda_builders.actions.os.path.join") + def test_must_copy(self, path_mock, listdir_mock, isdir_mock, copytree_mock, copy2_mock): + source_dir = "source" + artifact_dir = "artifact" + dest_dir = "dest" + + listdir_mock.side_effect = [[1], [1, 2, 3]] + path_mock.side_effect = ["dir1", "dir2", "file1", "file2"] + isdir_mock.side_effect = [True, False] + action = CopyDependenciesAction(source_dir, artifact_dir, dest_dir) + action.execute() + + listdir_mock.assert_any_call(source_dir) + listdir_mock.assert_any_call(artifact_dir) + copytree_mock.assert_called_once_with("dir1", "dir2") + copy2_mock.assert_called_once_with("file1", "file2") diff --git a/tests/unit/workflows/nodejs_npm/test_workflow.py b/tests/unit/workflows/nodejs_npm/test_workflow.py index 3dac6d791..1222d6107 100644 --- a/tests/unit/workflows/nodejs_npm/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm/test_workflow.py @@ -1,6 +1,6 @@ from unittest import TestCase -from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction from aws_lambda_builders.workflows.nodejs_npm.workflow import NodejsNpmWorkflow from aws_lambda_builders.workflows.nodejs_npm.actions import ( NodejsNpmPackAction, @@ -17,7 +17,7 @@ class TestNodejsNpmWorkflow(TestCase): this is just a quick wiring test to provide fast feedback if things are badly broken """ - def test_workflow_sets_up_npm_actions(self): + def test_workflow_sets_up_npm_actions_with_download_dependencies_without_dependencies_dir(self): workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest") @@ -32,3 +32,45 @@ def test_workflow_sets_up_npm_actions(self): self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) self.assertIsInstance(workflow.actions[4], NodejsNpmrcCleanUpAction) + + def test_workflow_sets_up_npm_actions_without_download_dependencies_with_dependencies_dir(self): + + workflow = NodejsNpmWorkflow( + "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=False + ) + + self.assertEqual(len(workflow.actions), 5) + + self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) + self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) + self.assertIsInstance(workflow.actions[2], CopySourceAction) + self.assertIsInstance(workflow.actions[3], CopySourceAction) + self.assertIsInstance(workflow.actions[4], NodejsNpmrcCleanUpAction) + + def test_workflow_sets_up_npm_actions_with_download_dependencies_and_dependencies_dir(self): + + workflow = NodejsNpmWorkflow( + "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=True + ) + + self.assertEqual(len(workflow.actions), 6) + + self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) + self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) + self.assertIsInstance(workflow.actions[2], CopySourceAction) + self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) + self.assertIsInstance(workflow.actions[4], CopyDependenciesAction) + self.assertIsInstance(workflow.actions[5], NodejsNpmrcCleanUpAction) + + def test_workflow_sets_up_npm_actions_without_download_dependencies_and_without_dependencies_dir(self): + + workflow = NodejsNpmWorkflow( + "source", "artifacts", "scratch_dir", "manifest", dependencies_dir=None, download_dependencies=False + ) + + self.assertEqual(len(workflow.actions), 4) + + self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) + self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) + self.assertIsInstance(workflow.actions[2], CopySourceAction) + self.assertIsInstance(workflow.actions[3], NodejsNpmrcCleanUpAction) diff --git a/tests/unit/workflows/ruby_bundler/test_workflow.py b/tests/unit/workflows/ruby_bundler/test_workflow.py index aa7ccafc3..180991fa1 100644 --- a/tests/unit/workflows/ruby_bundler/test_workflow.py +++ b/tests/unit/workflows/ruby_bundler/test_workflow.py @@ -1,6 +1,6 @@ from unittest import TestCase -from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction from aws_lambda_builders.workflows.ruby_bundler.workflow import RubyBundlerWorkflow from aws_lambda_builders.workflows.ruby_bundler.actions import RubyBundlerInstallAction, RubyBundlerVendorAction @@ -11,9 +11,42 @@ class TestRubyBundlerWorkflow(TestCase): this is just a quick wiring test to provide fast feedback if things are badly broken """ - def test_workflow_sets_up_bundler_actions(self): - workflow = RubyBundlerWorkflow("source", "artifacts", "scratch_dir", "manifest") + def test_workflow_sets_up_bundler_actions_with_download_dependencies_without_dependencies_dir(self): + workflow = RubyBundlerWorkflow( + "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=False + ) self.assertEqual(len(workflow.actions), 3) self.assertIsInstance(workflow.actions[0], CopySourceAction) self.assertIsInstance(workflow.actions[1], RubyBundlerInstallAction) self.assertIsInstance(workflow.actions[2], RubyBundlerVendorAction) + + def test_workflow_sets_up_npm_actions_without_download_dependencies_with_dependencies_dir(self): + workflow = RubyBundlerWorkflow( + "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=False + ) + + self.assertEqual(len(workflow.actions), 2) + + self.assertIsInstance(workflow.actions[0], CopySourceAction) + self.assertIsInstance(workflow.actions[1], CopySourceAction) + + def test_workflow_sets_up_bundler_actions_with_download_dependencies_without_dependencies_dir(self): + workflow = RubyBundlerWorkflow( + "source", "artifacts", "scratch_dir", "manifest", dependencies_dir="dep", download_dependencies=True + ) + + self.assertEqual(len(workflow.actions), 4) + + self.assertIsInstance(workflow.actions[0], CopySourceAction) + self.assertIsInstance(workflow.actions[1], RubyBundlerInstallAction) + self.assertIsInstance(workflow.actions[2], RubyBundlerVendorAction) + self.assertIsInstance(workflow.actions[3], CopyDependenciesAction) + + def test_workflow_sets_up_bundler_actions_without_download_dependencies_without_dependencies_dir(self): + workflow = RubyBundlerWorkflow( + "source", "artifacts", "scratch_dir", "manifest", dependencies_dir=None, download_dependencies=False + ) + + self.assertEqual(len(workflow.actions), 1) + + self.assertIsInstance(workflow.actions[0], CopySourceAction)