diff --git a/.appveyor.yml b/.appveyor.yml index 808e7951a..61a5b8032 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,7 +4,6 @@ image: - Ubuntu environment: - GOPATH: c:\gopath GOVERSION: 1.11 GRADLE_OPTS: -Dorg.gradle.daemon=false nodejs_version: "8.10.0" @@ -45,6 +44,9 @@ for: only: - image: Visual Studio 2017 + environment: + GOPATH: c:\gopath + install: # To run Nodejs workflow integ tests - ps: Install-Product node 8.10 @@ -65,6 +67,10 @@ for: - "choco install dep" - setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;" - "go version" + # set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory + # https://blog.golang.org/go116-module-changes#TOC_2. + # This is required for the go dep integration tests + - "go env -w GO111MODULE=auto" - "go env" # setup Gradle @@ -102,6 +108,10 @@ for: - sh: "wget https://services.gradle.org/distributions/gradle-5.5-bin.zip -P /tmp" - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip" - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH" + # set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory + # https://blog.golang.org/go116-module-changes#TOC_2. + # This is required for the go dep integration tests + - sh: "go env -w GO111MODULE=auto" build_script: - "python -c \"import sys; print(sys.executable)\"" diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 000ee822e..c427a958b 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -150,13 +150,23 @@ class DependencyBuilder(object): packager. """ - _MANYLINUX_COMPATIBLE_PLATFORM = { - "any", - "linux_x86_64", - "manylinux1_x86_64", - "manylinux2010_x86_64", - "manylinux2014_x86_64", + _ADDITIONAL_COMPATIBLE_PLATFORM = {"any", "linux_x86_64"} + _MANYLINUX_LEGACY_MAP = { + "manylinux1_x86_64": "manylinux_2_5_x86_64", + "manylinux2010_x86_64": "manylinux_2_12_x86_64", + "manylinux2014_x86_64": "manylinux_2_17_x86_64", } + # Mapping of abi to glibc version in Lambda runtime. + _RUNTIME_GLIBC = { + "cp27mu": (2, 17), + "cp36m": (2, 17), + "cp37m": (2, 17), + "cp38": (2, 26), + } + # Fallback version if we're on an unknown python version + # not in _RUNTIME_GLIBC. + # Unlikely to hit this case. + _DEFAULT_GLIBC = (2, 17) _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"} def __init__(self, osutils, runtime, pip_runner=None): @@ -341,30 +351,62 @@ def _categorize_wheel_files(self, directory): def _is_compatible_wheel_filename(self, filename): wheel = filename[:-4] - implementation, abi, platform = wheel.split("-")[-3:] - # Verify platform is compatible - if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM: - return False - lambda_runtime_abi = get_lambda_abi(self.runtime) + for implementation, abi, platform in self._iter_all_compatibility_tags(wheel): + if not self._is_compatible_platform_tag(lambda_runtime_abi, platform): + continue + + # Verify that the ABI is compatible with lambda. Either none or the + # correct type for the python version cp27mu for py27 and cp36m for + # py36. + if abi == "none": + return True + prefix_version = implementation[:3] + if prefix_version == "cp3": + # Deploying python 3 function which means we need cp36m abi + # We can also accept abi3 which is the CPython 3 Stable ABI and + # will work on any version of python 3. + if abi == lambda_runtime_abi or abi == "abi3": + return True + elif prefix_version == "cp2": + # Deploying to python 2 function which means we need cp27mu abi + if abi == "cp27mu": + return True + # Don't know what we have but it didn't pass compatibility tests. + return False - # Verify that the ABI is compatible with lambda. Either none or the - # correct type for the python version cp27mu for py27 and cp36m for - # py36. - if abi == "none": + def _is_compatible_platform_tag(self, expected_abi, platform): + """ + Verify if a platform tag is compatible based on PEP 600 + https://www.python.org/dev/peps/pep-0600/#specification + + In addition to checking the tag pattern, we also need to verify the glibc version + """ + if platform in self._ADDITIONAL_COMPATIBLE_PLATFORM: return True - prefix_version = implementation[:3] - if prefix_version == "cp3": - # Deploying python 3 function which means we need cp36m abi - # We can also accept abi3 which is the CPython 3 Stable ABI and - # will work on any version of python 3. - return abi == lambda_runtime_abi or abi == "abi3" - elif prefix_version == "cp2": - # Deploying to python 2 function which means we need cp27mu abi - return abi == "cp27mu" - # Don't know what we have but it didn't pass compatibility tests. + elif platform.startswith("manylinux"): + perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform) + m = re.match("manylinux_([0-9]+)_([0-9]+)_(.*)", perennial_tag) + if m is None: + return False + tag_major, tag_minor = [int(x) for x in m.groups()[:2]] + runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC) + if (tag_major, tag_minor) <= (runtime_major, runtime_minor): + # glibc version is compatible with Lambda Runtime + return True return False + def _iter_all_compatibility_tags(self, wheel): + """ + Generates all possible combination of tag sets as described in PEP 425 + https://www.python.org/dev/peps/pep-0425/#compressed-tag-sets + """ + implementation_tag, abi_tag, platform_tag = wheel.split("-")[-3:] + for implementation in implementation_tag.split("."): + for abi in abi_tag.split("."): + for platform in platform_tag.split("."): + yield (implementation, abi, platform) + def _apply_wheel_allowlist(self, compatible_wheels, incompatible_wheels): compatible_wheels = set(compatible_wheels) actual_incompatible_wheels = set() diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 10dfb0910..7c00aa66c 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -449,6 +449,48 @@ def test_can_get_whls_mixed_compat(self, tmpdir, osutils, pip_runner): for req in reqs: assert req in installed_packages + def test_can_support_pep_600_tags(self, tmpdir, osutils, pip_runner): + reqs = ["foo"] + pip, runner = pip_runner + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.packages_to_download( + expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], + packages=[ + "foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl", + ], + ) + + site_packages = os.path.join(appdir, ".chalice.", "site-packages") + with osutils.tempdir() as scratch_dir: + builder.build_site_packages(requirements_file, site_packages, scratch_dir) + installed_packages = os.listdir(site_packages) + + pip.validate() + for req in reqs: + assert req in installed_packages + + def test_can_support_compressed_tags(self, tmpdir, osutils, pip_runner): + reqs = ["foo"] + pip, runner = pip_runner + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.packages_to_download( + expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], + packages=[ + "foo-1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", + ], + ) + + site_packages = os.path.join(appdir, ".chalice.", "site-packages") + with osutils.tempdir() as scratch_dir: + builder.build_site_packages(requirements_file, site_packages, scratch_dir) + installed_packages = os.listdir(site_packages) + + pip.validate() + for req in reqs: + assert req in installed_packages + def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner): reqs = ["foo", "bar", "baz"] pip, runner = pip_runner @@ -539,6 +581,36 @@ def test_does_fail_on_python_1_whl(self, tmpdir, osutils, pip_runner): assert missing_packages[0].identifier == "baz==1.5" assert len(installed_packages) == 0 + def test_does_fail_on_pep_600_tag_with_unsupported_glibc_version(self, tmpdir, osutils, pip_runner): + reqs = ["foo", "bar", "baz", "qux"] + pip, runner = pip_runner + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.packages_to_download( + expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], + packages=[ + "foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl", + "bar-1.2-cp36-cp36m-manylinux_2_999_x86_64.whl", + "baz-1.2-cp36-cp36m-manylinux_3_12_x86_64.whl", + "qux-1.2-cp36-cp36m-manylinux_3_999_x86_64.whl", + ], + ) + + site_packages = os.path.join(appdir, ".chalice.", "site-packages") + with osutils.tempdir() as scratch_dir: + with pytest.raises(MissingDependencyError) as e: + builder.build_site_packages(requirements_file, site_packages, scratch_dir) + installed_packages = os.listdir(site_packages) + + missing_packages = list(e.value.missing) + pip.validate() + assert len(missing_packages) == 3 + missing_package_identifies = [package.identifier for package in missing_packages] + assert "bar==1.2" in missing_package_identifies + assert "baz==1.2" in missing_package_identifies + assert "qux==1.2" in missing_package_identifies + assert len(installed_packages) == 1 + def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): reqs = ["foo", "bar"] pip, runner = pip_runner diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index dec957957..745e560d7 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -47,8 +47,10 @@ def test_must_build_python_project(self): if self.runtime == "python2.7": expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) - else: + elif self.runtime == "python3.6": expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) + else: + expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) diff --git a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt index ff68cfc1c..5a57de403 100644 --- a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt +++ b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt @@ -1,2 +1,3 @@ numpy==1.15.4; python_version == '2.7' -numpy==1.17.4; python_version >= '3.6' +numpy==1.17.4; python_version == '3.6' +numpy==1.20.3; python_version >= '3.7'