From 9c0407720d733ebb9e9cebf8b3050d44c9e115f8 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Fri, 14 May 2021 18:07:11 -0700 Subject: [PATCH 1/8] Support PEP 600 platform tags --- .../workflows/python_pip/packager.py | 92 ++++++++++++++----- .../workflows/python_pip/test_packager.py | 71 ++++++++++++++ 2 files changed, 138 insertions(+), 25 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 000ee822e..ebca1bafb 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -150,13 +150,23 @@ class DependencyBuilder(object): packager. """ - _MANYLINUX_COMPATIBLE_PLATFORM = { - "any", - "linux_x86_64", - "manylinux1_x86_64", - "manylinux2010_x86_64", - "manylinux2014_x86_64", + _ADDITIONAL_COMPATIBLE_PLATFORM = {"any", "linux_x86_64"} + _MANYLINUX_LEGACY_MAP = { + "manylinux1_x86_64": "manylinux_2_5_x86_64", + "manylinux2010_x86_64": "manylinux_2_12_x86_64", + "manylinux2014_x86_64": "manylinux_2_17_x86_64", } + # Mapping of abi to glibc version in Lambda runtime. + _RUNTIME_GLIBC = { + "cp27mu": (2, 17), + "cp36m": (2, 17), + "cp37m": (2, 17), + "cp38": (2, 26), + } + # Fallback version if we're on an unknown python version + # not in _RUNTIME_GLIBC. + # Unlikely to hit this case. + _DEFAULT_GLIBC = (2, 17) _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"} def __init__(self, osutils, runtime, pip_runner=None): @@ -341,30 +351,62 @@ def _categorize_wheel_files(self, directory): def _is_compatible_wheel_filename(self, filename): wheel = filename[:-4] - implementation, abi, platform = wheel.split("-")[-3:] - # Verify platform is compatible - if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM: - return False - lambda_runtime_abi = get_lambda_abi(self.runtime) + for implementation, abi, platform in self._iter_all_compatibility_tags(wheel): + if not self._is_compatible_platform_tag(lambda_runtime_abi, platform): + continue + + # Verify that the ABI is compatible with lambda. Either none or the + # correct type for the python version cp27mu for py27 and cp36m for + # py36. + if abi == "none": + return True + prefix_version = implementation[:3] + if prefix_version == "cp3": + # Deploying python 3 function which means we need cp36m abi + # We can also accept abi3 which is the CPython 3 Stable ABI and + # will work on any version of python 3. + if abi == lambda_runtime_abi or abi == "abi3": + return True + elif prefix_version == "cp2": + # Deploying to python 2 function which means we need cp27mu abi + if abi == "cp27mu": + return True + # Don't know what we have but it didn't pass compatibility tests. + return False - # Verify that the ABI is compatible with lambda. Either none or the - # correct type for the python version cp27mu for py27 and cp36m for - # py36. - if abi == "none": + def _is_compatible_platform_tag(self, expected_abi, platform): + """ + Verify if a platform tag is compatible based on PEP 600 + https://www.python.org/dev/peps/pep-0600/#specification + + In addition to checking the tag pattern, we also need to verify the glibc version + """ + if platform in self._ADDITIONAL_COMPATIBLE_PLATFORM: return True - prefix_version = implementation[:3] - if prefix_version == "cp3": - # Deploying python 3 function which means we need cp36m abi - # We can also accept abi3 which is the CPython 3 Stable ABI and - # will work on any version of python 3. - return abi == lambda_runtime_abi or abi == "abi3" - elif prefix_version == "cp2": - # Deploying to python 2 function which means we need cp27mu abi - return abi == "cp27mu" - # Don't know what we have but it didn't pass compatibility tests. + elif platform.startswith("manylinux"): + perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform) + m = re.match("manylinux_([0-9]+)_([0-9]+)_(.*)", perennial_tag) + if m is None: + return False + tag_major, tag_minor = [int(x) for x in m.groups()[:2]] + runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC) + if (tag_major, tag_minor) <= (runtime_major, runtime_minor): + # glibc version is compatible with Lambda Runtime + return True return False + def _iter_all_compatibility_tags(self, wheel): + """ + Generates all possible combination of tag sets as described in PEP 425 + https://www.python.org/dev/peps/pep-0425/#id15 + """ + implementation_tag, abi_tag, platform_tag = wheel.split("-")[-3:] + for implementation in implementation_tag.split("."): + for abi in abi_tag.split("."): + for platform in platform_tag.split("."): + yield (implementation, abi, platform) + def _apply_wheel_allowlist(self, compatible_wheels, incompatible_wheels): compatible_wheels = set(compatible_wheels) actual_incompatible_wheels = set() diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 10dfb0910..2b17fa626 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -449,6 +449,48 @@ def test_can_get_whls_mixed_compat(self, tmpdir, osutils, pip_runner): for req in reqs: assert req in installed_packages + def test_can_support_pep_600_tags(self, tmpdir, osutils, pip_runner): + reqs = ["foo"] + pip, runner = pip_runner + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.packages_to_download( + expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], + packages=[ + "foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl", + ], + ) + + site_packages = os.path.join(appdir, ".chalice.", "site-packages") + with osutils.tempdir() as scratch_dir: + builder.build_site_packages(requirements_file, site_packages, scratch_dir) + installed_packages = os.listdir(site_packages) + + pip.validate() + for req in reqs: + assert req in installed_packages + + def test_can_support_compressed_tags(self, tmpdir, osutils, pip_runner): + reqs = ["foo"] + pip, runner = pip_runner + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.packages_to_download( + expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], + packages=[ + "foo-1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", + ], + ) + + site_packages = os.path.join(appdir, ".chalice.", "site-packages") + with osutils.tempdir() as scratch_dir: + builder.build_site_packages(requirements_file, site_packages, scratch_dir) + installed_packages = os.listdir(site_packages) + + pip.validate() + for req in reqs: + assert req in installed_packages + def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner): reqs = ["foo", "bar", "baz"] pip, runner = pip_runner @@ -539,6 +581,35 @@ def test_does_fail_on_python_1_whl(self, tmpdir, osutils, pip_runner): assert missing_packages[0].identifier == "baz==1.5" assert len(installed_packages) == 0 + def test_does_fail_on_pep_600_tag_with_unsupported_glibc_version(self, tmpdir, osutils, pip_runner): + reqs = ["foo", "bar", "baz", "qux"] + pip, runner = pip_runner + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.packages_to_download( + expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], + packages=[ + "foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl", + "bar-1.2-cp36-cp36m-manylinux_2_999_x86_64.whl", + "baz-1.2-cp36-cp36m-manylinux_3_12_x86_64.whl", + "qux-1.2-cp36-cp36m-manylinux_3_999_x86_64.whl", + ], + ) + + site_packages = os.path.join(appdir, ".chalice.", "site-packages") + with osutils.tempdir() as scratch_dir: + with pytest.raises(MissingDependencyError) as e: + builder.build_site_packages(requirements_file, site_packages, scratch_dir) + installed_packages = os.listdir(site_packages) + + missing_packages = list(e.value.missing) + pip.validate() + assert len(missing_packages) == 3 + assert missing_packages[0].identifier == "bar==1.2" + assert missing_packages[1].identifier == "baz==1.2" + assert missing_packages[2].identifier == "qux==1.2" + assert len(installed_packages) == 1 + def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): reqs = ["foo", "bar"] pip, runner = pip_runner From 4b35d781f0c7ed88283c29f21233f32e5cf9b899 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Fri, 14 May 2021 18:32:06 -0700 Subject: [PATCH 2/8] fix test --- tests/functional/workflows/python_pip/test_packager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 2b17fa626..7c00aa66c 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -605,9 +605,10 @@ def test_does_fail_on_pep_600_tag_with_unsupported_glibc_version(self, tmpdir, o missing_packages = list(e.value.missing) pip.validate() assert len(missing_packages) == 3 - assert missing_packages[0].identifier == "bar==1.2" - assert missing_packages[1].identifier == "baz==1.2" - assert missing_packages[2].identifier == "qux==1.2" + missing_package_identifies = [package.identifier for package in missing_packages] + assert "bar==1.2" in missing_package_identifies + assert "baz==1.2" in missing_package_identifies + assert "qux==1.2" in missing_package_identifies assert len(installed_packages) == 1 def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): From 3bf13b059f10343e746959e315535ae1dfc4db78 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Mon, 17 May 2021 11:13:09 -0700 Subject: [PATCH 3/8] Fix integration test in appveyor by setting GO111MODULE to auto --- .appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 808e7951a..bf6c6fb76 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -65,6 +65,7 @@ for: - "choco install dep" - setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;" - "go version" + - "go env -w GO111MODULE=auto" - "go env" # setup Gradle @@ -102,6 +103,7 @@ for: - sh: "wget https://services.gradle.org/distributions/gradle-5.5-bin.zip -P /tmp" - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip" - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH" + - sh: "go env -w GO111MODULE=auto" build_script: - "python -c \"import sys; print(sys.executable)\"" From ca17b799c4f064406d1361a072124683d23b50f3 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Mon, 17 May 2021 11:48:21 -0700 Subject: [PATCH 4/8] Fix GOPATH env var in appveyor script --- .appveyor.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index bf6c6fb76..bf62b8bd0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,7 +4,6 @@ image: - Ubuntu environment: - GOPATH: c:\gopath GOVERSION: 1.11 GRADLE_OPTS: -Dorg.gradle.daemon=false nodejs_version: "8.10.0" @@ -45,6 +44,9 @@ for: only: - image: Visual Studio 2017 + environment: + GOPATH: c:\gopath + install: # To run Nodejs workflow integ tests - ps: Install-Product node 8.10 From 337a948b3d2509bb6eda34b9a9f5dbad1baec144 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Mon, 17 May 2021 12:17:39 -0700 Subject: [PATCH 5/8] Add note about setting GO111MODULE in appveyor script --- .appveyor.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index bf62b8bd0..61a5b8032 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -67,6 +67,9 @@ for: - "choco install dep" - setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;" - "go version" + # set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory + # https://blog.golang.org/go116-module-changes#TOC_2. + # This is required for the go dep integration tests - "go env -w GO111MODULE=auto" - "go env" @@ -105,6 +108,9 @@ for: - sh: "wget https://services.gradle.org/distributions/gradle-5.5-bin.zip -P /tmp" - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip" - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH" + # set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory + # https://blog.golang.org/go116-module-changes#TOC_2. + # This is required for the go dep integration tests - sh: "go env -w GO111MODULE=auto" build_script: From 9657532e3731f91217f3db85d056e1a512c864dc Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Tue, 18 May 2021 09:49:06 -0700 Subject: [PATCH 6/8] Update python_pip integration test --- tests/integration/workflows/python_pip/test_python_pip.py | 2 +- .../workflows/python_pip/testdata/requirements-numpy.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index dec957957..cd25de7dd 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -48,7 +48,7 @@ def test_must_build_python_project(self): if self.runtime == "python2.7": expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) else: - expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) + expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) diff --git a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt index ff68cfc1c..4ddfa98cd 100644 --- a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt +++ b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt @@ -1,2 +1,2 @@ numpy==1.15.4; python_version == '2.7' -numpy==1.17.4; python_version >= '3.6' +numpy==1.20.3; python_version >= '3.6' From 1bba6185792e9ee952c4474a072e0bc160ec5254 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Tue, 18 May 2021 10:47:14 -0700 Subject: [PATCH 7/8] Fix numpy version for py36 in integ test --- tests/integration/workflows/python_pip/test_python_pip.py | 2 ++ .../workflows/python_pip/testdata/requirements-numpy.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index cd25de7dd..745e560d7 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -47,6 +47,8 @@ def test_must_build_python_project(self): if self.runtime == "python2.7": expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) + elif self.runtime == "python3.6": + expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) else: expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) output_files = set(os.listdir(self.artifacts_dir)) diff --git a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt index 4ddfa98cd..5a57de403 100644 --- a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt +++ b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt @@ -1,2 +1,3 @@ numpy==1.15.4; python_version == '2.7' -numpy==1.20.3; python_version >= '3.6' +numpy==1.17.4; python_version == '3.6' +numpy==1.20.3; python_version >= '3.7' From 58404385bfd4e74d55223fa41c4b720fb04da5e8 Mon Sep 17 00:00:00 2001 From: Chris Rehn Date: Thu, 20 May 2021 12:41:51 -0700 Subject: [PATCH 8/8] Update aws_lambda_builders/workflows/python_pip/packager.py --- aws_lambda_builders/workflows/python_pip/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index ebca1bafb..c427a958b 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -399,7 +399,7 @@ def _is_compatible_platform_tag(self, expected_abi, platform): def _iter_all_compatibility_tags(self, wheel): """ Generates all possible combination of tag sets as described in PEP 425 - https://www.python.org/dev/peps/pep-0425/#id15 + https://www.python.org/dev/peps/pep-0425/#compressed-tag-sets """ implementation_tag, abi_tag, platform_tag = wheel.split("-")[-3:] for implementation in implementation_tag.split("."):