Support PEP 600 platform tags

.appveyor.yml

@@ -4,7 +4,6 @@ image:
   - Ubuntu
 
 environment:
-  GOPATH: c:\gopath
   GOVERSION: 1.11
   GRADLE_OPTS: -Dorg.gradle.daemon=false
   nodejs_version: "8.10.0"
@@ -45,6 +44,9 @@ for:
       only:
         - image: Visual Studio 2017
 
+    environment:
+      GOPATH: c:\gopath
+
     install:
     # To run Nodejs workflow integ tests
     - ps: Install-Product node 8.10
@@ -65,6 +67,10 @@ for:
     - "choco install dep"
     - setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
     - "go version"
+    # set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory
+    # https://blog.golang.org/go116-module-changes#TOC_2.
+    # This is required for the go dep integration tests
+    - "go env -w GO111MODULE=auto"
     - "go env"
 
     # setup Gradle
@@ -102,6 +108,10 @@ for:
       - sh: "wget https://services.gradle.org/distributions/gradle-5.5-bin.zip -P /tmp"
       - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip"
       - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH"
+      # set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory
+      # https://blog.golang.org/go116-module-changes#TOC_2.
+      # This is required for the go dep integration tests
+      - sh: "go env -w GO111MODULE=auto"
 
     build_script:
       - "python -c \"import sys; print(sys.executable)\""

aws_lambda_builders/workflows/python_pip/packager.py

@@ -150,13 +150,23 @@ class DependencyBuilder(object):
     packager.
     """
 
-    _MANYLINUX_COMPATIBLE_PLATFORM = {
-        "any",
-        "linux_x86_64",
-        "manylinux1_x86_64",
-        "manylinux2010_x86_64",
-        "manylinux2014_x86_64",
+    _ADDITIONAL_COMPATIBLE_PLATFORM = {"any", "linux_x86_64"}
+    _MANYLINUX_LEGACY_MAP = {
+        "manylinux1_x86_64": "manylinux_2_5_x86_64",
+        "manylinux2010_x86_64": "manylinux_2_12_x86_64",
+        "manylinux2014_x86_64": "manylinux_2_17_x86_64",
     }
+    # Mapping of abi to glibc version in Lambda runtime.
+    _RUNTIME_GLIBC = {
+        "cp27mu": (2, 17),
+        "cp36m": (2, 17),
+        "cp37m": (2, 17),
+        "cp38": (2, 26),
+    }
+    # Fallback version if we're on an unknown python version
+    # not in _RUNTIME_GLIBC.
+    # Unlikely to hit this case.
+    _DEFAULT_GLIBC = (2, 17)
     _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"}
 
     def __init__(self, osutils, runtime, pip_runner=None):
@@ -341,30 +351,62 @@ def _categorize_wheel_files(self, directory):
 
     def _is_compatible_wheel_filename(self, filename):
         wheel = filename[:-4]
-        implementation, abi, platform = wheel.split("-")[-3:]
-        # Verify platform is compatible
-        if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM:
-            return False
-
         lambda_runtime_abi = get_lambda_abi(self.runtime)
+        for implementation, abi, platform in self._iter_all_compatibility_tags(wheel):
+            if not self._is_compatible_platform_tag(lambda_runtime_abi, platform):
+                continue
+
+            # Verify that the ABI is compatible with lambda. Either none or the
+            # correct type for the python version cp27mu for py27 and cp36m for
+            # py36.
+            if abi == "none":
+                return True
+            prefix_version = implementation[:3]
+            if prefix_version == "cp3":
+                # Deploying python 3 function which means we need cp36m abi
+                # We can also accept abi3 which is the CPython 3 Stable ABI and
+                # will work on any version of python 3.
+                if abi == lambda_runtime_abi or abi == "abi3":
+                    return True
+            elif prefix_version == "cp2":
+                # Deploying to python 2 function which means we need cp27mu abi
+                if abi == "cp27mu":
+                    return True
+        # Don't know what we have but it didn't pass compatibility tests.
+        return False
 
-        # Verify that the ABI is compatible with lambda. Either none or the
-        # correct type for the python version cp27mu for py27 and cp36m for
-        # py36.
-        if abi == "none":
+    def _is_compatible_platform_tag(self, expected_abi, platform):
+        """
+        Verify if a platform tag is compatible based on PEP 600
+        https://www.python.org/dev/peps/pep-0600/#specification
+
+        In addition to checking the tag pattern, we also need to verify the glibc version
+        """
+        if platform in self._ADDITIONAL_COMPATIBLE_PLATFORM:
             return True
-        prefix_version = implementation[:3]
-        if prefix_version == "cp3":
-            # Deploying python 3 function which means we need cp36m abi
-            # We can also accept abi3 which is the CPython 3 Stable ABI and
-            # will work on any version of python 3.
-            return abi == lambda_runtime_abi or abi == "abi3"
-        elif prefix_version == "cp2":
-            # Deploying to python 2 function which means we need cp27mu abi
-            return abi == "cp27mu"
-        # Don't know what we have but it didn't pass compatibility tests.
+        elif platform.startswith("manylinux"):
+            perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform)
+            m = re.match("manylinux_([0-9]+)_([0-9]+)_(.*)", perennial_tag)
+            if m is None:
+                return False
+            tag_major, tag_minor = [int(x) for x in m.groups()[:2]]
+            runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC)
+            if (tag_major, tag_minor) <= (runtime_major, runtime_minor):
+                # glibc version is compatible with Lambda Runtime
+                return True
         return False
 
+    def _iter_all_compatibility_tags(self, wheel):
+        """
+        Generates all possible combination of tag sets as described in PEP 425
+        https://www.python.org/dev/peps/pep-0425/#compressed-tag-sets
+        """
+        implementation_tag, abi_tag, platform_tag = wheel.split("-")[-3:]
+        for implementation in implementation_tag.split("."):
+            for abi in abi_tag.split("."):
+                for platform in platform_tag.split("."):
+                    yield (implementation, abi, platform)
+
     def _apply_wheel_allowlist(self, compatible_wheels, incompatible_wheels):
         compatible_wheels = set(compatible_wheels)
         actual_incompatible_wheels = set()

tests/functional/workflows/python_pip/test_packager.py

@@ -449,6 +449,48 @@ def test_can_get_whls_mixed_compat(self, tmpdir, osutils, pip_runner):
         for req in reqs:
             assert req in installed_packages
 
+    def test_can_support_pep_600_tags(self, tmpdir, osutils, pip_runner):
+        reqs = ["foo"]
+        pip, runner = pip_runner
+        appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
+        requirements_file = os.path.join(appdir, "requirements.txt")
+        pip.packages_to_download(
+            expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
+            packages=[
+                "foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl",
+            ],
+        )
+
+        site_packages = os.path.join(appdir, ".chalice.", "site-packages")
+        with osutils.tempdir() as scratch_dir:
+            builder.build_site_packages(requirements_file, site_packages, scratch_dir)
+        installed_packages = os.listdir(site_packages)
+
+        pip.validate()
+        for req in reqs:
+            assert req in installed_packages
+
+    def test_can_support_compressed_tags(self, tmpdir, osutils, pip_runner):
+        reqs = ["foo"]
+        pip, runner = pip_runner
+        appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
+        requirements_file = os.path.join(appdir, "requirements.txt")
+        pip.packages_to_download(
+            expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
+            packages=[
+                "foo-1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl",
+            ],
+        )
+
+        site_packages = os.path.join(appdir, ".chalice.", "site-packages")
+        with osutils.tempdir() as scratch_dir:
+            builder.build_site_packages(requirements_file, site_packages, scratch_dir)
+        installed_packages = os.listdir(site_packages)
+
+        pip.validate()
+        for req in reqs:
+            assert req in installed_packages
+
     def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner):
         reqs = ["foo", "bar", "baz"]
         pip, runner = pip_runner
@@ -539,6 +581,36 @@ def test_does_fail_on_python_1_whl(self, tmpdir, osutils, pip_runner):
         assert missing_packages[0].identifier == "baz==1.5"
         assert len(installed_packages) == 0
 
+    def test_does_fail_on_pep_600_tag_with_unsupported_glibc_version(self, tmpdir, osutils, pip_runner):
+        reqs = ["foo", "bar", "baz", "qux"]
+        pip, runner = pip_runner
+        appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
+        requirements_file = os.path.join(appdir, "requirements.txt")
+        pip.packages_to_download(
+            expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
+            packages=[
+                "foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl",
+                "bar-1.2-cp36-cp36m-manylinux_2_999_x86_64.whl",
+                "baz-1.2-cp36-cp36m-manylinux_3_12_x86_64.whl",
+                "qux-1.2-cp36-cp36m-manylinux_3_999_x86_64.whl",
+            ],
+        )
+
+        site_packages = os.path.join(appdir, ".chalice.", "site-packages")
+        with osutils.tempdir() as scratch_dir:
+            with pytest.raises(MissingDependencyError) as e:
+                builder.build_site_packages(requirements_file, site_packages, scratch_dir)
+        installed_packages = os.listdir(site_packages)
+
+        missing_packages = list(e.value.missing)
+        pip.validate()
+        assert len(missing_packages) == 3
+        missing_package_identifies = [package.identifier for package in missing_packages]
+        assert "bar==1.2" in missing_package_identifies
+        assert "baz==1.2" in missing_package_identifies
+        assert "qux==1.2" in missing_package_identifies
+        assert len(installed_packages) == 1
+
     def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner):
         reqs = ["foo", "bar"]
         pip, runner = pip_runner

tests/integration/workflows/python_pip/test_python_pip.py

@@ -47,8 +47,10 @@ def test_must_build_python_project(self):
 
         if self.runtime == "python2.7":
             expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
-        else:
+        elif self.runtime == "python3.6":
             expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"})
+        else:
+            expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"})
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEqual(expected_files, output_files)
 

tests/integration/workflows/python_pip/testdata/requirements-numpy.txt

@@ -1,2 +1,3 @@
 numpy==1.15.4; python_version == '2.7'
-numpy==1.17.4; python_version >= '3.6'
+numpy==1.17.4; python_version == '3.6'
+numpy==1.20.3; python_version >= '3.7'