From e31a7fe9e93c8c8d884bd1e24a4124497c268ccb Mon Sep 17 00:00:00 2001 From: Sean O Brien Date: Thu, 16 Mar 2023 22:40:37 +0000 Subject: [PATCH 1/3] feat: add python3.10 support (#32) * Add python3.10 support. * add support for py3.10 * add ci jobs for py3.10 * Fix python3.10 workflows. --------- Co-authored-by: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> --- .github/workflows/build.yml | 34 ++++++++++++------- aws_lambda_builders/validator.py | 1 + .../workflows/python_pip/DESIGN.md | 2 +- .../workflows/python_pip/packager.py | 3 +- setup.py | 1 + .../workflows/python_pip/test_python_pip.py | 6 +++- .../testdata/requirements-numpy.txt | 5 ++- .../workflows/python_pip/test_packager.py | 3 ++ 8 files changed, 39 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f11dbc7de..2f7b85c95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,6 +20,7 @@ jobs: - "3.7" - "3.8" - "3.9" + - "3.10" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -44,6 +45,7 @@ jobs: - "3.7" - "3.8" - "3.9" + - "3.10" npm: - 8 - 9 @@ -75,6 +77,7 @@ jobs: - "3.7" - "3.8" - "3.9" + - "3.10" npm: - 8 - 9 @@ -106,6 +109,7 @@ jobs: - "3.7" - "3.8" - "3.9" + - "3.10" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -128,9 +132,10 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.9" - - "3.8" - "3.7" + - "3.8" + - "3.9" + - "3.10" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -156,9 +161,10 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.9" - - "3.8" - "3.7" + - "3.8" + - "3.9" + - "3.10" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -182,9 +188,10 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.9" - - "3.8" - "3.7" + - "3.8" + - "3.9" + - "3.10" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -204,9 +211,10 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.9" - - "3.8" - "3.7" + - "3.8" + - "3.9" + - "3.10" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -226,9 +234,10 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.9" - - "3.8" - "3.7" + - "3.8" + - "3.9" + - "3.10" ruby: - "3.1" - "2.7" @@ -254,9 +263,10 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.9" - - "3.8" - "3.7" + - "3.8" + - "3.9" + - "3.10" steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py index ce66282a7..17f27935a 100644 --- a/aws_lambda_builders/validator.py +++ b/aws_lambda_builders/validator.py @@ -19,6 +19,7 @@ "python3.7": [X86_64], "python3.8": [ARM64, X86_64], "python3.9": [ARM64, X86_64], + "python3.10": [ARM64, X86_64], "ruby2.7": [ARM64, X86_64], "java8": [ARM64, X86_64], "java11": [ARM64, X86_64], diff --git a/aws_lambda_builders/workflows/python_pip/DESIGN.md b/aws_lambda_builders/workflows/python_pip/DESIGN.md index 7513c25f1..3376f62f8 100644 --- a/aws_lambda_builders/workflows/python_pip/DESIGN.md +++ b/aws_lambda_builders/workflows/python_pip/DESIGN.md @@ -49,7 +49,7 @@ def build_dependencies(artifacts_dir_path, :type runtime: str :param runtime: Python version to build dependencies for. This can - either be python3.6, python3.7, python3.8 or python3.9. These are + either be python3.6, python3.7, python3.8, python3.9 or python3.10. These are currently the only supported values. :type ui: :class:`lambda_builders.actions.python_pip.utils.UI` diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index f43b0c274..bd22f9151 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -86,6 +86,7 @@ def get_lambda_abi(runtime): "python3.7": "cp37m", "python3.8": "cp38", "python3.9": "cp39", + "python3.10": "cp310", } if runtime not in supported: @@ -100,7 +101,7 @@ def __init__(self, runtime, osutils=None, dependency_builder=None, architecture= :type runtime: str :param runtime: Python version to build dependencies for. This can - either be python3.6, python3.7, python3.8 or python3.9. These are currently the + either be python3.6, python3.7, python3.8, python3.9 or python3.10. These are currently the only supported values. :type osutils: :class:`lambda_builders.utils.OSUtils` diff --git a/setup.py b/setup.py index 2d578ab07..978d5dc71 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def read_version(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Internet", "Topic :: Software Development :: Build Tools", "Topic :: Utilities", diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 3e1fe6cb3..214c7b463 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -18,7 +18,7 @@ logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow") IS_WINDOWS = platform.system().lower() == "windows" NOT_ARM = platform.processor() != "aarch64" -ARM_RUNTIMES = {"python3.8", "python3.9"} +ARM_RUNTIMES = {"python3.8", "python3.9", "python3.10"} @parameterized_class(("experimental_flags",), [([]), ([EXPERIMENTAL_FLAG_BUILD_PERFORMANCE])]) @@ -61,6 +61,7 @@ def setUp(self): "python3.7": "python3.8", "python3.8": "python3.9", "python3.9": "python3.7", + "python3.10": "python3.9", } def tearDown(self): @@ -96,6 +97,9 @@ def test_must_build_python_project(self): if self.runtime == "python3.6": self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) + elif self.runtime == "python3.10": + self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.23.5.dist-info", "numpy.libs"}) else: self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) diff --git a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt index 5a57de403..66c383b7f 100644 --- a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt +++ b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt @@ -1,3 +1,6 @@ numpy==1.15.4; python_version == '2.7' numpy==1.17.4; python_version == '3.6' -numpy==1.20.3; python_version >= '3.7' +numpy==1.20.3; python_version == '3.7' +numpy==1.20.3; python_version == '3.8' +numpy==1.20.3; python_version == '3.9' +numpy==1.23.5; python_version == '3.10' \ No newline at end of file diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 7eaeb30b1..aac86f365 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -102,6 +102,9 @@ def test_get_lambda_abi_python38(self): def test_get_lambda_abi_python39(self): assert "cp39" == get_lambda_abi("python3.9") + def test_get_lambda_abi_python310(self): + assert "cp310" == get_lambda_abi("python3.10") + class TestPythonPipDependencyBuilder(object): def test_can_call_dependency_builder(self, osutils): From 88b992cfd7a87be1b2ad047ee179dfcd4a25746f Mon Sep 17 00:00:00 2001 From: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Date: Fri, 17 Mar 2023 14:51:18 -0700 Subject: [PATCH 2/3] Update DESIGN.md --- aws_lambda_builders/workflows/python_pip/DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/workflows/python_pip/DESIGN.md b/aws_lambda_builders/workflows/python_pip/DESIGN.md index 3376f62f8..3f8a96be4 100644 --- a/aws_lambda_builders/workflows/python_pip/DESIGN.md +++ b/aws_lambda_builders/workflows/python_pip/DESIGN.md @@ -49,7 +49,7 @@ def build_dependencies(artifacts_dir_path, :type runtime: str :param runtime: Python version to build dependencies for. This can - either be python3.6, python3.7, python3.8, python3.9 or python3.10. These are + either be python3.7, python3.8, python3.9 or python3.10. These are currently the only supported values. :type ui: :class:`lambda_builders.actions.python_pip.utils.UI` From f3fb014cdb6932e99cecfb1ab9fdba35cf1c729c Mon Sep 17 00:00:00 2001 From: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Date: Fri, 17 Mar 2023 15:58:07 -0700 Subject: [PATCH 3/3] update test cases with py310 differences --- .../workflows/python_pip/test_python_pip.py | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 9c7908006..a5b19f620 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -94,7 +94,7 @@ def test_must_build_python_project(self): ) if self.runtime == "python3.10": - self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-1.23.5.dist-info", "numpy.libs"}) else: self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) @@ -105,25 +105,31 @@ def test_must_build_python_project(self): def test_must_build_python_project_python3_binary(self): python_paths = which("python") - executable_dir = pathlib.Path(tempfile.gettempdir()) - new_python_path = executable_dir.joinpath("python3") - os.symlink(python_paths[0], new_python_path) - # Build with access to the newly symlinked python3 binary. - self.builder.build( - self.source_dir, - self.artifacts_dir, - self.scratch_dir, - self.manifest_path_valid, - runtime=self.runtime, - experimental_flags=self.experimental_flags, - executable_search_paths=[executable_dir], - ) - self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) - expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) + with tempfile.TemporaryDirectory() as executable_dir_str: + executable_dir = pathlib.Path(executable_dir_str) + new_python_path = executable_dir.joinpath("python3") + os.symlink(python_paths[0], new_python_path) + # Build with access to the newly symlinked python3 binary. + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime, + experimental_flags=self.experimental_flags, + executable_search_paths=[executable_dir], + ) - output_files = set(os.listdir(self.artifacts_dir)) - self.assertEqual(expected_files, output_files) - os.unlink(new_python_path) + if self.runtime == "python3.10": + self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.23.5.dist-info", "numpy.libs"}) + else: + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) + + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + os.unlink(new_python_path) @skipIf(NOT_ARM, "Skip if not running on ARM64") def test_must_build_python_project_from_sdist_with_arm(self): @@ -159,10 +165,15 @@ def test_must_build_python_project_with_arm_architecture(self): experimental_flags=self.experimental_flags, ) expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.20.3.dist-info"}) + if self.runtime == "python3.10": + expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.23.5.dist-info"}) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) - self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"]) + if self.runtime == "python3.10": + self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_aarch64"]) + else: + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"]) def test_mismatch_runtime_python_project(self): # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python3.7