diff --git a/.github/scripts/manylinux_build_and_test.sh b/.github/scripts/manylinux_build_and_test.sh new file mode 100644 index 00000000..01c6867f --- /dev/null +++ b/.github/scripts/manylinux_build_and_test.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -eu + +: "${PY_ABI:?PY_ABI is required}" +: "${MANYLINUX_IMAGE:?MANYLINUX_IMAGE is required}" + +# Make local build helpers importable for isolated PEP 517 backend subprocesses. +export PYTHONPATH="$PWD${PYTHONPATH:+:${PYTHONPATH}}" +# Ensure dependency archives are read from the restored workspace cache even in isolated builds. +export PYXMLSEC_LIBS_DIR="$PWD/libs" + +# Step: Install system build dependencies (manylinux only) +echo "== [container] Step: Install system build dependencies (manylinux only) ==" +case "$MANYLINUX_IMAGE" in + manylinux*) + yum install -y perl-core + ;; +esac + +# Step: Install python build dependencies +echo "== [container] Step: Install python build dependencies ==" +/opt/python/${PY_ABI}/bin/pip install --upgrade pip setuptools wheel build 'setuptools_scm>=8' + +# Step: Set environment variables +echo "== [container] Step: Set environment variables ==" +PKGVER=$(/opt/python/${PY_ABI}/bin/python setup.py --version) +echo "PKGVER=$PKGVER" + +# Step: Build linux_x86_64 wheel +echo "== [container] Step: Build linux_x86_64 wheel ==" +/opt/python/${PY_ABI}/bin/python -m build + +# Step: Label manylinux wheel +echo "== [container] Step: Label manylinux wheel ==" +ls -la dist/ +auditwheel show dist/xmlsec-${PKGVER}-${PY_ABI}-linux_x86_64.whl +auditwheel repair dist/xmlsec-${PKGVER}-${PY_ABI}-linux_x86_64.whl +ls -la wheelhouse/ +auditwheel show wheelhouse/xmlsec-${PKGVER}-${PY_ABI}-*${MANYLINUX_IMAGE}*.whl + +# Step: Install test dependencies +echo "== [container] Step: Install test dependencies ==" +/opt/python/${PY_ABI}/bin/pip install --upgrade -r requirements-test.txt +/opt/python/${PY_ABI}/bin/pip install xmlsec --only-binary=xmlsec --no-index --find-links=wheelhouse/ + +# Step: Run tests +echo "== [container] Step: Run tests ==" +/opt/python/${PY_ABI}/bin/pytest -v --color=yes + +# Step: Fix mounted workspace file ownership on host +echo "== [container] Step: Fix mounted workspace file ownership on host ==" +chown -R "${HOST_UID}:${HOST_GID}" dist wheelhouse build libs || true diff --git a/.github/workflows/cache_libs.yml b/.github/workflows/cache_libs.yml new file mode 100644 index 00000000..77c83353 --- /dev/null +++ b/.github/workflows/cache_libs.yml @@ -0,0 +1,134 @@ +name: Cache library dependencies + +on: + workflow_call: + inputs: + LIBICONV_VERSION: + default: "1.18" + required: false + type: string + LIBXML2_VERSION: + default: "2.14.6" + required: false + type: string + LIBXSLT_VERSION: + default: "1.1.43" + required: false + type: string + OPENSSL_VERSION: + default: "3.6.0" + required: false + type: string + XMLSEC1_VERSION: + default: "1.3.9" + required: false + type: string + ZLIB_VERSION: + default: "1.3.1" + required: false + type: string + WIN_LIBICONV_VERSION: + default: "1.18-1" + required: false + type: string + WIN_LIBXML2_VERSION: + default: "2.11.9-3" + required: false + type: string + WIN_LIBXSLT_VERSION: + default: "1.1.39" + required: false + type: string + WIN_OPENSSL_VERSION: + default: "3.0.16.pl1" + required: false + type: string + WIN_XMLSEC1_VERSION: + default: "1.3.7" + required: false + type: string + WIN_ZLIB_VERSION: + default: "1.3.1" + required: false + type: string + + outputs: + LIBICONV_VERSION: + value: ${{ inputs.LIBICONV_VERSION }} + LIBXML2_VERSION: + value: ${{ inputs.LIBXML2_VERSION }} + LIBXSLT_VERSION: + value: ${{ inputs.LIBXSLT_VERSION }} + OPENSSL_VERSION: + value: ${{ inputs.OPENSSL_VERSION }} + XMLSEC1_VERSION: + value: ${{ inputs.XMLSEC1_VERSION }} + ZLIB_VERSION: + value: ${{ inputs.ZLIB_VERSION }} + WIN_LIBICONV_VERSION: + value: ${{ inputs.WIN_LIBICONV_VERSION }} + WIN_LIBXML2_VERSION: + value: ${{ inputs.WIN_LIBXML2_VERSION }} + WIN_LIBXSLT_VERSION: + value: ${{ inputs.WIN_LIBXSLT_VERSION }} + WIN_OPENSSL_VERSION: + value: ${{ inputs.WIN_OPENSSL_VERSION }} + WIN_XMLSEC1_VERSION: + value: ${{ inputs.WIN_XMLSEC1_VERSION }} + WIN_ZLIB_VERSION: + value: ${{ inputs.WIN_ZLIB_VERSION }} + +jobs: + cache_libs: + strategy: + fail-fast: false + matrix: + os: + - "ubuntu-22.04" + - "ubuntu-22.04-arm" + - "macos-latest" + - "windows-2022" + - "windows-11-arm" + + runs-on: ${{ matrix.os }} + + env: + LIBICONV_VERSION: ${{ contains(matrix.os, 'windows-') && inputs.WIN_LIBICONV_VERSION || inputs.LIBICONV_VERSION }} + LIBXML2_VERSION: ${{ contains(matrix.os, 'windows-') && inputs.WIN_LIBXML2_VERSION || inputs.LIBXML2_VERSION }} + LIBXSLT_VERSION: ${{ contains(matrix.os, 'windows-') && inputs.WIN_LIBXSLT_VERSION || inputs.LIBXSLT_VERSION }} + OPENSSL_VERSION: ${{ contains(matrix.os, 'windows-') && inputs.WIN_OPENSSL_VERSION || inputs.OPENSSL_VERSION }} + XMLSEC1_VERSION: ${{ contains(matrix.os, 'windows-') && inputs.WIN_XMLSEC1_VERSION || inputs.XMLSEC1_VERSION }} + ZLIB_VERSION: ${{ contains(matrix.os, 'windows-') && inputs.WIN_ZLIB_VERSION || inputs.ZLIB_VERSION }} + + steps: + - uses: actions/checkout@v6 + + - name: Cache [libs] + uses: actions/cache@v4.3.0 + with: + path: | + libs/*.xz + libs/*.gz + libs/*.zip + key: libs-${{ runner.os }}-${{ runner.arch }}-${{ env.LIBXML2_VERSION }}-${{ env.LIBXSLT_VERSION }} + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install setuptools shim + run: python -m pip install --upgrade pip setuptools + + - name: Download latest libraries + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python build_libs_xmlsec.py --download-only + + - name: Check Windows library versions + if: ${{ contains(matrix.os, 'windows-') }} + run: | + bash -c ' + for file in libs/iconv-${{ inputs.WIN_LIBICONV_VERSION }}.*.zip libs/libxml2-${{ inputs.WIN_LIBXML2_VERSION }}.*.zip libs/libxslt-${{ inputs.WIN_LIBXSLT_VERSION }}.*.zip libs/openssl-${{ inputs.WIN_OPENSSL_VERSION }}.*.zip libs/xmlsec-${{ inputs.WIN_XMLSEC1_VERSION }}.*.zip libs/zlib-${{ inputs.WIN_ZLIB_VERSION }}.*.zip; do + [[ -f "$file" ]] || { echo "MISSING: $file" ; exit 1; } + done + ' diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 1c6d9543..51b0db1e 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -3,27 +3,34 @@ on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref_name != 'master' }} + jobs: linuxbrew: runs-on: ubuntu-latest + strategy: matrix: python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + env: # For some unknown reason, linuxbrew tries to use "gcc-11" by default, which doesn't exist. CC: gcc + steps: - uses: actions/checkout@v3 + - name: Install brew run: | sudo apt install -y build-essential procps curl file git /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" echo "/home/linuxbrew/.linuxbrew/bin" >> $GITHUB_PATH + - name: Install build dependencies run: | brew update brew install python@${{ matrix.python }} gcc libxml2 libxmlsec1 pkg-config echo "/home/linuxbrew/.linuxbrew/opt/python@${{ matrix.python }}/libexec/bin" >> $GITHUB_PATH + - name: Build wheel run: | python3 -m venv build_venv @@ -33,6 +40,7 @@ jobs: export LDFLAGS="-L$(brew --prefix)/lib" python3 -m build rm -rf build/ + - name: Run tests run: | python3 -m venv test_venv diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index e2e2a0df..522a2a0a 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -3,23 +3,47 @@ on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref_name != 'master' }} + jobs: + cache_libs: + uses: ./.github/workflows/cache_libs.yml + secrets: inherit + macosx: + needs: cache_libs runs-on: macos-latest + + env: + LIBXML2_VERSION: ${{ needs.cache_libs.outputs.LIBXML2_VERSION }} + LIBXSLT_VERSION: ${{ needs.cache_libs.outputs.LIBXSLT_VERSION }} + strategy: matrix: python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] static_deps: ["static", ""] + steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 + - uses: actions/checkout@v5.0.0 + + - name: Cache [libs] + id: cache-libs + uses: actions/cache/restore@v4.3.0 + with: + path: | + libs/*.xz + libs/*.gz + libs/*.zip + key: libs-${{ runner.os }}-${{ runner.arch }}-${{ env.LIBXML2_VERSION }}-${{ env.LIBXSLT_VERSION }} + + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel build brew install libxml2 libxmlsec1 pkg-config + - name: Build macosx_x86_64 wheel env: CC: clang @@ -29,26 +53,38 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" - export PYXMLSEC_LIBXML2_VERSION="$(pkg-config --modversion libxml-2.0)" + export PYXMLSEC_LIBS_DIR="$PWD/libs" python -m build rm -rf build/ + - name: Set environment variables shell: bash run: | echo "PKGVER=$(python setup.py --version)" >> $GITHUB_ENV echo "LLVM_PROFILE_FILE=pyxmlsec.profraw" >> $GITHUB_ENV - - name: Install test dependencies + + - name: Install test dependencies (static only) + if: matrix.static_deps == 'static' + run: | + pip install coverage --upgrade -r requirements-test.txt + pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ + + + - name: Install test dependencies (non-static only) + if: matrix.static_deps != 'static' run: | export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" pip install coverage --upgrade --no-binary=lxml -r requirements-test.txt pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV + - name: Run tests run: | coverage run -m pytest -v --color=yes + - name: Report coverage to codecov + if: matrix.static_deps != 'static' run: | /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=$(uname -m) --instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt - if: matrix.static_deps != 'static' diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index d1c205d7..ff04f9ae 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -3,9 +3,20 @@ on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref_name != 'master' }} + jobs: + cache_libs: + uses: ./.github/workflows/cache_libs.yml + secrets: inherit + manylinux: + needs: cache_libs runs-on: ubuntu-latest + + env: + LIBXML2_VERSION: ${{ needs.cache_libs.outputs.LIBXML2_VERSION }} + LIBXSLT_VERSION: ${{ needs.cache_libs.outputs.LIBXSLT_VERSION }} + strategy: matrix: python-abi: [cp39-cp39, cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313, cp314-cp314] @@ -13,40 +24,39 @@ jobs: - manylinux2014_x86_64 - manylinux_2_28_x86_64 - musllinux_1_2_x86_64 - container: quay.io/pypa/${{ matrix.image }} + steps: - - uses: actions/checkout@v1 - - name: Install python build dependencies - run: | - # https://github.com/actions/runner/issues/2033 - chown -R $(id -u):$(id -g) $PWD - /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade pip setuptools wheel build - - name: Install system build dependencies (manylinux) - run: | - yum install -y perl-core - if: contains(matrix.image, 'manylinux') - - name: Set environment variables - shell: bash - run: | - echo "PKGVER=$(/opt/python/${{ matrix.python-abi }}/bin/python setup.py --version)" >> $GITHUB_ENV - - name: Build linux_x86_64 wheel + - uses: actions/checkout@v5.0.0 + with: + fetch-depth: 0 + + - name: Cache [libs] + uses: actions/cache/restore@v4.3.0 + with: + path: | + libs/*.xz + libs/*.gz + libs/*.zip + key: libs-${{ runner.os }}-${{ runner.arch }}-${{ env.LIBXML2_VERSION }}-${{ env.LIBXSLT_VERSION }} + + # Keep this job on the host runner so JS-based actions (for example actions/cache) + # can run, then execute build/test inside the target manylinux/musllinux container. + - name: Build and test in container env: PYXMLSEC_STATIC_DEPS: true - PYXMLSEC_LIBXML2_VERSION: 2.14.6 # Lock it to libxml2 2.14.6 to match it with lxml GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PY_ABI: ${{ matrix.python-abi }} + MANYLINUX_IMAGE: ${{ matrix.image }} run: | - /opt/python/${{ matrix.python-abi }}/bin/python -m build - - name: Label manylinux wheel - run: | - ls -la dist/ - auditwheel show dist/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-linux_x86_64.whl - auditwheel repair dist/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-linux_x86_64.whl - ls -la wheelhouse/ - auditwheel show wheelhouse/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-*${{ matrix.image }}*.whl - - name: Install test dependencies - run: | - /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade -r requirements-test.txt - /opt/python/${{ matrix.python-abi }}/bin/pip install xmlsec --only-binary=xmlsec --no-index --find-links=wheelhouse/ - - name: Run tests - run: | - /opt/python/${{ matrix.python-abi }}/bin/pytest -v --color=yes + set -euxo pipefail + docker run --rm \ + -v "$PWD:$PWD" \ + -w "$PWD" \ + -e GH_TOKEN \ + -e PYXMLSEC_STATIC_DEPS \ + -e PY_ABI \ + -e MANYLINUX_IMAGE \ + -e HOST_UID="$(id -u)" \ + -e HOST_GID="$(id -g)" \ + "quay.io/pypa/${MANYLINUX_IMAGE}" \ + sh .github/scripts/manylinux_build_and_test.sh diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index ecc53c31..f48ca02c 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -3,6 +3,7 @@ on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref_name != 'master' }} + jobs: sdist: # Avoid Ubuntu 24.04 in sdist workflows, because it contains libxmlsec1-dev @@ -10,27 +11,33 @@ jobs: # (It thinks the softhsm engine has a public key instead of a private key.) # libxmlsec1 <=1.2.33 or >=1.2.42 works. TODO: Try 26.04 when available. runs-on: ubuntu-22.04 + strategy: matrix: python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + steps: - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 + + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + - name: Install build dependencies run: | - pip install --upgrade pip setuptools wheel + pip install --upgrade pip setuptools wheel 'setuptools_scm>=8' + - name: Package source dist run: | python setup.py sdist + - name: Install test dependencies run: | sudo apt-get update sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl pip install --upgrade -r requirements-test.txt --no-binary lxml pip install dist/xmlsec-$(python setup.py --version).tar.gz + - name: Run tests run: | pytest -v --color=yes diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 1d4564a6..fa80a645 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -22,6 +22,10 @@ concurrency: permissions: {} jobs: + cache_libs: + uses: ./.github/workflows/cache_libs.yml + secrets: inherit + sdist: # Avoid Ubuntu 24.04 in sdist workflows, because it contains libxmlsec1-dev # v1.2.39, which has a bug that causes tests/test_pkcs11.py to fail. @@ -33,18 +37,18 @@ jobs: contents: write steps: - - uses: actions/checkout@v4.1.1 + - uses: actions/checkout@v5.0.0 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@v6.0.0 with: - python-version: "3.x" + python-version: '3.14' - name: Install build dependencies run: | - pip install --upgrade pip setuptools wheel + pip install --upgrade pip setuptools wheel setuptools_scm>=8 pkgconfig>=1.5.1 - name: Package source dist run: python setup.py sdist @@ -60,34 +64,35 @@ jobs: run: pytest -v --color=yes - name: Upload sdist - uses: actions/upload-artifact@v4.3.1 + uses: actions/upload-artifact@v5.0.0 with: name: sdist path: dist/*.tar.gz - generate-wheels-matrix: + generate_wheels_matrix: # Create a matrix of all architectures & versions to build. # This enables the next step to run cibuildwheel in parallel. # From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210 name: Generate wheels matrix - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + outputs: include: ${{ steps.set-matrix.outputs.include }} + steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5.0.0 + - name: Install cibuildwheel - # Nb. keep cibuildwheel version pin consistent with job below - run: pipx install cibuildwheel==3.1.4 + # N.B. Keep cibuildwheel version pin consistent with "build_wheels" job below. + run: pipx install cibuildwheel==3.3 + - id: set-matrix - # Once we have the windows build figured out, it can be added here - # by updating the matrix to include windows builds as well. - # See example here: - # https://github.com/lxml/lxml/blob/3ccc7d583e325ceb0ebdf8fc295bbb7fc8cd404d/.github/workflows/wheels.yml#L95C1-L106C51 run: | MATRIX=$( { cibuildwheel --print-build-identifiers --platform linux \ - | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + | jq -nRc '{"only": inputs, "os": "ubuntu-22.04"}' \ + | sed -e '/aarch64/s|ubuntu-22.04|ubuntu-22.04-arm|' \ && cibuildwheel --print-build-identifiers --platform macos \ | jq -nRc '{"only": inputs, "os": "macos-latest"}' \ && cibuildwheel --print-build-identifiers --platform windows \ @@ -101,38 +106,48 @@ jobs: build_wheels: name: Build for ${{ matrix.only }} - needs: generate-wheels-matrix + needs: [cache_libs, generate_wheels_matrix] runs-on: ${{ matrix.os }} + env: + LIBXML2_VERSION: ${{ contains(matrix.os, 'windows-') && needs.cache_libs.outputs.WIN_LIBXML2_VERSION || needs.cache_libs.outputs.LIBXML2_VERSION }} + LIBXSLT_VERSION: ${{ contains(matrix.os, 'windows-') && needs.cache_libs.outputs.WIN_LIBXSLT_VERSION || needs.cache_libs.outputs.LIBXSLT_VERSION }} + strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} - - env: - PYXMLSEC_LIBXML2_VERSION: 2.14.6 - PYXMLSEC_LIBXSLT_VERSION: 1.1.43 + include: ${{ fromJson(needs.generate_wheels_matrix.outputs.include) }} steps: - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@v5.0.0 with: fetch-depth: 0 + - name: Cache [libs] + uses: actions/cache/restore@v4.3.0 + with: + path: | + libs/*.xz + libs/*.gz + libs/*.zip + key: libs-${{ runner.os }}-${{ runner.arch }}-${{ env.LIBXML2_VERSION }}-${{ env.LIBXSLT_VERSION }} + - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v3.7.0 with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v3.1.4 + uses: pypa/cibuildwheel@v3.3.0 with: only: ${{ matrix.only }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/upload-artifact@v4.3.1 + - name: Upload wheels + uses: actions/upload-artifact@v5.0.0 with: path: ./wheelhouse/*.whl name: xmlsec-wheel-${{ matrix.only }} diff --git a/build_libs_xmlsec.py b/build_libs_xmlsec.py new file mode 100644 index 00000000..a4a7e603 --- /dev/null +++ b/build_libs_xmlsec.py @@ -0,0 +1,55 @@ +import argparse +import os +import sys +from pathlib import Path + +from build_support.lib_xmlsec_dependency_builder import LibXmlsecDependencyBuilder + + +def _console_info(message): + print(message) + + +def main(argv=None): + parser = argparse.ArgumentParser(description='Download and build static dependency libraries for python-xmlsec.') + parser.add_argument( + '--platform', + default=sys.platform, + help='Target platform (default: current interpreter platform).', + ) + parser.add_argument( + '--plat-name', + default=os.environ.get('PYXMLSEC_PLAT_NAME'), + help='Target platform tag for cross-compiling (for example macosx-11.0-arm64).', + ) + parser.add_argument( + '--libs-dir', + default=os.environ.get('PYXMLSEC_LIBS_DIR', 'libs'), + help='Directory where source/binary archives are stored.', + ) + parser.add_argument( + '--buildroot', + default=Path('build', 'tmp'), + type=Path, + help='Build root for extracted/build artifacts.', + ) + parser.add_argument( + '--download-only', + action='store_true', + help='Only download dependency archives; do not extract/build.', + ) + + args = parser.parse_args(argv) + builder = LibXmlsecDependencyBuilder( + platform_name=args.platform, + info=_console_info, + libs_dir=Path(args.libs_dir), + buildroot=args.buildroot, + plat_name=args.plat_name, + ) + builder.prepare(download_only=args.download_only) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/build_support/build_ext.py b/build_support/build_ext.py index c4fb5bc9..262acd97 100644 --- a/build_support/build_ext.py +++ b/build_support/build_ext.py @@ -9,15 +9,18 @@ class build_ext(build_ext_orig): - def info(self, message): - self.announce(message, level=log.INFO) + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) - def run(self): - ext = self.ext_map['xmlsec'] self.debug = os.environ.get('PYXMLSEC_ENABLE_DEBUG', False) self.static = os.environ.get('PYXMLSEC_STATIC_DEPS', False) self.size_opt = os.environ.get('PYXMLSEC_OPTIMIZE_SIZE', True) + def info(self, message) -> None: + self.announce(message, level=log.INFO) + + def run(self) -> None: + ext = self.ext_map['xmlsec'] if self.static or sys.platform == 'win32': helper = StaticBuildHelper(self) helper.prepare(sys.platform) diff --git a/build_support/lib_xmlsec_dependency_builder.py b/build_support/lib_xmlsec_dependency_builder.py new file mode 100644 index 00000000..00b519dd --- /dev/null +++ b/build_support/lib_xmlsec_dependency_builder.py @@ -0,0 +1,417 @@ +import multiprocessing +import os +import platform +import subprocess +import sys +import tarfile +import zipfile +from dataclasses import dataclass +from distutils.errors import DistutilsError +from pathlib import Path +from typing import ClassVar +from urllib.parse import urljoin +from urllib.request import urlcleanup + +from .network import download_lib +from .releases import ( + latest_libiconv_release, + latest_libxml2_release, + latest_libxslt_release, + latest_openssl_release, + latest_xmlsec_release, + latest_zlib_release, +) + + +@dataclass +class CrossCompileInfo: + host: str + arch: str + compiler: str + + @property + def triplet(self) -> str: + return f'{self.host}-{self.arch}-{self.compiler}' + + +class LibXmlsecDependencyBuilder: + WINDOWS_LIBS_DOWNLOAD_RELEASE_URL = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2025.07.10/' + LIB_VERSION_ENV_VARS: ClassVar[dict[str, str]] = { + 'libiconv_version': 'PYXMLSEC_LIBICONV_VERSION', + 'libxml2_version': 'PYXMLSEC_LIBXML2_VERSION', + 'libxslt_version': 'PYXMLSEC_LIBXSLT_VERSION', + 'openssl_version': 'PYXMLSEC_OPENSSL_VERSION', + 'xmlsec1_version': 'PYXMLSEC_XMLSEC1_VERSION', + 'zlib_version': 'PYXMLSEC_ZLIB_VERSION', + } + UNIX_DEFAULT_LIB_VERSIONS: ClassVar[dict[str, str]] = { + 'libiconv_version': '1.18', + 'libxml2_version': '2.14.6', # Make sure it matches with lxml + 'libxslt_version': '1.1.43', + 'openssl_version': '3.6.0', + 'xmlsec1_version': '1.3.9', + 'zlib_version': '1.3.1', + } + WINDOWS_DEFAULT_LIB_VERSIONS: ClassVar[dict[str, str]] = { + 'libiconv_version': '1.18-1', + 'libxml2_version': '2.11.9-3', # Make sure it matches with lxml + 'libxslt_version': '1.1.39', + 'openssl_version': '3.0.16.pl1', + 'xmlsec1_version': '1.3.7', + 'zlib_version': '1.3.1', + } + + def __init__(self, platform_name, info=None, libs_dir=None, buildroot=None, plat_name=None): + self.platform_name = platform_name + self.info = info or print + self.plat_name = plat_name + + self._prepare_directories(libs_dir=libs_dir, buildroot=buildroot) + self._set_library_versions(build_platform=platform_name) + + @property + def versions(self): + return {attr: getattr(self, attr) for attr in self.LIB_VERSION_ENV_VARS} + + def prepare(self, download_only=False): + self.info(f'preparing dependency build on {self.platform_name}') + if self.platform_name == 'win32': + self._prepare_windows_build(download_only=download_only) + elif 'linux' in self.platform_name or 'darwin' in self.platform_name: + self._prepare_unix_build(build_platform=self.platform_name, download_only=download_only) + else: + raise DistutilsError(f'Unsupported static build platform: {self.platform_name}') + + def _prepare_directories(self, libs_dir=None, buildroot=None): + buildroot_path = Path(buildroot) if buildroot else Path('build', 'tmp') + + prefix_dir = buildroot_path / 'prefix' + prefix_dir.mkdir(parents=True, exist_ok=True) + self.prefix_dir = prefix_dir.absolute() + + build_libs_dir = buildroot_path / 'libs' + build_libs_dir.mkdir(parents=True, exist_ok=True) + self.build_libs_dir = build_libs_dir + + libs_root = libs_dir if libs_dir is not None else os.environ.get('PYXMLSEC_LIBS_DIR', 'libs') + libs_dir_path = Path(libs_root) + libs_dir_path.mkdir(parents=True, exist_ok=True) + self.libs_dir = libs_dir_path + + self.info('{:20} {}'.format('Lib sources in:', self.libs_dir.absolute())) + + def _set_library_versions(self, build_platform): + defaults = self.UNIX_DEFAULT_LIB_VERSIONS + if build_platform == 'win32': + defaults = self.WINDOWS_DEFAULT_LIB_VERSIONS + + for version_attr, env_var in self.LIB_VERSION_ENV_VARS.items(): + setattr(self, version_attr, os.environ.get(env_var, defaults[version_attr])) + + def _prepare_windows_build(self, download_only=False): + if platform.machine() == 'ARM64': + suffix = 'win-arm64' + elif sys.maxsize > 2**32: + suffix = 'win64' + else: + suffix = 'win32' + + libs = [ + f'libxml2-{self.libxml2_version}.{suffix}.zip', + f'libxslt-{self.libxslt_version}.{suffix}.zip', + f'zlib-{self.zlib_version}.{suffix}.zip', + f'iconv-{self.libiconv_version}.{suffix}.zip', + f'openssl-{self.openssl_version}.{suffix}.zip', + f'xmlsec-{self.xmlsec1_version}.{suffix}.zip', + ] + + for libfile in libs: + url = urljoin(self.WINDOWS_LIBS_DOWNLOAD_RELEASE_URL, libfile) + destfile = self.libs_dir / libfile + if destfile.is_file(): + self.info(f'Using local copy of "{url}"') + else: + self.info(f'Retrieving "{url}" to "{destfile}"') + urlcleanup() + download_lib(url, str(destfile)) + + if download_only: + return + + for package in self.libs_dir.glob('*.zip'): + with zipfile.ZipFile(str(package)) as archive: + archive.extractall(path=str(self.build_libs_dir)) + + def _prepare_unix_build(self, build_platform, download_only=False): + archives = self._ensure_source_archives() + if download_only: + return + + self._extract_archives(archives) + + env, prefix_arg, ldflags, cross_compile = self._prepare_build_environment(build_platform) + self._build_dependencies(env, prefix_arg, ldflags, cross_compile) + + def _ensure_source_archives(self): + return [ + self._ensure_source( + name='OpenSSL', + glob='openssl*.tar.gz', + filename='openssl.tar.gz', + version=self.openssl_version, + env_label='PYXMLSEC_OPENSSL_VERSION', + default_url=latest_openssl_release, + version_url=lambda v: f'https://api.github.com/repos/openssl/openssl/tarball/openssl-{v}', + ), + self._ensure_source( + name='zlib', + glob='zlib*.tar.gz', + filename='zlib.tar.gz', + version=self.zlib_version, + env_label='PYXMLSEC_ZLIB_VERSION', + default_url=latest_zlib_release, + version_url=lambda v: f'https://zlib.net/fossils/zlib-{v}.tar.gz', + ), + self._ensure_source( + name='libiconv', + glob='libiconv*.tar.gz', + filename='libiconv.tar.gz', + version=self.libiconv_version, + env_label='PYXMLSEC_LIBICONV_VERSION', + default_url=latest_libiconv_release, + version_url=lambda v: f'https://ftpmirror.gnu.org/libiconv/libiconv-{v}.tar.gz', + ), + self._ensure_source( + name='libxml2', + glob='libxml2*.tar.xz', + filename='libxml2.tar.xz', + version=self.libxml2_version, + env_label='PYXMLSEC_LIBXML2_VERSION', + default_url=latest_libxml2_release, + version_url=lambda v: self._libxml_related_url('libxml2', v), + ), + self._ensure_source( + name='libxslt', + glob='libxslt*.tar.xz', + filename='libxslt.tar.xz', + version=self.libxslt_version, + env_label='PYXMLSEC_LIBXSLT_VERSION', + default_url=latest_libxslt_release, + version_url=lambda v: self._libxml_related_url('libxslt', v), + ), + self._ensure_source( + name='xmlsec1', + glob='xmlsec1*.tar.gz', + filename='xmlsec1.tar.gz', + version=self.xmlsec1_version, + env_label='PYXMLSEC_XMLSEC1_VERSION', + default_url=latest_xmlsec_release, + version_url=lambda v: f'https://github.com/lsh123/xmlsec/releases/download/{v}/xmlsec1-{v}.tar.gz', + ), + ] + + def _ensure_source(self, name, glob, filename, version, env_label, default_url, version_url): + archive = next(self.libs_dir.glob(glob), None) + if archive is not None: + return archive + + self.info('{:10}: {}'.format(name, 'source tar not found, downloading ...')) + archive = self.libs_dir / filename + if version is None: + url = default_url() + self.info('{:10}: {}'.format(name, f'{env_label} unset, downloading latest from {url}')) + else: + url = version_url(version) + self.info('{:10}: {}'.format(name, f'{env_label}={version}, downloading from {url}')) + download_lib(url, str(archive)) + return archive + + def _libxml_related_url(self, lib_name, version): + version_prefix, _ = version.rsplit('.', 1) + return f'https://download.gnome.org/sources/{lib_name}/{version_prefix}/{lib_name}-{version}.tar.xz' + + def _extract_archives(self, archives): + for archive in archives: + self.info(f'Unpacking {archive.name}') + try: + with tarfile.open(str(archive)) as tar: + if sys.version_info >= (3, 12): + tar.extractall(path=str(self.build_libs_dir), filter='data') + else: + tar.extractall(path=str(self.build_libs_dir)) + except EOFError as error: + raise DistutilsError(f'Bad {archive.name} downloaded; remove it and try again.') from error + + def _prepare_build_environment(self, build_platform): + prefix_arg = f'--prefix={self.prefix_dir}' + env = os.environ.copy() + + cflags = [] + if env.get('CFLAGS'): + cflags.append(env['CFLAGS']) + cflags.append('-fPIC') + + ldflags = [] + if env.get('LDFLAGS'): + ldflags.append(env['LDFLAGS']) + + cross_compile = None + if build_platform == 'darwin': + if self.plat_name: + arch = self.plat_name.rsplit('-', 1)[1] + if arch != platform.machine() and arch in ('x86_64', 'arm64'): + self.info(f'Cross-compiling for {arch}') + cflags.append(f'-arch {arch}') + ldflags.append(f'-arch {arch}') + cross_compile = CrossCompileInfo('darwin64', arch, 'cc') + major_version, _ = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) + if major_version >= 11 and 'MACOSX_DEPLOYMENT_TARGET' not in env: + env['MACOSX_DEPLOYMENT_TARGET'] = '11.0' + + env['CFLAGS'] = ' '.join(cflags) + env['LDFLAGS'] = ' '.join(ldflags) + return env, prefix_arg, ldflags, cross_compile + + def _build_dependencies(self, env, prefix_arg, ldflags, cross_compile): + self._build_openssl(env, prefix_arg, cross_compile) + self._build_zlib(env, prefix_arg) + + host_arg = [f'--host={cross_compile.arch}'] if cross_compile else [] + self._build_libiconv(env, prefix_arg, host_arg) + self._build_libxml2(env, prefix_arg, host_arg) + self._build_libxslt(env, prefix_arg, host_arg) + + ldflags.append('-lpthread') + env['LDFLAGS'] = ' '.join(ldflags) + self._build_xmlsec1(env, prefix_arg, host_arg) + + def _build_openssl(self, env, prefix_arg, cross_compile): + self.info('Building OpenSSL') + openssl_dir = next(self.build_libs_dir.glob('openssl-*')) + openssl_config_cmd = [prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'] + if platform.machine() == 'riscv64': + # openssl(riscv64): disable ASM to avoid R_RISCV_JAL relocation failure on 3.5.2 + # OpenSSL 3.5.2 enables RISC-V64 AES assembly by default. When we statically + # link libcrypto alongside xmlsec, the AES asm path triggers a link-time error: + # relocation truncated to fit: R_RISCV_JAL against symbol `AES_set_encrypt_key' + # in .../libcrypto.a(libcrypto-lib-aes-riscv64.o) + # This appears to stem from a long-range jump emitted by the AES asm generator + # (see aes-riscv64.pl around L1069), which can exceed the JAL reach when objects + # end up far apart in the final static link. + # As a pragmatic workaround, disable ASM on riscv64 (pass `no-asm`) so the + # portable C implementation is used. This unblocks the build at the cost of + # some crypto performance on riscv64 only. + # Refs: + # - https://github.com/openssl/openssl/blob/0893a62/crypto/aes/asm/aes-riscv64.pl#L1069 + openssl_config_cmd.append('no-asm') + if cross_compile: + openssl_config_cmd.insert(0, './Configure') + openssl_config_cmd.append(cross_compile.triplet) + else: + openssl_config_cmd.insert(0, './config') + subprocess.check_call(openssl_config_cmd, cwd=str(openssl_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(openssl_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install_sw'], cwd=str(openssl_dir), env=env) + + def _build_zlib(self, env, prefix_arg): + self.info('Building zlib') + zlib_dir = next(self.build_libs_dir.glob('zlib-*')) + subprocess.check_call(['./configure', prefix_arg], cwd=str(zlib_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(zlib_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(zlib_dir), env=env) + + def _build_libiconv(self, env, prefix_arg, host_arg): + self.info('Building libiconv') + libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) + subprocess.check_call( + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + *host_arg, + ], + cwd=str(libiconv_dir), + env=env, + ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libiconv_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libiconv_dir), env=env) + + def _build_libxml2(self, env, prefix_arg, host_arg): + self.info('Building LibXML2') + libxml2_dir = next(self.build_libs_dir.glob('libxml2-*')) + subprocess.check_call( + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + '--without-lzma', + '--without-python', + f'--with-iconv={self.prefix_dir}', + f'--with-zlib={self.prefix_dir}', + *host_arg, + ], + cwd=str(libxml2_dir), + env=env, + ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxml2_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxml2_dir), env=env) + + def _build_libxslt(self, env, prefix_arg, host_arg): + self.info('Building libxslt') + libxslt_dir = next(self.build_libs_dir.glob('libxslt-*')) + subprocess.check_call( + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + '--without-python', + '--without-crypto', + f'--with-libxml-prefix={self.prefix_dir}', + *host_arg, + ], + cwd=str(libxslt_dir), + env=env, + ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxslt_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxslt_dir), env=env) + + def _build_xmlsec1(self, env, prefix_arg, host_arg): + self.info('Building xmlsec1') + xmlsec1_dir = next(self.build_libs_dir.glob('xmlsec1-*')) + subprocess.check_call( + [ + './configure', + prefix_arg, + '--disable-shared', + '--disable-gost', + '--enable-md5', + '--enable-ripemd160', + '--disable-crypto-dl', + '--enable-static=yes', + '--enable-shared=no', + '--enable-static-linking=yes', + '--with-default-crypto=openssl', + f'--with-openssl={self.prefix_dir}', + f'--with-libxml={self.prefix_dir}', + f'--with-libxslt={self.prefix_dir}', + *host_arg, + ], + cwd=str(xmlsec1_dir), + env=env, + ) + include_flags = [ + f'-I{self.prefix_dir / "include"}', + f'-I{self.prefix_dir / "include" / "libxml"}', + ] + subprocess.check_call( + ['make', f'-j{multiprocessing.cpu_count() + 1}', *include_flags], + cwd=str(xmlsec1_dir), + env=env, + ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(xmlsec1_dir), env=env) + + +__all__ = ('CrossCompileInfo', 'LibXmlsecDependencyBuilder') diff --git a/build_support/static_build.py b/build_support/static_build.py index 09e2039a..34d06ae6 100644 --- a/build_support/static_build.py +++ b/build_support/static_build.py @@ -1,35 +1,7 @@ -import multiprocessing -import os -import platform -import subprocess import sys -import tarfile -import zipfile from distutils.errors import DistutilsError -from pathlib import Path -from urllib.parse import urljoin -from urllib.request import urlcleanup -from .network import download_lib -from .releases import ( - latest_libiconv_release, - latest_libxml2_release, - latest_libxslt_release, - latest_openssl_release, - latest_xmlsec_release, - latest_zlib_release, -) - - -class CrossCompileInfo: - def __init__(self, host, arch, compiler): - self.host = host - self.arch = arch - self.compiler = compiler - - @property - def triplet(self): - return f'{self.host}-{self.arch}-{self.compiler}' +from .lib_xmlsec_dependency_builder import CrossCompileInfo, LibXmlsecDependencyBuilder class StaticBuildHelper: @@ -37,70 +9,35 @@ def __init__(self, builder): self.builder = builder self.ext = builder.ext_map['xmlsec'] self.info = builder.info - self._prepare_directories() def prepare(self, platform_name): self.info(f'starting static build on {sys.platform}') - if platform_name == 'win32': - self._prepare_windows_build() - elif 'linux' in platform_name or 'darwin' in platform_name: - self._prepare_unix_build(platform_name) - else: - raise DistutilsError(f'Unsupported static build platform: {platform_name}') - - def _prepare_directories(self): - buildroot = Path('build', 'tmp') - - prefix_dir = buildroot / 'prefix' - prefix_dir.mkdir(parents=True, exist_ok=True) - self.prefix_dir = prefix_dir.absolute() - - build_libs_dir = buildroot / 'libs' - build_libs_dir.mkdir(exist_ok=True) - self.build_libs_dir = build_libs_dir + deps_builder = LibXmlsecDependencyBuilder( + platform_name=platform_name, + info=self.info, + plat_name=getattr(self.builder, 'plat_name', None), + ) + deps_builder.prepare() - libs_dir = Path(os.environ.get('PYXMLSEC_LIBS_DIR', 'libs')) - libs_dir.mkdir(exist_ok=True) - self.libs_dir = libs_dir - self.info('{:20} {}'.format('Lib sources in:', self.libs_dir.absolute())) + self.prefix_dir = deps_builder.prefix_dir + self.build_libs_dir = deps_builder.build_libs_dir + self.libs_dir = deps_builder.libs_dir self.builder.prefix_dir = self.prefix_dir self.builder.build_libs_dir = self.build_libs_dir self.builder.libs_dir = self.libs_dir - def _prepare_windows_build(self): - release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2025.07.10/' - if platform.machine() == 'ARM64': - suffix = 'win-arm64' - elif sys.maxsize > 2**32: - suffix = 'win64' - else: - suffix = 'win32' + for version_attr, value in deps_builder.versions.items(): + setattr(self.builder, version_attr, value) - libs = [ - f'libxml2-2.11.9-3.{suffix}.zip', - f'libxslt-1.1.39.{suffix}.zip', - f'zlib-1.3.1.{suffix}.zip', - f'iconv-1.18-1.{suffix}.zip', - f'openssl-3.0.16.pl1.{suffix}.zip', - f'xmlsec-1.3.7.{suffix}.zip', - ] - - for libfile in libs: - url = urljoin(release_url, libfile) - destfile = self.libs_dir / libfile - if destfile.is_file(): - self.info(f'Using local copy of "{url}"') - else: - self.info(f'Retrieving "{url}" to "{destfile}"') - urlcleanup() - download_lib(url, str(destfile)) - - for package in self.libs_dir.glob('*.zip'): - with zipfile.ZipFile(str(package)) as archive: - destdir = self.build_libs_dir - archive.extractall(path=str(destdir)) + if platform_name == 'win32': + self._configure_windows_extension_for_static() + elif 'linux' in platform_name or 'darwin' in platform_name: + self._configure_unix_extension_for_static(platform_name) + else: + raise DistutilsError(f'Unsupported static build platform: {platform_name}') + def _configure_windows_extension_for_static(self): self.ext.define_macros = [ ('XMLSEC_CRYPTO', '\\"openssl\\"'), ('__XMLSEC_FUNCTION__', '__FUNCTION__'), @@ -137,282 +74,7 @@ def _prepare_windows_build(self): includes.append(next(path / 'xmlsec' for path in includes if (path / 'xmlsec').is_dir())) self.ext.include_dirs = [str(path.absolute()) for path in includes] - def _prepare_unix_build(self, build_platform): - self._capture_version_overrides() - archives = self._ensure_source_archives() - self._extract_archives(archives) - - env, prefix_arg, ldflags, cross_compile = self._prepare_build_environment(build_platform) - self._build_dependencies(env, prefix_arg, ldflags, cross_compile) - self._configure_extension_for_static(build_platform) - - def _capture_version_overrides(self): - builder = self.builder - builder.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '3.6.0') - builder.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION', '1.18') - builder.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION', '2.14.6') - builder.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION', '1.1.43') - builder.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION', '1.3.1') - builder.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.3.9') - - def _ensure_source_archives(self): - return [ - self._ensure_source( - name='OpenSSL', - glob='openssl*.tar.gz', - filename='openssl.tar.gz', - version=self.builder.openssl_version, - env_label='PYXMLSEC_OPENSSL_VERSION', - default_url=latest_openssl_release, - version_url=lambda v: f'https://api.github.com/repos/openssl/openssl/tarball/openssl-{v}', - ), - self._ensure_source( - name='zlib', - glob='zlib*.tar.gz', - filename='zlib.tar.gz', - version=self.builder.zlib_version, - env_label='PYXMLSEC_ZLIB_VERSION', - default_url=latest_zlib_release, - version_url=lambda v: f'https://zlib.net/fossils/zlib-{v}.tar.gz', - ), - self._ensure_source( - name='libiconv', - glob='libiconv*.tar.gz', - filename='libiconv.tar.gz', - version=self.builder.libiconv_version, - env_label='PYXMLSEC_LIBICONV_VERSION', - default_url=latest_libiconv_release, - version_url=lambda v: f'https://ftpmirror.gnu.org/libiconv/libiconv-{v}.tar.gz', - ), - self._ensure_source( - name='libxml2', - glob='libxml2*.tar.xz', - filename='libxml2.tar.xz', - version=self.builder.libxml2_version, - env_label='PYXMLSEC_LIBXML2_VERSION', - default_url=latest_libxml2_release, - version_url=lambda v: self._libxml_related_url('libxml2', v), - ), - self._ensure_source( - name='libxslt', - glob='libxslt*.tar.xz', - filename='libxslt.tar.xz', - version=self.builder.libxslt_version, - env_label='PYXMLSEC_LIBXSLT_VERSION', - default_url=latest_libxslt_release, - version_url=lambda v: self._libxml_related_url('libxslt', v), - ), - self._ensure_source( - name='xmlsec1', - glob='xmlsec1*.tar.gz', - filename='xmlsec1.tar.gz', - version=self.builder.xmlsec1_version, - env_label='PYXMLSEC_XMLSEC1_VERSION', - default_url=latest_xmlsec_release, - version_url=lambda v: f'https://github.com/lsh123/xmlsec/releases/download/{v}/xmlsec1-{v}.tar.gz', - ), - ] - - def _ensure_source(self, name, glob, filename, version, env_label, default_url, version_url): - archive = next(self.libs_dir.glob(glob), None) - if archive is not None: - return archive - - self.info('{:10}: {}'.format(name, 'source tar not found, downloading ...')) - archive = self.libs_dir / filename - if version is None: - url = default_url() - self.info('{:10}: {}'.format(name, f'{env_label} unset, downloading latest from {url}')) - else: - url = version_url(version) - self.info('{:10}: {}'.format(name, f'{env_label}={version}, downloading from {url}')) - download_lib(url, str(archive)) - return archive - - def _libxml_related_url(self, lib_name, version): - version_prefix, _ = version.rsplit('.', 1) - return f'https://download.gnome.org/sources/{lib_name}/{version_prefix}/{lib_name}-{version}.tar.xz' - - def _extract_archives(self, archives): - for archive in archives: - self.info(f'Unpacking {archive.name}') - try: - with tarfile.open(str(archive)) as tar: - tar.extractall(path=str(self.build_libs_dir)) - except EOFError as error: - raise DistutilsError(f'Bad {archive.name} downloaded; remove it and try again.') from error - - def _prepare_build_environment(self, build_platform): - prefix_arg = f'--prefix={self.prefix_dir}' - env = os.environ.copy() - - cflags = [] - if env.get('CFLAGS'): - cflags.append(env['CFLAGS']) - cflags.append('-fPIC') - - ldflags = [] - if env.get('LDFLAGS'): - ldflags.append(env['LDFLAGS']) - - cross_compile = None - if build_platform == 'darwin': - arch = self.builder.plat_name.rsplit('-', 1)[1] - if arch != platform.machine() and arch in ('x86_64', 'arm64'): - self.info(f'Cross-compiling for {arch}') - cflags.append(f'-arch {arch}') - ldflags.append(f'-arch {arch}') - cross_compile = CrossCompileInfo('darwin64', arch, 'cc') - major_version, _ = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) - if major_version >= 11 and 'MACOSX_DEPLOYMENT_TARGET' not in env: - env['MACOSX_DEPLOYMENT_TARGET'] = '11.0' - - env['CFLAGS'] = ' '.join(cflags) - env['LDFLAGS'] = ' '.join(ldflags) - return env, prefix_arg, ldflags, cross_compile - - def _build_dependencies(self, env, prefix_arg, ldflags, cross_compile): - self._build_openssl(env, prefix_arg, cross_compile) - self._build_zlib(env, prefix_arg) - - host_arg = [f'--host={cross_compile.arch}'] if cross_compile else [] - self._build_libiconv(env, prefix_arg, host_arg) - self._build_libxml2(env, prefix_arg, host_arg) - self._build_libxslt(env, prefix_arg, host_arg) - - ldflags.append('-lpthread') - env['LDFLAGS'] = ' '.join(ldflags) - self._build_xmlsec1(env, prefix_arg, host_arg) - - def _build_openssl(self, env, prefix_arg, cross_compile): - self.info('Building OpenSSL') - openssl_dir = next(self.build_libs_dir.glob('openssl-*')) - openssl_config_cmd = [prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'] - if platform.machine() == 'riscv64': - # openssl(riscv64): disable ASM to avoid R_RISCV_JAL relocation failure on 3.5.2 - # OpenSSL 3.5.2 enables RISC-V64 AES assembly by default. When we statically - # link libcrypto alongside xmlsec, the AES asm path triggers a link-time error: - # relocation truncated to fit: R_RISCV_JAL against symbol `AES_set_encrypt_key' - # in .../libcrypto.a(libcrypto-lib-aes-riscv64.o) - # This appears to stem from a long-range jump emitted by the AES asm generator - # (see aes-riscv64.pl around L1069), which can exceed the JAL reach when objects - # end up far apart in the final static link. - # As a pragmatic workaround, disable ASM on riscv64 (pass `no-asm`) so the - # portable C implementation is used. This unblocks the build at the cost of - # some crypto performance on riscv64 only. - # Refs: - # - https://github.com/openssl/openssl/blob/0893a62/crypto/aes/asm/aes-riscv64.pl#L1069 - openssl_config_cmd.append('no-asm') - if cross_compile: - openssl_config_cmd.insert(0, './Configure') - openssl_config_cmd.append(cross_compile.triplet) - else: - openssl_config_cmd.insert(0, './config') - subprocess.check_call(openssl_config_cmd, cwd=str(openssl_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(openssl_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install_sw'], cwd=str(openssl_dir), env=env) - - def _build_zlib(self, env, prefix_arg): - self.info('Building zlib') - zlib_dir = next(self.build_libs_dir.glob('zlib-*')) - subprocess.check_call(['./configure', prefix_arg], cwd=str(zlib_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(zlib_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(zlib_dir), env=env) - - def _build_libiconv(self, env, prefix_arg, host_arg): - self.info('Building libiconv') - libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) - subprocess.check_call( - [ - './configure', - prefix_arg, - '--disable-dependency-tracking', - '--disable-shared', - *host_arg, - ], - cwd=str(libiconv_dir), - env=env, - ) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libiconv_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libiconv_dir), env=env) - - def _build_libxml2(self, env, prefix_arg, host_arg): - self.info('Building LibXML2') - libxml2_dir = next(self.build_libs_dir.glob('libxml2-*')) - subprocess.check_call( - [ - './configure', - prefix_arg, - '--disable-dependency-tracking', - '--disable-shared', - '--without-lzma', - '--without-python', - f'--with-iconv={self.prefix_dir}', - f'--with-zlib={self.prefix_dir}', - *host_arg, - ], - cwd=str(libxml2_dir), - env=env, - ) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxml2_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxml2_dir), env=env) - - def _build_libxslt(self, env, prefix_arg, host_arg): - self.info('Building libxslt') - libxslt_dir = next(self.build_libs_dir.glob('libxslt-*')) - subprocess.check_call( - [ - './configure', - prefix_arg, - '--disable-dependency-tracking', - '--disable-shared', - '--without-python', - '--without-crypto', - f'--with-libxml-prefix={self.prefix_dir}', - *host_arg, - ], - cwd=str(libxslt_dir), - env=env, - ) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxslt_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxslt_dir), env=env) - - def _build_xmlsec1(self, env, prefix_arg, host_arg): - self.info('Building xmlsec1') - xmlsec1_dir = next(self.build_libs_dir.glob('xmlsec1-*')) - subprocess.check_call( - [ - './configure', - prefix_arg, - '--disable-shared', - '--disable-gost', - '--enable-md5', - '--enable-ripemd160', - '--disable-crypto-dl', - '--enable-static=yes', - '--enable-shared=no', - '--enable-static-linking=yes', - '--with-default-crypto=openssl', - f'--with-openssl={self.prefix_dir}', - f'--with-libxml={self.prefix_dir}', - f'--with-libxslt={self.prefix_dir}', - *host_arg, - ], - cwd=str(xmlsec1_dir), - env=env, - ) - include_flags = [ - f'-I{self.prefix_dir / "include"}', - f'-I{self.prefix_dir / "include" / "libxml"}', - ] - subprocess.check_call( - ['make', f'-j{multiprocessing.cpu_count() + 1}', *include_flags], - cwd=str(xmlsec1_dir), - env=env, - ) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(xmlsec1_dir), env=env) - - def _configure_extension_for_static(self, build_platform): + def _configure_unix_extension_for_static(self, build_platform): self.ext.define_macros = [ ('__XMLSEC_FUNCTION__', '__func__'), ('XMLSEC_NO_SIZE_T', None), diff --git a/pyproject.toml b/pyproject.toml index 7c7b4bf3..89ace626 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,56 @@ [build-system] -requires = ["setuptools==80.9.0", "wheel", "setuptools_scm[toml]>=3.4", "pkgconfig>=1.5.1", "lxml==6.0.2"] +requires = ["setuptools", "wheel", "setuptools_scm>=8", "pkgconfig>=1.5.1", "lxml>=3.8"] +[project] +name = "xmlsec" +dynamic = ["version"] +description = "Python bindings for the XML Security Library" +readme = {file = "README.md", content-type = "text/markdown"} +requires-python = ">=3.9" +dependencies = ["lxml>=3.8"] +keywords = ["xmlsec"] +authors = [ + {name = "Bulat Gaifullin", email = "support@mehcode.com"} +] +maintainers = [ + {name = "Oleg Hoefling", email = "oleg.hoefling@gmail.com"}, + {name = "Amin Solhizadeh", email = "amin.solhizadeh@gmail.com"} +] +license = "MIT" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: OS Independent", + "Programming Language :: C", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Text Processing :: Markup :: XML", + "Typing :: Typed" +] + +[project.urls] +Documentation = "https://xmlsec.readthedocs.io" +Source = "https://github.com/xmlsec/python-xmlsec" +Changelog = "https://github.com/xmlsec/python-xmlsec/releases" + +# setuptools +[tool.setuptools] +zip-safe = false +packages = ["xmlsec"] +package-dir = {"" = "src"} + +[tool.setuptools.package-data] +xmlsec = ["py.typed", "*.pyi"] + +[tool.setuptools_scm] + +# mypy [tool.mypy] files = ['src'] ignore_missing_imports = false @@ -22,6 +72,19 @@ warn_no_return = true no_implicit_reexport = true show_error_codes = true +# TODO: Remove this override after adding full type annotations in build tooling modules. +[[tool.mypy.overrides]] +module = [ + "build_support.network", + "build_support.releases", + "build_support.lib_xmlsec_dependency_builder", + "build_libs_xmlsec" +] +disallow_untyped_calls = false +disallow_untyped_defs = false +disable_error_code = ["attr-defined"] + +# ruff [tool.ruff] # Maximum line length, same as your original Black + Flake8 config line-length = 130 @@ -88,6 +151,7 @@ skip-magic-trailing-comma = false # Enforce Unix-style line endings (LF) line-ending = "lf" +# cibuildwheel [tool.cibuildwheel] build = [ "cp39-*", @@ -98,6 +162,7 @@ build = [ "cp314-*" ] build-verbosity = 1 +environment = {PYXMLSEC_STATIC_DEPS="true"} build-frontend = "build" skip = [ "pp*", # Skips PyPy builds (pp38-*, pp39-*, etc.) @@ -105,23 +170,16 @@ skip = [ ] test-command = "pytest -v --color=yes {package}/tests" before-test = "pip install -r requirements-test.txt" -test-skip = "*-macosx_arm64" - -[tool.cibuildwheel.environment] -PYXMLSEC_STATIC_DEPS = "true" [tool.cibuildwheel.linux] archs = ["x86_64", "aarch64", "riscv64"] environment-pass = [ - "PYXMLSEC_LIBXML2_VERSION", - "PYXMLSEC_LIBXSLT_VERSION", "PYXMLSEC_STATIC_DEPS", "GH_TOKEN" ] [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] -before-all = "brew install perl" [tool.cibuildwheel.windows] archs = ["AMD64"] diff --git a/requirements-test.txt b/requirements-test.txt index ad135d97..70fe9703 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,4 +2,4 @@ pytest==8.4.1 lxml-stubs==0.5.1 -ruff[format]==0.13.0 +ruff[format]==0.14.4 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 8762c654..00000000 --- a/setup.cfg +++ /dev/null @@ -1,22 +0,0 @@ -[metadata] -description_file = README.md - -[bdist_rpm] -release = 1 -build_requires = pkg-config xmlsec1-devel libxml2-devel xmlsec1-openssl-devel -group = Development/Libraries -requires = xmlsec1 xmlsec1-openssl - -[build_sphinx] -source-dir = doc/source -build-dir = doc/build -all_files = 1 - -# [flake8] -# per-file-ignores = -# *.pyi: E301, E302, E305, E501, E701, F401, F822 -# doc/source/conf.py: D1 -# doc/source/examples/*.py: D1, E501 -# tests/*.py: D1 -# exclude = .venv*,.git,*_pb2.pyi,build,dist,libs,.eggs,.direnv* -# max-line-length = 130 diff --git a/setup.py b/setup.py index 4100a52b..946855f1 100644 --- a/setup.py +++ b/setup.py @@ -5,57 +5,11 @@ from build_support.build_ext import build_ext src_root = Path(__file__).parent / 'src' -sources = [str(path.absolute()) for path in src_root.rglob('*.c')] +sources = [str(path.relative_to(Path(__file__).parent)) for path in src_root.rglob('*.c')] pyxmlsec = Extension('xmlsec', sources=sources) -setup_reqs = ['setuptools_scm[toml]>=3.4', 'pkgconfig>=1.5.1', 'lxml>=3.8'] - - -with open('README.md', encoding='utf-8') as readme: - long_desc = readme.read() setup( - name='xmlsec', - use_scm_version=True, - description='Python bindings for the XML Security Library', - long_description=long_desc, - long_description_content_type='text/markdown', ext_modules=[pyxmlsec], cmdclass={'build_ext': build_ext}, - python_requires='>=3.9', - setup_requires=setup_reqs, - install_requires=['lxml>=3.8'], - author='Bulat Gaifullin', - author_email='support@mehcode.com', - maintainer='Oleg Hoefling', - maintainer_email='oleg.hoefling@gmail.com', - url='https://github.com/mehcode/python-xmlsec', - project_urls={ - 'Documentation': 'https://xmlsec.readthedocs.io', - 'Source': 'https://github.com/mehcode/python-xmlsec', - 'Changelog': 'https://github.com/mehcode/python-xmlsec/releases', - }, - license='MIT', - keywords=['xmlsec'], - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: C', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'Topic :: Text Processing :: Markup :: XML', - 'Typing :: Typed', - ], - zip_safe=False, - packages=['xmlsec'], - package_dir={'': 'src'}, - package_data={'xmlsec': ['py.typed', '*.pyi']}, )