From 1a5babd403feae3dd3612656f771f28edced6136 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 7 Jan 2022 10:25:09 +0100 Subject: [PATCH] incorporate changes from the stand-alone trexio-pypi-test repo --- .github/workflows/build-wheels.yml | 195 +++++++++++++++++++++++++++++ docker/build_manylinux_wheels.sh | 17 ++- python/setup.py | 26 +++- 3 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-wheels.yml diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml new file mode 100644 index 0000000..5b830ed --- /dev/null +++ b/.github/workflows/build-wheels.yml @@ -0,0 +1,195 @@ + +# Controls when the workflow will run +on: + # Triggers the workflow on push events but only for the master branch + push: + branches: [ master ] + release: + types: + - published +# Workflow to build and publish wheels. +# +# in the get_commit_message job: Include [wheel build] in your commit message to trigger the build. +name: PyPI wheels build + +jobs: + + get_commit_message: + name: Get commit message + runs-on: ubuntu-latest + outputs: + message: ${{ steps.commit_message.outputs.message }} + steps: + - name: Checkout the repo + uses: actions/checkout@v2 + # Gets the correct commit message for pull request + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Get commit message + id: commit_message + run: | + set -xe + COMMIT_MSG=$(git log --no-merges -1 --oneline) + echo "::set-output name=message::$COMMIT_MSG" + + + build_linux_wheels: + name: Build Linux wheels for different versions of CPython on manylinux_x86_64 + needs: get_commit_message + if: >- + contains(needs.get_commit_message.outputs.message, '[wheel build]') || + github.event_name == 'release' + runs-on: ubuntu-latest + strategy: + matrix: + manylinux_tag: [2010_x86_64, 2014_x86_64, 2_24_x86_64] + + steps: + - name: Checkout the branch + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.9' + + - name: Install build dependencies + run: python -m pip install -U setuptools + + - name: Run SWIG to procude the wrapper code for Python + run: | + swig -python -py3 -o pytrexio_wrap.c pytrexio.i + cp pytrexio.py ../pytrexio/ + working-directory: src + + - name: Build distribution tarball + run: | + python setup.py sdist + mv ./dist/trexio-*.tar.gz . + + - name: Build manylinux wheels + run: | + docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest + docker run --rm --env PLAT=manylinux${{ matrix.manylinux_tag }} --volume `pwd`:/tmp --workdir /tmp ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} /bin/bash build_manylinux_wheels.sh trexio-*.tar.gz + + - name: Upload produced wheels as artifacts + uses: actions/upload-artifact@v2 + with: + name: pytrexio-manylinux-${{ matrix.manylinux_tag }} + path: ./wheelhouse/*.whl + + # only upload the source code tarball once + - name: Upload source code tarball as an artifact (only once) + uses: actions/upload-artifact@v2 + with: + name: pytrexio-source + path: ./trexio-*.tar.gz + if: ${{ matrix.manylinux_tag == '2014_x86_64' }} + + + build_macos_wheels: + name: Build MacOS wheels for different versions of CPython + needs: get_commit_message + if: >- + contains(needs.get_commit_message.outputs.message, '[wheel build]') || + github.event_name == 'release' + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-11, macos-10.15] + python-version: ['3.7', '3.8', '3.9', '3.10'] + #exclude: + # - os: macos-10.15 + # python-version: '3.8' + env: + H5_LDFLAGS: '-L/usr/local/Cellar/hdf5/1.12.1/lib' + H5_CFLAGS: '-I/usr/local/Cellar/hdf5/1.12.1/include' + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Display Python version + run: python --version + + - name: Install build dependencies + run: | + brew install hdf5@1.12 + python -m pip install -U setuptools build delocate numpy + + # this step is needed to produce wheels with the correct platform tag + - name: Set MACOSX_DEPLOYMENT_TARGET environment variable + if: ${{ matrix.os == 'macos-11' }} + # it is not possible to set ENV variables conditionally, so we improvise below + run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV + + - name: Run SWIG to procude the wrapper code for Python + run: | + swig -python -py3 -o pytrexio_wrap.c pytrexio.i + cp pytrexio.py ../pytrexio/ + working-directory: src + + - name: Build wheel for a given version of CPython + run: | + source tools/set_NUMPY_INCLUDEDIR.sh + python -m build --wheel --outdir ./ + delocate-wheel trexio-*.whl + + - name: Upload produced wheels as artifacts + uses: actions/upload-artifact@v2 + with: + name: pytrexio-${{ matrix.os }} + path: ./*.whl + + + publish_wheels: + name: Publish all wheels on PyPI + needs: [build_linux_wheels, build_macos_wheels] + runs-on: ubuntu-latest + + steps: + - name: Checkout the branch + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.9' + + - name: Install build dependencies + run: python -m pip install -U setuptools twine + + - name: Download the build artifacts + uses: actions/download-artifact@v2 + with: + path: dist + # if name is not specified - all artifacts will be downloaded + # name: pytrexio-manylinux + + # the artifacts have to be in dist/ directory so that + # pypa/gh-action-pypi-publish action can discover them + - name: Display and rearrange the downloaded artifacts + run: | + ls -R + mv pytrexio-manylinux-*/trexio-*.whl ./ + mv pytrexio-macos-*/trexio-*.whl ./ + mv pytrexio-source/trexio-*.tar.gz ./ + rm -rf -- pytrexio-manylinux/ pytrexio-macos/ pytrexio-source/ + ls -sh -w 1 + working-directory: dist + + #- name: Publish distribution 📦 to Test PyPI + # uses: pypa/gh-action-pypi-publish@master + # with: + # password: ${{ secrets.TEST_PYPI_API_TOKEN }} + # repository_url: https://test.pypi.org/legacy/ + #verbose: true + + #- name: Publish distribution 📦 to PyPI + # if: startsWith(github.ref, 'refs/tags') + # uses: pypa/gh-action-pypi-publish@master + # with: + # password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/docker/build_manylinux_wheels.sh b/docker/build_manylinux_wheels.sh index 6a51193..fa9fa6a 100755 --- a/docker/build_manylinux_wheels.sh +++ b/docker/build_manylinux_wheels.sh @@ -6,6 +6,10 @@ set -e export H5_LDFLAGS=-L/usr/local/lib export H5_CFLAGS=-I/usr/local/include +readonly ROOTDIR=${PWD} + +# create the wheelhouse directory +mkdir -p ${ROOTDIR}/wheelhouse # build wheel directly from developer-provided .tar.gz of TREXIO (generated with `python setup.py sdist`) # note: trexio-VERSION.tar.gz has to be in the root directory of the host machine and provided as an argument to this script @@ -26,7 +30,7 @@ TR_VERSION=${tmp%.tar.gz*} echo "TREXIO VERSION:" ${TR_VERSION} # unzip and enter the folder with TREXIO Python API -gzip -cd /tmp/trexio-${TR_VERSION}.tar.gz | tar xvf - +gzip -cd ${TREXIO_SOURCE} | tar xvf - cd trexio-${TR_VERSION} # the function below build manylinux wheels based on the provided version of python (e.g. build_wheel_for_py 36) @@ -41,6 +45,15 @@ function build_wheel_for_py() # derive PYVERSION from the input argument PYVERSION=${1} + # derive manylinux glibc tag from the PLAT env variable provided to docker run + # this is needed to avoid building wheel for 2010_x86_64 with CPython 3.10 + # because NumPy does not have wheels for it + MANYLINUX_TAG=${PLAT:9:4} + if [[ ${PYVERSION} -eq 310 ]] && [[ ${MANYLINUX_TAG} -eq 2010 ]]; then + echo "Skip build of the wheel for CPython 3.10 on manylinux2010_x86_64" + return + fi + # python versions <= 3.7 required additional "m" in the platform tag, e.g. cp37-cp37m if [[ ${PYVERSION} -eq 36 ]] || [[ ${PYVERSION} -eq 37 ]]; then PYM="m" @@ -93,6 +106,8 @@ function build_wheel_for_py() # remove the virtual environment rm -rf -- trexio-manylinux-py${PYVERSION} + # move the wheelhouse directory to the ROOTDIR + mv wheelhouse/trexio-${TR_VERSION}-${CPYTHON}-manylinux*.whl ${ROOTDIR}/wheelhouse/ } diff --git a/python/setup.py b/python/setup.py index 5ead098..c7b7a65 100644 --- a/python/setup.py +++ b/python/setup.py @@ -4,9 +4,23 @@ setup.py file for TREXIO Python package """ -import os +import os, sys from setuptools import setup, Extension +def parse_setuppy_commands(): + """Check the commands and respond appropriately. + At the moment it is adapted to ignore checks for numpy, plgconfig, HDF5 flags + when building the distribution tarball with sdist option. + """ + args = sys.argv[1:] + + if 'sdist' in args: + return True + else: + return False + +do_sdist = parse_setuppy_commands() + # this was recommended to solve the problem of the missing numpy header files # bit it causes `pip install .` to fail with numpy module not found error #try: @@ -23,7 +37,7 @@ from setuptools import setup, Extension numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None) numpy_isUndefined = numpy_includedir is None or numpy_includedir=="" -if numpy_isUndefined: +if numpy_isUndefined and not do_sdist: raise Exception("NUMPY_INCLUDEDIR environment variable is not specified. Please do it manually or execute set_NUMPY_INCLUDEDIR.sh script.") @@ -56,7 +70,7 @@ h5_cflags_withI = os.environ.get("H5_CFLAGS", None) h5_ldflags_isUndefined = h5_ldflags_withl is None or h5_ldflags_withl=="" h5_cflags_isUndefined = h5_cflags_withI is None or h5_cflags_withI=="" -if h5_ldflags_isUndefined or h5_cflags_isUndefined: +if (h5_ldflags_isUndefined or h5_cflags_isUndefined) and not do_sdist: try: import pkgconfig as pk @@ -71,8 +85,9 @@ if h5_ldflags_isUndefined or h5_cflags_isUndefined: h5_cflags_withI = pk.cflags('hdf5') h5_ldflags_withl = pk.libs('hdf5') -h5_cflags = h5_cflags_withI.replace("-I","").split(" ")[0] -h5_ldflags = h5_ldflags_withl.split(" ")[0] + +h5_cflags = h5_cflags_withI.replace("-I","").split(" ")[0] if not do_sdist else "" +h5_ldflags = h5_ldflags_withl.split(" ")[0] if not do_sdist else "" # ============================ End of the HDF5 block ============================ # @@ -115,4 +130,3 @@ setup(name = 'trexio', python_requires = ">=3.6", install_requires = ['numpy>=1.17.3'] ) -