Automate build and upload of CPython wheels (#72)

* use PyPA build package to produce wheels * update the Makefile and install_pytrexio to use PyPA build package * remove MacOS-11 from runners * [MacOS] portable expression for FreeBSD sed * disable usage of NUMPY_INCLUDEDIR env variable * activate PyPI upload and disable TestPyPI
2024-07-22 10:47:43 +02:00 · 2022-01-07 18:47:23 +01:00 · 2022-01-07 18:47:23 +01:00 · dcb976010f
commit dcb976010f
parent 1aaca05b51
9 changed files with 139 additions and 101 deletions
--- a/.github/workflows/actions.yml
+++ b/.github/workflows/actions.yml
@ -34,6 +34,7 @@ jobs:

    runs-on: ubuntu-latest
    name: x86 Ubuntu latest
+    needs: get_commit_message

    steps:
      - uses: actions/checkout@v2
@ -61,16 +62,16 @@ jobs:

      - name: install Python API
        run: make python-install
-        # alternatively we can also run pip install trexio to check PyPI installation

      - name: check Python API
        run: make python-test

-      - name: build Python API distribution
-        run: make python-sdist
+      - name: build and move Python API distribution
+        run: |
+          make python-sdist
+          cp python/dist/trexio-*.tar.gz .

      - name: publish Python API distribution as an artifact
-        needs: get_commit_message
        if: >-
          contains(needs.get_commit_message.outputs.message, '[wheel build]') ||
          github.event_name == 'release'
@ -78,7 +79,6 @@ jobs:
        with:
          name: pytrexio-source
          path: ./trexio-*.tar.gz
-        working-directory: python/dist

      - name: clean
        run: make clean
--- a/.github/workflows/build-wheels.yml
+++ b/.github/workflows/build-wheels.yml
@ -1,7 +1,7 @@

 # Controls when the workflow will run
 on:
-  # run this workflow after the TREXIO CI completed
+  # Run this workflow after the TREXIO CI completed
  workflow_run:
    workflows: [ "TREXIO CI" ]
    branches: [ master ]
@ -9,10 +9,8 @@ on:
      - completed

 # Workflow to build and publish wheels.
-#
-# in the get_commit_message job: Include [wheel build] in your commit message to trigger the build.
-name: PyPI wheels build
-
+# in the get_commit_message job: Include [wheel build] in your commit message to trigger this build.
+name: Build CPython wheels
 jobs:

  get_commit_message:
@ -45,7 +43,7 @@ jobs:
    strategy:
      matrix:
        manylinux_tag: [2010_x86_64, 2014_x86_64, 2_24_x86_64]
-
+      
    steps:
      - name: Checkout the branch
        uses: actions/checkout@v2
@ -57,26 +55,44 @@ jobs:

      - name: Install build dependencies
        run: python -m pip install -U setuptools
+        
+      - name: Compute the PYTREXIO_VERSION environment variable
+        run: echo "PYTREXIO_VERSION=$(grep __version__ python/pytrexio/_version.py | cut -d\  -f3 | tr -d '"')" >> $GITHUB_ENV
+          
+      - name: Print the PYTREXIO_VERSION
+        run: echo ${{ env.PYTREXIO_VERSION }}

+      # Conventional download-artifact action does not work for artifact produced in a different workflow,
+      # which is the case here (TREXIO CI produced the Python API distribution tarball)
      - name: Download the Python API distribution tarball
-        uses: actions/download-artifact@v2
+        uses: dawidd6/action-download-artifact@v2
        with:
+          # Specify the name of the workflow file which uploaded the tarball
+          workflow: actions.yml
+          workflow_conclusion: success
          name: pytrexio-source
          path: python
+      
+      # at the moment we have to pull the custom container with pre-installed HDF5
+      # the containers are built and stored in GitHub container registry ghcr.io/q-posev
+      - name: Pull the manylinux Docker container with HDF5
+        run: docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest

-      - name: Build manylinux wheels
-        run: |
-          docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest
-          docker run --rm --env PLAT=manylinux${{ matrix.manylinux_tag }} --volume `pwd`:/tmp --workdir /tmp ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} /bin/bash build_manylinux_wheels.sh trexio-*.tar.gz
+      - name: Build wheels for different versions of CPython inside the Docker container
+        run: >
+          docker run --rm 
+          --env PLAT=manylinux${{ matrix.manylinux_tag }} 
+          --volume `pwd`:/tmp 
+          --workdir /tmp 
+          ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} 
+          /bin/bash build_manylinux_wheels.sh trexio-${{ env.PYTREXIO_VERSION }}.tar.gz
        working-directory: python

      - name: Upload produced wheels as artifacts
        uses: actions/upload-artifact@v2
        with:
          name: pytrexio-manylinux-${{ matrix.manylinux_tag }}
-          path: ./wheelhouse/*.whl
-        working-directory: python
-
+          path: ./python/wheelhouse/*.whl

  build_macos_wheels:
    name: Build MacOS wheels for different versions of CPython
@ -87,11 +103,14 @@ jobs:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
-        os: [macos-11, macos-10.15]
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        os: [macos-10.15]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
+# TODO: normally, one could include macos-11 and the OS list above but the produced wheels receive an error upon installation:
+# ERROR: trexio-1.1.0-cp39-cp39-macosx_11_0_x86_64.whl is not a supported wheel on this platform.
+# This happens even with the MACOSX_DEPLOYMENT_TARGET trick. Perhaps it can be solved by configuring the build system
+# to produce the wheels for MacOS-11 from the very beginning
        #exclude:
-        #  - os: macos-10.15
-        #    python-version: '3.8'
+        #  - os: macos-11
    env:
        H5_LDFLAGS: '-L/usr/local/Cellar/hdf5/1.12.1/lib'
        H5_CFLAGS: '-I/usr/local/Cellar/hdf5/1.12.1/include'
@ -106,36 +125,58 @@ jobs:
      - name: Display Python version
        run: python --version

-      - name: Install build dependencies
-        run: |
-          brew install hdf5@1.12
-          python -m pip install -U setuptools build delocate numpy
+      - name: Install HDF5
+        run: brew install hdf5@1.12

-      # this step is needed to produce wheels with the correct platform tag
-      - name: Set MACOSX_DEPLOYMENT_TARGET environment variable
-        if: ${{ matrix.os == 'macos-11' }}
-        # it is not possible to set ENV variables conditionally, so we improvise below
-        run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
+      # This step is needed to produce wheels with the correct platform tag for MacOS-11 (Big Sur)
+      #- name: Set MACOSX_DEPLOYMENT_TARGET environment variable
+      #  if: ${{ matrix.os == 'macos-11' }}
+      #  run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
+        
+      - name: Compute the PYTREXIO_VERSION environment variable
+        run: echo "PYTREXIO_VERSION=$(grep __version__ python/pytrexio/_version.py | cut -d\  -f3 | tr -d '"')" >> $GITHUB_ENV
+          
+      - name: Print the PYTREXIO_VERSION
+        run: echo ${{ env.PYTREXIO_VERSION }}

      - name: Download the Python API distribution tarball
-        uses: actions/download-artifact@v2
+        uses: dawidd6/action-download-artifact@v2
        with:
+          workflow: actions.yml
+          workflow_conclusion: success
          name: pytrexio-source
          path: python
-
+          
+      - name: Extract the Python distribution
+        run: gzip -cd trexio-${{ env.PYTREXIO_VERSION }}.tar.gz | tar xvf -
+        working-directory: python
+        
+      - name: Install Python dependencies
+        run: pip install --upgrade pip setuptools build delocate
+       
      - name: Build wheel for a given version of CPython
        run: |
-          source tools/set_NUMPY_INCLUDEDIR.sh
-          python -m build --wheel --outdir ./
+          mkdir wheelhouse/
+          cd trexio-${{ env.PYTREXIO_VERSION }}/
+          python -m build --wheel --outdir=./
          delocate-wheel trexio-*.whl
+          mv trexio-*.whl ../wheelhouse/
        working-directory: python
-
+        
+      # Some issues with Python 3.10 wheels on MacOS-11
+      - name: Install the wheel
+        run: python -m pip install wheelhouse/trexio-*.whl
+        working-directory: python
+        
+      - name: Test the wheel
+        run: python test_api.py
+        working-directory: python/test
+        
      - name: Upload produced wheels as artifacts
        uses: actions/upload-artifact@v2
        with:
          name: pytrexio-${{ matrix.os }}
-          path: ./*.whl
-        working-directory: python
+          path: ./python/wheelhouse/*.whl


  publish_wheels:
@ -155,34 +196,40 @@ jobs:
      - name: Install build dependencies
        run: python -m pip install -U setuptools twine

-      - name: Download the build artifacts
+      - name: Download the build artifacts (wheels) of this workflow
        uses: actions/download-artifact@v2
        with:
          path: dist
-     # if name is not specified - all artifacts will be downloaded
-     #    name: pytrexio-manylinux

-      # the artifacts have to be in dist/ directory so that
+      - name: Download the Python API distribution tarball
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: actions.yml
+          workflow_conclusion: success
+          name: pytrexio-source
+          path: dist
+
+      # The artifacts have to be in dist/ directory so that
      # pypa/gh-action-pypi-publish action can discover them
      - name: Display and rearrange the downloaded artifacts
        run: |
          ls -R
          mv pytrexio-manylinux-*/trexio-*.whl ./
          mv pytrexio-macos-*/trexio-*.whl ./
-          mv pytrexio-source/trexio-*.tar.gz ./
-          rm -rf -- pytrexio-manylinux/ pytrexio-macos/ pytrexio-source/
+          rm -rf -- pytrexio-manylinux-*/ pytrexio-macos-*/
          ls -sh -w 1
        working-directory: dist

      #- name: Publish distribution 📦 to Test PyPI
      #  uses: pypa/gh-action-pypi-publish@master
-      #  with:
+      # with:
      #    password: ${{ secrets.TEST_PYPI_API_TOKEN }}
      #    repository_url: https://test.pypi.org/legacy/
          #verbose: true
-
-      #- name: Publish distribution 📦 to PyPI
-      #  if: startsWith(github.ref, 'refs/tags')
-      #  uses: pypa/gh-action-pypi-publish@master
-      #  with:
-      #    password: ${{ secrets.PYPI_API_TOKEN }}
+      
+      # Only upload to PyPI if the commit was tagged !
+      - name: Publish distribution 📦 to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
--- a/Makefile.am
+++ b/Makefile.am
@ -223,7 +223,7 @@ python-install: $(pytrexio_py) $(setup_py) $(setup_cfg)

 python-sdist: $(pytrexio_py) $(setup_py) $(setup_cfg)
 	cd python && \
-		python setup.py sdist
+		python3 -m build --sdist

 $(pytrexio_py): $(pytrexio_c)
 	cd tools && ./prepare_python.sh
--- a/python/README.md
+++ b/python/README.md
@ -22,7 +22,7 @@ In short, you can run the following command:

 However, it is good practice to first check for updates of the build-system packages. This can be achieved by running

-`python3 -m pip install --upgrade pip setuptools wheel`
+`python -m pip install --upgrade pip setuptools build wheel`

 **Note: we highly recommend to use virtual environments to avoid compatibility issues and to improve reproducibility.** 
 For more details, see the corresponding part of the [Python documentation](https://docs.python.org/3/library/venv.html#creating-virtual-environments).
@ -30,24 +30,23 @@ For more details, see the corresponding part of the [Python documentation](https

 ## Additional requirements (for installation from source)

- C compiler 	(gcc/icc)
+- C compiler 	(gcc/icc/clang)
 - HDF5 library	(>= 1.8)
 - pkgconfig     (Python package)
-
+- build         (Python package)

 ## Installation from source

 1. Download the `trexio-<version>.tar.gz` file with the latest Python API
 2. `gzip -cd trexio-<version>.tar.gz | tar xvf -`
 3. `cd trexio-<version>`
-4. `pip3 install -r requirements.txt` (this installs all required python dependencies)
+4. `pip install -r requirements.txt` (this installs all required python dependencies)
 5. Export custom environment variables needed for the installation following the procedure below and replacing `/path/to/hdf5/` with your paths. 
-Steps (i) and (ii) can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options).
+The following two steps can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options).
   1. `export H5_CFLAGS=-I/path/to/hdf5/include`
   2. `export H5_LDFLAGS=-L/path/to/hdf5/lib`
-   3. `source tools/set_NUMPY_INCLUDEDIR.sh`
-6. `pip3 install .` (this installs `trexio` in your environment)
-7. `cd test && python3 test_api.py` (this executes several tests that verify the installation)
+6. `pip install .` (this installs `trexio` in your environment)
+7. `cd test && python test_api.py` (this executes several tests that verify the installation)

 You are ready to go!

--- a/python/build_manylinux_wheels.sh
+++ b/python/build_manylinux_wheels.sh
@ -71,22 +71,10 @@ function build_wheel_for_py()
   # upgrade pip, otherwise it complains that manylinux wheel is "...not supported wheel on this platform"
   pip install --upgrade pip
   # install dependencies needed to build manylinux wheel
-   pip install --upgrade setuptools wheel auditwheel
-   if [ ${PYVERSION} -eq 36 ] || [ ${PYVERSION} -eq 37 ]; then
-       pip install numpy==1.17.3
-   elif [ ${PYVERSION} -eq 38 ]; then
-       pip install numpy==1.18.3
-   elif [ ${PYVERSION} -eq 39 ]; then
-       pip install numpy==1.19.3
-   else
-       pip install numpy==1.21.4
-   fi
-
-   # set an environment variable needed to locate numpy header files
-   source tools/set_NUMPY_INCLUDEDIR.sh
+   pip install --upgrade setuptools build

   # produce conventional (non-manylinux) wheel
-   python3 setup.py bdist_wheel
+   python3 -m build --wheel --outdir dist/

   # use auditwheel from PyPA to repair all wheels and make them manylinux-compatible
   auditwheel repair dist/trexio-${TR_VERSION}-${CPYTHON}-*.whl
--- a/python/install_pytrexio.sh
+++ b/python/install_pytrexio.sh
@ -26,33 +26,42 @@ else
 fi

 # Install/upgrade packages required for the installation
-python3 -m pip install --upgrade setuptools wheel pip
+python3 -m pip install --upgrade setuptools build pip
 python3 -m pip install -r requirements.txt

 # export NUMPY_INCLUDEDIR environment variable needed for the proper setup
-source tools/set_NUMPY_INCLUDEDIR.sh
-
-if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then
-    echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)."
-    exit 1
-fi
+#source tools/set_NUMPY_INCLUDEDIR.sh
+#if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then
+#    echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)."
+#    exit 1
+#fi

 # Create build directory and compile extension files (*.c)
 # --no-user-cfg disables custom .cfg files of the user machine, so that only setup.cfg is used
-python3 -s setup.py --no-user-cfg build 
+#python3 -s setup.py --no-user-cfg build 

 # Local inplace build of the .so module with SWIG-produced pytrexio_wrap.c (from the SWIG documentation)
 #python3 setup.py build_ext --inplace --swig-opts="-modern"

 # Create distributions: 
+
+# OLD WAY (DEPRECATED BY PYPA)
 #   1) sdist produces .tar.gz with all files necessary for manual compilation;
 #   2) bdist_whell produces .whl wheel distribution (see https://www.python.org/dev/peps/pep-0425/).
-python3 setup.py sdist bdist_wheel
+#python3 setup.py sdist bdist_wheel
+
+# NEW WAY (USING BUILD PACKAGE OF PYPA)
+python3 -m build --sdist --wheel --outdir dist/

 # Install pytrexio in the current environment from the aforementioned wheel
+
+# OLD WAY
 # --force-reinstall is needed here because build-system pre-installs pytrexio in the environment 
 #                   but does not install things in the corresponding site-packages directory
-python3 -m pip install dist/trexio-*.whl --force-reinstall
+#python3 -m pip install dist/trexio-*.whl --force-reinstall
+
+# NEW WAY
+python3 -m pip install dist/trexio-*.whl

 # Run the command below in the root directory to install the package in 'editable' (-e) mode without dependencies (--no-deps)
 #python -m pip install -e . --no-deps
--- a/python/setup.py
+++ b/python/setup.py
@ -22,19 +22,16 @@ def parse_setuppy_commands():
 do_sdist = parse_setuppy_commands()

 # this was recommended to solve the problem of the missing numpy header files
-# bit it causes `pip install .` to fail with numpy module not found error
-#try:
-#    import numpy
-#except ImportError:
-#    raise Exception("numpy Python package cannot be imported.")
-#numpy_includedir = numpy.get_include()
+try:
+    import numpy
+except ImportError:
+    raise Exception("numpy Python package cannot be imported.")

-# this does not cause aforementioned issue but the includedir points to system-wide numpy and not to venv-wide
-#from distutils.sysconfig import get_python_inc
-#numpy_includedir = os.path.join(get_python_inc(plat_specific=1), 'numpy')
+numpy_includedir = numpy.get_include()

 # dirty workaround: get numpy includedir from the environment variable that can be pre-set using set_NUMPY_INCLUDEDIR.sh
-numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None)
+#numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None)
+
 numpy_isUndefined = numpy_includedir is None or numpy_includedir==""

 if numpy_isUndefined and not do_sdist:
--- a/tools/build_doc.sh
+++ b/tools/build_doc.sh
@ -123,10 +123,6 @@ function main() {
    then
        cd ${DOCS}
        rm -f index.html
-	# enter Google verificator into the HTML head
-        local GOOGLE_VERIF="<meta name="google-site-verification" content="jdDnuP2rYGJVy8AHSd-8LkmOmvK_dyz5buZ98wilYII" />"
-        LINE_NO=$(($(awk '/meta name="viewport"/{print NR}' README.html) + 1))
-        sed -i "$LINE_NO i ${GOOGLE_VERIF}" README.html
        
 	ln README.html index.html
        exit 0
--- a/tools/prepare_python.sh
+++ b/tools/prepare_python.sh
@ -31,8 +31,10 @@ cp ${INCLUDIR}/trexio.h ${PYDIR}/src

 # fix needed to define HAVE_HDF5 symbol so that Python extension is always compiled with HDF5 (without including config.h)
 # add "#define HAVE_HDF5 1" line after "#include stdint.h" using awk and sed
-LINE_NO=$(($(awk '/stdint.h/{print NR}' ${PYDIR}/src/trexio.h) + 1))
-sed -i "$LINE_NO i #define HAVE_HDF5 1" ${PYDIR}/src/trexio.h 
+export LINE_NO=$(($(awk '/stdint.h/{print NR}' ${PYDIR}/src/trexio.h) + 1))
+sed -i'' -e "$LINE_NO"'i \
+#define HAVE_HDF5 1
+' -- ${PYDIR}/src/trexio.h 

 # Copy additional info
 cp ${TREXIO_ROOT}/AUTHORS ${TREXIO_ROOT}/LICENSE ${PYDIR}