Automate build and upload of CPython wheels (#72)

* use PyPA build package to produce wheels

* update the Makefile and install_pytrexio to use PyPA build package

* remove MacOS-11 from runners

* [MacOS] portable expression for FreeBSD sed

* disable usage of NUMPY_INCLUDEDIR env variable

* activate PyPI upload and disable TestPyPI

.github/workflows/actions.yml

@@ -34,6 +34,7 @@ jobs:
 
     runs-on: ubuntu-latest
     name: x86 Ubuntu latest
+    needs: get_commit_message
 
     steps:
       - uses: actions/checkout@v2
@@ -61,16 +62,16 @@ jobs:
 
       - name: install Python API
         run: make python-install
-        # alternatively we can also run pip install trexio to check PyPI installation
 
       - name: check Python API
         run: make python-test
 
-      - name: build Python API distribution
+      - name: build and move Python API distribution
-        run: make python-sdist
+        run: |
+          make python-sdist
+          cp python/dist/trexio-*.tar.gz .
 
       - name: publish Python API distribution as an artifact
-        needs: get_commit_message
         if: >-
           contains(needs.get_commit_message.outputs.message, '[wheel build]') ||
           github.event_name == 'release'
@@ -78,7 +79,6 @@ jobs:
         with:
           name: pytrexio-source
           path: ./trexio-*.tar.gz
-        working-directory: python/dist
 
       - name: clean
         run: make clean

.github/workflows/build-wheels.yml

@@ -1,7 +1,7 @@
 
 # Controls when the workflow will run
 on:
-  # run this workflow after the TREXIO CI completed
+  # Run this workflow after the TREXIO CI completed
   workflow_run:
     workflows: [ "TREXIO CI" ]
     branches: [ master ]
@@ -9,10 +9,8 @@ on:
       - completed
 
 # Workflow to build and publish wheels.
-#
+# in the get_commit_message job: Include [wheel build] in your commit message to trigger this build.
-# in the get_commit_message job: Include [wheel build] in your commit message to trigger the build.
+name: Build CPython wheels
-name: PyPI wheels build
-
 jobs:
 
   get_commit_message:
@@ -58,25 +56,43 @@ jobs:
       - name: Install build dependencies
         run: python -m pip install -U setuptools
         
+      - name: Compute the PYTREXIO_VERSION environment variable
+        run: echo "PYTREXIO_VERSION=$(grep __version__ python/pytrexio/_version.py | cut -d\  -f3 | tr -d '"')" >> $GITHUB_ENV
+          
+      - name: Print the PYTREXIO_VERSION
+        run: echo ${{ env.PYTREXIO_VERSION }}
+
+      # Conventional download-artifact action does not work for artifact produced in a different workflow,
+      # which is the case here (TREXIO CI produced the Python API distribution tarball)
       - name: Download the Python API distribution tarball
-        uses: actions/download-artifact@v2
+        uses: dawidd6/action-download-artifact@v2
         with:
+          # Specify the name of the workflow file which uploaded the tarball
+          workflow: actions.yml
+          workflow_conclusion: success
           name: pytrexio-source
           path: python
       
-      - name: Build manylinux wheels
+      # at the moment we have to pull the custom container with pre-installed HDF5
-        run: |
+      # the containers are built and stored in GitHub container registry ghcr.io/q-posev
-          docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest
+      - name: Pull the manylinux Docker container with HDF5
-          docker run --rm --env PLAT=manylinux${{ matrix.manylinux_tag }} --volume `pwd`:/tmp --workdir /tmp ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} /bin/bash build_manylinux_wheels.sh trexio-*.tar.gz
+        run: docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest
+
+      - name: Build wheels for different versions of CPython inside the Docker container
+        run: >
+          docker run --rm 
+          --env PLAT=manylinux${{ matrix.manylinux_tag }} 
+          --volume `pwd`:/tmp 
+          --workdir /tmp 
+          ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} 
+          /bin/bash build_manylinux_wheels.sh trexio-${{ env.PYTREXIO_VERSION }}.tar.gz
         working-directory: python
 
       - name: Upload produced wheels as artifacts
         uses: actions/upload-artifact@v2
         with:
           name: pytrexio-manylinux-${{ matrix.manylinux_tag }}
-          path: ./wheelhouse/*.whl
+          path: ./python/wheelhouse/*.whl
-        working-directory: python
-
 
   build_macos_wheels:
     name: Build MacOS wheels for different versions of CPython
@@ -87,11 +103,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-11, macos-10.15]
+        os: [macos-10.15]
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
+# TODO: normally, one could include macos-11 and the OS list above but the produced wheels receive an error upon installation:
+# ERROR: trexio-1.1.0-cp39-cp39-macosx_11_0_x86_64.whl is not a supported wheel on this platform.
+# This happens even with the MACOSX_DEPLOYMENT_TARGET trick. Perhaps it can be solved by configuring the build system
+# to produce the wheels for MacOS-11 from the very beginning
         #exclude:
-        #  - os: macos-10.15
+        #  - os: macos-11
-        #    python-version: '3.8'
     env:
         H5_LDFLAGS: '-L/usr/local/Cellar/hdf5/1.12.1/lib'
         H5_CFLAGS: '-I/usr/local/Cellar/hdf5/1.12.1/include'
@@ -106,36 +125,58 @@ jobs:
       - name: Display Python version
         run: python --version
 
-      - name: Install build dependencies
+      - name: Install HDF5
-        run: |
+        run: brew install hdf5@1.12
-          brew install hdf5@1.12
-          python -m pip install -U setuptools build delocate numpy
 
-      # this step is needed to produce wheels with the correct platform tag
+      # This step is needed to produce wheels with the correct platform tag for MacOS-11 (Big Sur)
-      - name: Set MACOSX_DEPLOYMENT_TARGET environment variable
+      #- name: Set MACOSX_DEPLOYMENT_TARGET environment variable
-        if: ${{ matrix.os == 'macos-11' }}
+      #  if: ${{ matrix.os == 'macos-11' }}
-        # it is not possible to set ENV variables conditionally, so we improvise below
+      #  run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
-        run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
+        
+      - name: Compute the PYTREXIO_VERSION environment variable
+        run: echo "PYTREXIO_VERSION=$(grep __version__ python/pytrexio/_version.py | cut -d\  -f3 | tr -d '"')" >> $GITHUB_ENV
+          
+      - name: Print the PYTREXIO_VERSION
+        run: echo ${{ env.PYTREXIO_VERSION }}
 
       - name: Download the Python API distribution tarball
-        uses: actions/download-artifact@v2
+        uses: dawidd6/action-download-artifact@v2
         with:
+          workflow: actions.yml
+          workflow_conclusion: success
           name: pytrexio-source
           path: python
           
+      - name: Extract the Python distribution
+        run: gzip -cd trexio-${{ env.PYTREXIO_VERSION }}.tar.gz | tar xvf -
+        working-directory: python
+        
+      - name: Install Python dependencies
+        run: pip install --upgrade pip setuptools build delocate
+       
       - name: Build wheel for a given version of CPython
         run: |
-          source tools/set_NUMPY_INCLUDEDIR.sh
+          mkdir wheelhouse/
-          python -m build --wheel --outdir ./
+          cd trexio-${{ env.PYTREXIO_VERSION }}/
+          python -m build --wheel --outdir=./
           delocate-wheel trexio-*.whl
+          mv trexio-*.whl ../wheelhouse/
         working-directory: python
         
+      # Some issues with Python 3.10 wheels on MacOS-11
+      - name: Install the wheel
+        run: python -m pip install wheelhouse/trexio-*.whl
+        working-directory: python
+        
+      - name: Test the wheel
+        run: python test_api.py
+        working-directory: python/test
+        
       - name: Upload produced wheels as artifacts
         uses: actions/upload-artifact@v2
         with:
           name: pytrexio-${{ matrix.os }}
-          path: ./*.whl
+          path: ./python/wheelhouse/*.whl
-        working-directory: python
 
 
   publish_wheels:
@@ -155,34 +196,40 @@ jobs:
       - name: Install build dependencies
         run: python -m pip install -U setuptools twine
 
-      - name: Download the build artifacts
+      - name: Download the build artifacts (wheels) of this workflow
         uses: actions/download-artifact@v2
         with:
           path: dist
-     # if name is not specified - all artifacts will be downloaded
-     #    name: pytrexio-manylinux
 
-      # the artifacts have to be in dist/ directory so that
+      - name: Download the Python API distribution tarball
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: actions.yml
+          workflow_conclusion: success
+          name: pytrexio-source
+          path: dist
+
+      # The artifacts have to be in dist/ directory so that
       # pypa/gh-action-pypi-publish action can discover them
       - name: Display and rearrange the downloaded artifacts
         run: |
           ls -R
           mv pytrexio-manylinux-*/trexio-*.whl ./
           mv pytrexio-macos-*/trexio-*.whl ./
-          mv pytrexio-source/trexio-*.tar.gz ./
+          rm -rf -- pytrexio-manylinux-*/ pytrexio-macos-*/
-          rm -rf -- pytrexio-manylinux/ pytrexio-macos/ pytrexio-source/
           ls -sh -w 1
         working-directory: dist
 
       #- name: Publish distribution 📦 to Test PyPI
       #  uses: pypa/gh-action-pypi-publish@master
-      #  with:
+      # with:
       #    password: ${{ secrets.TEST_PYPI_API_TOKEN }}
       #    repository_url: https://test.pypi.org/legacy/
           #verbose: true
       
-      #- name: Publish distribution 📦 to PyPI
+      # Only upload to PyPI if the commit was tagged !
-      #  if: startsWith(github.ref, 'refs/tags')
+      - name: Publish distribution 📦 to PyPI
-      #  uses: pypa/gh-action-pypi-publish@master
+        if: startsWith(github.ref, 'refs/tags')
-      #  with:
+        uses: pypa/gh-action-pypi-publish@master
-      #    password: ${{ secrets.PYPI_API_TOKEN }}
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

Makefile.am

@@ -223,7 +223,7 @@ python-install: $(pytrexio_py) $(setup_py) $(setup_cfg)
 
 python-sdist: $(pytrexio_py) $(setup_py) $(setup_cfg)
 	cd python && \
-		python setup.py sdist
+		python3 -m build --sdist
 
 $(pytrexio_py): $(pytrexio_c)
 	cd tools && ./prepare_python.sh

python/README.md

@@ -22,7 +22,7 @@ In short, you can run the following command:
 
 However, it is good practice to first check for updates of the build-system packages. This can be achieved by running
 
-`python3 -m pip install --upgrade pip setuptools wheel`
+`python -m pip install --upgrade pip setuptools build wheel`
 
 **Note: we highly recommend to use virtual environments to avoid compatibility issues and to improve reproducibility.** 
 For more details, see the corresponding part of the [Python documentation](https://docs.python.org/3/library/venv.html#creating-virtual-environments).
@@ -30,24 +30,23 @@ For more details, see the corresponding part of the [Python documentation](https
 
 ## Additional requirements (for installation from source)
 
-- C compiler 	(gcc/icc)
+- C compiler 	(gcc/icc/clang)
 - HDF5 library	(>= 1.8)
 - pkgconfig     (Python package)
-
+- build         (Python package)
 
 ## Installation from source
 
 1. Download the `trexio-<version>.tar.gz` file with the latest Python API
 2. `gzip -cd trexio-<version>.tar.gz | tar xvf -`
 3. `cd trexio-<version>`
-4. `pip3 install -r requirements.txt` (this installs all required python dependencies)
+4. `pip install -r requirements.txt` (this installs all required python dependencies)
 5. Export custom environment variables needed for the installation following the procedure below and replacing `/path/to/hdf5/` with your paths. 
-Steps (i) and (ii) can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options).
+The following two steps can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options).
    1. `export H5_CFLAGS=-I/path/to/hdf5/include`
    2. `export H5_LDFLAGS=-L/path/to/hdf5/lib`
-   3. `source tools/set_NUMPY_INCLUDEDIR.sh`
+6. `pip install .` (this installs `trexio` in your environment)
-6. `pip3 install .` (this installs `trexio` in your environment)
+7. `cd test && python test_api.py` (this executes several tests that verify the installation)
-7. `cd test && python3 test_api.py` (this executes several tests that verify the installation)
 
 You are ready to go!
 

python/build_manylinux_wheels.sh

@@ -71,22 +71,10 @@ function build_wheel_for_py()
    # upgrade pip, otherwise it complains that manylinux wheel is "...not supported wheel on this platform"
    pip install --upgrade pip
    # install dependencies needed to build manylinux wheel
-   pip install --upgrade setuptools wheel auditwheel
+   pip install --upgrade setuptools build
-   if [ ${PYVERSION} -eq 36 ] || [ ${PYVERSION} -eq 37 ]; then
-       pip install numpy==1.17.3
-   elif [ ${PYVERSION} -eq 38 ]; then
-       pip install numpy==1.18.3
-   elif [ ${PYVERSION} -eq 39 ]; then
-       pip install numpy==1.19.3
-   else
-       pip install numpy==1.21.4
-   fi
-
-   # set an environment variable needed to locate numpy header files
-   source tools/set_NUMPY_INCLUDEDIR.sh
 
    # produce conventional (non-manylinux) wheel
-   python3 setup.py bdist_wheel
+   python3 -m build --wheel --outdir dist/
 
    # use auditwheel from PyPA to repair all wheels and make them manylinux-compatible
    auditwheel repair dist/trexio-${TR_VERSION}-${CPYTHON}-*.whl

python/install_pytrexio.sh

@@ -26,33 +26,42 @@ else
 fi
 
 # Install/upgrade packages required for the installation
-python3 -m pip install --upgrade setuptools wheel pip
+python3 -m pip install --upgrade setuptools build pip
 python3 -m pip install -r requirements.txt
 
 # export NUMPY_INCLUDEDIR environment variable needed for the proper setup
-source tools/set_NUMPY_INCLUDEDIR.sh
+#source tools/set_NUMPY_INCLUDEDIR.sh
-
+#if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then
-if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then
+#    echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)."
-    echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)."
+#    exit 1
-    exit 1
+#fi
-fi
 
 # Create build directory and compile extension files (*.c)
 # --no-user-cfg disables custom .cfg files of the user machine, so that only setup.cfg is used
-python3 -s setup.py --no-user-cfg build 
+#python3 -s setup.py --no-user-cfg build 
 
 # Local inplace build of the .so module with SWIG-produced pytrexio_wrap.c (from the SWIG documentation)
 #python3 setup.py build_ext --inplace --swig-opts="-modern"
 
 # Create distributions: 
+
+# OLD WAY (DEPRECATED BY PYPA)
 #   1) sdist produces .tar.gz with all files necessary for manual compilation;
 #   2) bdist_whell produces .whl wheel distribution (see https://www.python.org/dev/peps/pep-0425/).
-python3 setup.py sdist bdist_wheel
+#python3 setup.py sdist bdist_wheel
+
+# NEW WAY (USING BUILD PACKAGE OF PYPA)
+python3 -m build --sdist --wheel --outdir dist/
 
 # Install pytrexio in the current environment from the aforementioned wheel
+
+# OLD WAY
 # --force-reinstall is needed here because build-system pre-installs pytrexio in the environment 
 #                   but does not install things in the corresponding site-packages directory
-python3 -m pip install dist/trexio-*.whl --force-reinstall
+#python3 -m pip install dist/trexio-*.whl --force-reinstall
+
+# NEW WAY
+python3 -m pip install dist/trexio-*.whl
 
 # Run the command below in the root directory to install the package in 'editable' (-e) mode without dependencies (--no-deps)
 #python -m pip install -e . --no-deps

python/setup.py

@@ -22,19 +22,16 @@ def parse_setuppy_commands():
 do_sdist = parse_setuppy_commands()
 
 # this was recommended to solve the problem of the missing numpy header files
-# bit it causes `pip install .` to fail with numpy module not found error
+try:
-#try:
+    import numpy
-#    import numpy
+except ImportError:
-#except ImportError:
+    raise Exception("numpy Python package cannot be imported.")
-#    raise Exception("numpy Python package cannot be imported.")
-#numpy_includedir = numpy.get_include()
 
-# this does not cause aforementioned issue but the includedir points to system-wide numpy and not to venv-wide
+numpy_includedir = numpy.get_include()
-#from distutils.sysconfig import get_python_inc
-#numpy_includedir = os.path.join(get_python_inc(plat_specific=1), 'numpy')
 
 # dirty workaround: get numpy includedir from the environment variable that can be pre-set using set_NUMPY_INCLUDEDIR.sh
-numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None)
+#numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None)
+
 numpy_isUndefined = numpy_includedir is None or numpy_includedir==""
 
 if numpy_isUndefined and not do_sdist:

tools/build_doc.sh

@@ -123,10 +123,6 @@ function main() {
     then
         cd ${DOCS}
         rm -f index.html
-	# enter Google verificator into the HTML head
-        local GOOGLE_VERIF="<meta name="google-site-verification" content="jdDnuP2rYGJVy8AHSd-8LkmOmvK_dyz5buZ98wilYII" />"
-        LINE_NO=$(($(awk '/meta name="viewport"/{print NR}' README.html) + 1))
-        sed -i "$LINE_NO i ${GOOGLE_VERIF}" README.html
         
 	ln README.html index.html
         exit 0

tools/prepare_python.sh

@@ -31,8 +31,10 @@ cp ${INCLUDIR}/trexio.h ${PYDIR}/src
 
 # fix needed to define HAVE_HDF5 symbol so that Python extension is always compiled with HDF5 (without including config.h)
 # add "#define HAVE_HDF5 1" line after "#include stdint.h" using awk and sed
-LINE_NO=$(($(awk '/stdint.h/{print NR}' ${PYDIR}/src/trexio.h) + 1))
+export LINE_NO=$(($(awk '/stdint.h/{print NR}' ${PYDIR}/src/trexio.h) + 1))
-sed -i "$LINE_NO i #define HAVE_HDF5 1" ${PYDIR}/src/trexio.h 
+sed -i'' -e "$LINE_NO"'i \
+#define HAVE_HDF5 1
+' -- ${PYDIR}/src/trexio.h 
 
 # Copy additional info
 cp ${TREXIO_ROOT}/AUTHORS ${TREXIO_ROOT}/LICENSE ${PYDIR}