incorporate changes from the stand-alone trexio-pypi-test repo

.github/workflows/build-wheels.yml

@@ -0,0 +1,195 @@
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push events but only for the master branch
+  push:
+    branches: [ master ]
+  release:
+    types:
+      - published
+# Workflow to build and publish wheels.
+#
+# in the get_commit_message job: Include [wheel build] in your commit message to trigger the build.
+name: PyPI wheels build
+
+jobs:
+
+  get_commit_message:
+    name: Get commit message
+    runs-on: ubuntu-latest
+    outputs:
+      message: ${{ steps.commit_message.outputs.message }}
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v2
+        # Gets the correct commit message for pull request
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Get commit message
+        id: commit_message
+        run: |
+          set -xe
+          COMMIT_MSG=$(git log --no-merges -1 --oneline)
+          echo "::set-output name=message::$COMMIT_MSG"
+
+
+  build_linux_wheels:
+    name: Build Linux wheels for different versions of CPython on manylinux_x86_64
+    needs: get_commit_message
+    if: >-
+      contains(needs.get_commit_message.outputs.message, '[wheel build]') ||
+      github.event_name == 'release'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        manylinux_tag: [2010_x86_64, 2014_x86_64, 2_24_x86_64]
+
+    steps:
+      - name: Checkout the branch
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.9'
+
+      - name: Install build dependencies
+        run: python -m pip install -U setuptools
+
+      - name: Run SWIG to procude the wrapper code for Python
+        run: |
+          swig -python -py3 -o pytrexio_wrap.c pytrexio.i
+          cp pytrexio.py ../pytrexio/
+        working-directory: src
+
+      - name: Build distribution tarball
+        run: |
+          python setup.py sdist
+          mv ./dist/trexio-*.tar.gz .
+
+      - name: Build manylinux wheels
+        run: |
+          docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest
+          docker run --rm --env PLAT=manylinux${{ matrix.manylinux_tag }} --volume `pwd`:/tmp --workdir /tmp ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} /bin/bash build_manylinux_wheels.sh trexio-*.tar.gz
+
+      - name: Upload produced wheels as artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: pytrexio-manylinux-${{ matrix.manylinux_tag }}
+          path: ./wheelhouse/*.whl
+
+      # only upload the source code tarball once
+      - name: Upload source code tarball as an artifact (only once)
+        uses: actions/upload-artifact@v2
+        with:
+          name: pytrexio-source
+          path: ./trexio-*.tar.gz
+        if: ${{ matrix.manylinux_tag == '2014_x86_64' }}
+
+
+  build_macos_wheels:
+    name: Build MacOS wheels for different versions of CPython
+    needs: get_commit_message
+    if: >-
+      contains(needs.get_commit_message.outputs.message, '[wheel build]') ||
+      github.event_name == 'release'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-11, macos-10.15]
+        python-version: ['3.7', '3.8', '3.9', '3.10']
+        #exclude:
+        #  - os: macos-10.15
+        #    python-version: '3.8'
+    env:
+        H5_LDFLAGS: '-L/usr/local/Cellar/hdf5/1.12.1/lib'
+        H5_CFLAGS: '-I/usr/local/Cellar/hdf5/1.12.1/include'
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Display Python version
+        run: python --version
+
+      - name: Install build dependencies
+        run: |
+          brew install hdf5@1.12
+          python -m pip install -U setuptools build delocate numpy
+
+      # this step is needed to produce wheels with the correct platform tag
+      - name: Set MACOSX_DEPLOYMENT_TARGET environment variable
+        if: ${{ matrix.os == 'macos-11' }}
+        # it is not possible to set ENV variables conditionally, so we improvise below
+        run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
+
+      - name: Run SWIG to procude the wrapper code for Python
+        run: |
+          swig -python -py3 -o pytrexio_wrap.c pytrexio.i
+          cp pytrexio.py ../pytrexio/
+        working-directory: src
+
+      - name: Build wheel for a given version of CPython
+        run: |
+          source tools/set_NUMPY_INCLUDEDIR.sh
+          python -m build --wheel --outdir ./
+          delocate-wheel trexio-*.whl
+
+      - name: Upload produced wheels as artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: pytrexio-${{ matrix.os }}
+          path: ./*.whl
+
+
+  publish_wheels:
+    name: Publish all wheels on PyPI
+    needs: [build_linux_wheels, build_macos_wheels]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout the branch
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.9'
+
+      - name: Install build dependencies
+        run: python -m pip install -U setuptools twine
+
+      - name: Download the build artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: dist
+     # if name is not specified - all artifacts will be downloaded
+     #    name: pytrexio-manylinux
+
+      # the artifacts have to be in dist/ directory so that
+      # pypa/gh-action-pypi-publish action can discover them
+      - name: Display and rearrange the downloaded artifacts
+        run: |
+          ls -R
+          mv pytrexio-manylinux-*/trexio-*.whl ./
+          mv pytrexio-macos-*/trexio-*.whl ./
+          mv pytrexio-source/trexio-*.tar.gz ./
+          rm -rf -- pytrexio-manylinux/ pytrexio-macos/ pytrexio-source/
+          ls -sh -w 1
+        working-directory: dist
+
+      #- name: Publish distribution 📦 to Test PyPI
+      #  uses: pypa/gh-action-pypi-publish@master
+      #  with:
+      #    password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+      #    repository_url: https://test.pypi.org/legacy/
+          #verbose: true
+
+      #- name: Publish distribution 📦 to PyPI
+      #  if: startsWith(github.ref, 'refs/tags')
+      #  uses: pypa/gh-action-pypi-publish@master
+      #  with:
+      #    password: ${{ secrets.PYPI_API_TOKEN }}

docker/build_manylinux_wheels.sh

@@ -6,6 +6,10 @@ set -e
 export H5_LDFLAGS=-L/usr/local/lib
 export H5_CFLAGS=-I/usr/local/include
 
+readonly ROOTDIR=${PWD}
+
+# create the wheelhouse directory
+mkdir -p ${ROOTDIR}/wheelhouse
 
 # build wheel directly from developer-provided .tar.gz of TREXIO (generated with `python setup.py sdist`)
 # note: trexio-VERSION.tar.gz has to be in the root directory of the host machine and provided as an argument to this script
@@ -26,7 +30,7 @@ TR_VERSION=${tmp%.tar.gz*}
 echo "TREXIO VERSION:" ${TR_VERSION}
 
 # unzip and enter the folder with TREXIO Python API
-gzip -cd /tmp/trexio-${TR_VERSION}.tar.gz | tar xvf -
+gzip -cd ${TREXIO_SOURCE} | tar xvf -
 cd trexio-${TR_VERSION}
 
 # the function below build manylinux wheels based on the provided version of python (e.g. build_wheel_for_py 36)
@@ -41,6 +45,15 @@ function build_wheel_for_py()
    # derive PYVERSION from the input argument
    PYVERSION=${1}
 
+   # derive manylinux glibc tag from the PLAT env variable provided to docker run
+   # this is needed to avoid building wheel for 2010_x86_64 with CPython 3.10
+   # because NumPy does not have wheels for it
+   MANYLINUX_TAG=${PLAT:9:4}
+   if [[ ${PYVERSION} -eq 310 ]] && [[ ${MANYLINUX_TAG} -eq 2010 ]]; then
+	echo "Skip build of the wheel for CPython 3.10 on manylinux2010_x86_64"
+	return
+   fi
+
    # python versions <= 3.7 required additional "m" in the platform tag, e.g. cp37-cp37m
    if [[ ${PYVERSION} -eq 36 ]] || [[ ${PYVERSION} -eq 37 ]]; then
        PYM="m"
@@ -93,6 +106,8 @@ function build_wheel_for_py()
    # remove the virtual environment
    rm -rf -- trexio-manylinux-py${PYVERSION}
 
+   # move the wheelhouse directory to the ROOTDIR
+   mv wheelhouse/trexio-${TR_VERSION}-${CPYTHON}-manylinux*.whl ${ROOTDIR}/wheelhouse/
 }
 
 

python/setup.py

@@ -4,9 +4,23 @@ setup.py file for TREXIO Python package
 """
 
 
-import os
+import os, sys
 from setuptools import setup, Extension
 
+def parse_setuppy_commands():
+    """Check the commands and respond appropriately.
+    At the moment it is adapted to ignore checks for numpy, plgconfig, HDF5 flags
+    when building the distribution tarball with sdist option.
+    """
+    args = sys.argv[1:]
+
+    if 'sdist' in args:
+        return True
+    else:
+        return False
+
+do_sdist = parse_setuppy_commands()
+
 # this was recommended to solve the problem of the missing numpy header files
 # bit it causes `pip install .` to fail with numpy module not found error
 #try:
@@ -23,7 +37,7 @@ from setuptools import setup, Extension
 numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None)
 numpy_isUndefined = numpy_includedir is None or numpy_includedir==""
 
-if numpy_isUndefined:
+if numpy_isUndefined and not do_sdist:
     raise Exception("NUMPY_INCLUDEDIR environment variable is not specified. Please do it manually or execute set_NUMPY_INCLUDEDIR.sh script.")
 
 
@@ -56,7 +70,7 @@ h5_cflags_withI  = os.environ.get("H5_CFLAGS", None)
 h5_ldflags_isUndefined = h5_ldflags_withl is None or h5_ldflags_withl==""
 h5_cflags_isUndefined = h5_cflags_withI is None or h5_cflags_withI==""
 
-if h5_ldflags_isUndefined or h5_cflags_isUndefined:
+if (h5_ldflags_isUndefined or h5_cflags_isUndefined) and not do_sdist:
 
     try:
         import pkgconfig as pk
@@ -71,8 +85,9 @@ if h5_ldflags_isUndefined or h5_cflags_isUndefined:
     h5_cflags_withI = pk.cflags('hdf5')
     h5_ldflags_withl = pk.libs('hdf5')
 
-h5_cflags = h5_cflags_withI.replace("-I","").split(" ")[0]
+
-h5_ldflags = h5_ldflags_withl.split(" ")[0]
+h5_cflags = h5_cflags_withI.replace("-I","").split(" ")[0] if not do_sdist else ""
+h5_ldflags = h5_ldflags_withl.split(" ")[0] if not do_sdist else ""
 
 # ============================ End of the HDF5 block ============================ #
 
@@ -115,4 +130,3 @@ setup(name             = 'trexio',
       python_requires = ">=3.6",
       install_requires = ['numpy>=1.17.3']
       )
-