diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 32a1343..f0570fb 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -34,6 +34,7 @@ jobs: runs-on: ubuntu-latest name: x86 Ubuntu latest + needs: get_commit_message steps: - uses: actions/checkout@v2 @@ -61,16 +62,16 @@ jobs: - name: install Python API run: make python-install - # alternatively we can also run pip install trexio to check PyPI installation - name: check Python API run: make python-test - - name: build Python API distribution - run: make python-sdist + - name: build and move Python API distribution + run: | + make python-sdist + cp python/dist/trexio-*.tar.gz . - name: publish Python API distribution as an artifact - needs: get_commit_message if: >- contains(needs.get_commit_message.outputs.message, '[wheel build]') || github.event_name == 'release' @@ -78,7 +79,6 @@ jobs: with: name: pytrexio-source path: ./trexio-*.tar.gz - working-directory: python/dist - name: clean run: make clean diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 4313042..2590dcd 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -1,7 +1,7 @@ # Controls when the workflow will run on: - # run this workflow after the TREXIO CI completed + # Run this workflow after the TREXIO CI completed workflow_run: workflows: [ "TREXIO CI" ] branches: [ master ] @@ -9,10 +9,8 @@ on: - completed # Workflow to build and publish wheels. -# -# in the get_commit_message job: Include [wheel build] in your commit message to trigger the build. -name: PyPI wheels build - +# in the get_commit_message job: Include [wheel build] in your commit message to trigger this build. +name: Build CPython wheels jobs: get_commit_message: @@ -45,7 +43,7 @@ jobs: strategy: matrix: manylinux_tag: [2010_x86_64, 2014_x86_64, 2_24_x86_64] - + steps: - name: Checkout the branch uses: actions/checkout@v2 @@ -57,26 +55,44 @@ jobs: - name: Install build dependencies run: python -m pip install -U setuptools + + - name: Compute the PYTREXIO_VERSION environment variable + run: echo "PYTREXIO_VERSION=$(grep __version__ python/pytrexio/_version.py | cut -d\ -f3 | tr -d '"')" >> $GITHUB_ENV + + - name: Print the PYTREXIO_VERSION + run: echo ${{ env.PYTREXIO_VERSION }} + # Conventional download-artifact action does not work for artifact produced in a different workflow, + # which is the case here (TREXIO CI produced the Python API distribution tarball) - name: Download the Python API distribution tarball - uses: actions/download-artifact@v2 + uses: dawidd6/action-download-artifact@v2 with: + # Specify the name of the workflow file which uploaded the tarball + workflow: actions.yml + workflow_conclusion: success name: pytrexio-source path: python + + # at the moment we have to pull the custom container with pre-installed HDF5 + # the containers are built and stored in GitHub container registry ghcr.io/q-posev + - name: Pull the manylinux Docker container with HDF5 + run: docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest - - name: Build manylinux wheels - run: | - docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest - docker run --rm --env PLAT=manylinux${{ matrix.manylinux_tag }} --volume `pwd`:/tmp --workdir /tmp ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} /bin/bash build_manylinux_wheels.sh trexio-*.tar.gz + - name: Build wheels for different versions of CPython inside the Docker container + run: > + docker run --rm + --env PLAT=manylinux${{ matrix.manylinux_tag }} + --volume `pwd`:/tmp + --workdir /tmp + ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} + /bin/bash build_manylinux_wheels.sh trexio-${{ env.PYTREXIO_VERSION }}.tar.gz working-directory: python - name: Upload produced wheels as artifacts uses: actions/upload-artifact@v2 with: name: pytrexio-manylinux-${{ matrix.manylinux_tag }} - path: ./wheelhouse/*.whl - working-directory: python - + path: ./python/wheelhouse/*.whl build_macos_wheels: name: Build MacOS wheels for different versions of CPython @@ -87,11 +103,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-11, macos-10.15] - python-version: ['3.7', '3.8', '3.9', '3.10'] + os: [macos-10.15] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] +# TODO: normally, one could include macos-11 and the OS list above but the produced wheels receive an error upon installation: +# ERROR: trexio-1.1.0-cp39-cp39-macosx_11_0_x86_64.whl is not a supported wheel on this platform. +# This happens even with the MACOSX_DEPLOYMENT_TARGET trick. Perhaps it can be solved by configuring the build system +# to produce the wheels for MacOS-11 from the very beginning #exclude: - # - os: macos-10.15 - # python-version: '3.8' + # - os: macos-11 env: H5_LDFLAGS: '-L/usr/local/Cellar/hdf5/1.12.1/lib' H5_CFLAGS: '-I/usr/local/Cellar/hdf5/1.12.1/include' @@ -106,36 +125,58 @@ jobs: - name: Display Python version run: python --version - - name: Install build dependencies - run: | - brew install hdf5@1.12 - python -m pip install -U setuptools build delocate numpy + - name: Install HDF5 + run: brew install hdf5@1.12 - # this step is needed to produce wheels with the correct platform tag - - name: Set MACOSX_DEPLOYMENT_TARGET environment variable - if: ${{ matrix.os == 'macos-11' }} - # it is not possible to set ENV variables conditionally, so we improvise below - run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV + # This step is needed to produce wheels with the correct platform tag for MacOS-11 (Big Sur) + #- name: Set MACOSX_DEPLOYMENT_TARGET environment variable + # if: ${{ matrix.os == 'macos-11' }} + # run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV + + - name: Compute the PYTREXIO_VERSION environment variable + run: echo "PYTREXIO_VERSION=$(grep __version__ python/pytrexio/_version.py | cut -d\ -f3 | tr -d '"')" >> $GITHUB_ENV + + - name: Print the PYTREXIO_VERSION + run: echo ${{ env.PYTREXIO_VERSION }} - name: Download the Python API distribution tarball - uses: actions/download-artifact@v2 + uses: dawidd6/action-download-artifact@v2 with: + workflow: actions.yml + workflow_conclusion: success name: pytrexio-source path: python - + + - name: Extract the Python distribution + run: gzip -cd trexio-${{ env.PYTREXIO_VERSION }}.tar.gz | tar xvf - + working-directory: python + + - name: Install Python dependencies + run: pip install --upgrade pip setuptools build delocate + - name: Build wheel for a given version of CPython run: | - source tools/set_NUMPY_INCLUDEDIR.sh - python -m build --wheel --outdir ./ + mkdir wheelhouse/ + cd trexio-${{ env.PYTREXIO_VERSION }}/ + python -m build --wheel --outdir=./ delocate-wheel trexio-*.whl + mv trexio-*.whl ../wheelhouse/ working-directory: python - + + # Some issues with Python 3.10 wheels on MacOS-11 + - name: Install the wheel + run: python -m pip install wheelhouse/trexio-*.whl + working-directory: python + + - name: Test the wheel + run: python test_api.py + working-directory: python/test + - name: Upload produced wheels as artifacts uses: actions/upload-artifact@v2 with: name: pytrexio-${{ matrix.os }} - path: ./*.whl - working-directory: python + path: ./python/wheelhouse/*.whl publish_wheels: @@ -155,34 +196,40 @@ jobs: - name: Install build dependencies run: python -m pip install -U setuptools twine - - name: Download the build artifacts + - name: Download the build artifacts (wheels) of this workflow uses: actions/download-artifact@v2 with: path: dist - # if name is not specified - all artifacts will be downloaded - # name: pytrexio-manylinux - # the artifacts have to be in dist/ directory so that + - name: Download the Python API distribution tarball + uses: dawidd6/action-download-artifact@v2 + with: + workflow: actions.yml + workflow_conclusion: success + name: pytrexio-source + path: dist + + # The artifacts have to be in dist/ directory so that # pypa/gh-action-pypi-publish action can discover them - name: Display and rearrange the downloaded artifacts run: | ls -R mv pytrexio-manylinux-*/trexio-*.whl ./ mv pytrexio-macos-*/trexio-*.whl ./ - mv pytrexio-source/trexio-*.tar.gz ./ - rm -rf -- pytrexio-manylinux/ pytrexio-macos/ pytrexio-source/ + rm -rf -- pytrexio-manylinux-*/ pytrexio-macos-*/ ls -sh -w 1 working-directory: dist #- name: Publish distribution 📦 to Test PyPI # uses: pypa/gh-action-pypi-publish@master - # with: + # with: # password: ${{ secrets.TEST_PYPI_API_TOKEN }} # repository_url: https://test.pypi.org/legacy/ #verbose: true - - #- name: Publish distribution 📦 to PyPI - # if: startsWith(github.ref, 'refs/tags') - # uses: pypa/gh-action-pypi-publish@master - # with: - # password: ${{ secrets.PYPI_API_TOKEN }} + + # Only upload to PyPI if the commit was tagged ! + - name: Publish distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/Makefile.am b/Makefile.am index baac474..ec9354f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -223,7 +223,7 @@ python-install: $(pytrexio_py) $(setup_py) $(setup_cfg) python-sdist: $(pytrexio_py) $(setup_py) $(setup_cfg) cd python && \ - python setup.py sdist + python3 -m build --sdist $(pytrexio_py): $(pytrexio_c) cd tools && ./prepare_python.sh diff --git a/python/README.md b/python/README.md index ccbd846..2859484 100644 --- a/python/README.md +++ b/python/README.md @@ -22,7 +22,7 @@ In short, you can run the following command: However, it is good practice to first check for updates of the build-system packages. This can be achieved by running -`python3 -m pip install --upgrade pip setuptools wheel` +`python -m pip install --upgrade pip setuptools build wheel` **Note: we highly recommend to use virtual environments to avoid compatibility issues and to improve reproducibility.** For more details, see the corresponding part of the [Python documentation](https://docs.python.org/3/library/venv.html#creating-virtual-environments). @@ -30,24 +30,23 @@ For more details, see the corresponding part of the [Python documentation](https ## Additional requirements (for installation from source) -- C compiler (gcc/icc) +- C compiler (gcc/icc/clang) - HDF5 library (>= 1.8) - pkgconfig (Python package) - +- build (Python package) ## Installation from source 1. Download the `trexio-.tar.gz` file with the latest Python API 2. `gzip -cd trexio-.tar.gz | tar xvf -` 3. `cd trexio-` -4. `pip3 install -r requirements.txt` (this installs all required python dependencies) +4. `pip install -r requirements.txt` (this installs all required python dependencies) 5. Export custom environment variables needed for the installation following the procedure below and replacing `/path/to/hdf5/` with your paths. -Steps (i) and (ii) can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options). +The following two steps can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options). 1. `export H5_CFLAGS=-I/path/to/hdf5/include` 2. `export H5_LDFLAGS=-L/path/to/hdf5/lib` - 3. `source tools/set_NUMPY_INCLUDEDIR.sh` -6. `pip3 install .` (this installs `trexio` in your environment) -7. `cd test && python3 test_api.py` (this executes several tests that verify the installation) +6. `pip install .` (this installs `trexio` in your environment) +7. `cd test && python test_api.py` (this executes several tests that verify the installation) You are ready to go! diff --git a/python/build_manylinux_wheels.sh b/python/build_manylinux_wheels.sh index fa9fa6a..85705e0 100755 --- a/python/build_manylinux_wheels.sh +++ b/python/build_manylinux_wheels.sh @@ -71,22 +71,10 @@ function build_wheel_for_py() # upgrade pip, otherwise it complains that manylinux wheel is "...not supported wheel on this platform" pip install --upgrade pip # install dependencies needed to build manylinux wheel - pip install --upgrade setuptools wheel auditwheel - if [ ${PYVERSION} -eq 36 ] || [ ${PYVERSION} -eq 37 ]; then - pip install numpy==1.17.3 - elif [ ${PYVERSION} -eq 38 ]; then - pip install numpy==1.18.3 - elif [ ${PYVERSION} -eq 39 ]; then - pip install numpy==1.19.3 - else - pip install numpy==1.21.4 - fi - - # set an environment variable needed to locate numpy header files - source tools/set_NUMPY_INCLUDEDIR.sh + pip install --upgrade setuptools build # produce conventional (non-manylinux) wheel - python3 setup.py bdist_wheel + python3 -m build --wheel --outdir dist/ # use auditwheel from PyPA to repair all wheels and make them manylinux-compatible auditwheel repair dist/trexio-${TR_VERSION}-${CPYTHON}-*.whl diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index c7402cb..a880193 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -26,33 +26,42 @@ else fi # Install/upgrade packages required for the installation -python3 -m pip install --upgrade setuptools wheel pip +python3 -m pip install --upgrade setuptools build pip python3 -m pip install -r requirements.txt # export NUMPY_INCLUDEDIR environment variable needed for the proper setup -source tools/set_NUMPY_INCLUDEDIR.sh - -if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then - echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)." - exit 1 -fi +#source tools/set_NUMPY_INCLUDEDIR.sh +#if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then +# echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)." +# exit 1 +#fi # Create build directory and compile extension files (*.c) # --no-user-cfg disables custom .cfg files of the user machine, so that only setup.cfg is used -python3 -s setup.py --no-user-cfg build +#python3 -s setup.py --no-user-cfg build # Local inplace build of the .so module with SWIG-produced pytrexio_wrap.c (from the SWIG documentation) #python3 setup.py build_ext --inplace --swig-opts="-modern" # Create distributions: + +# OLD WAY (DEPRECATED BY PYPA) # 1) sdist produces .tar.gz with all files necessary for manual compilation; # 2) bdist_whell produces .whl wheel distribution (see https://www.python.org/dev/peps/pep-0425/). -python3 setup.py sdist bdist_wheel +#python3 setup.py sdist bdist_wheel + +# NEW WAY (USING BUILD PACKAGE OF PYPA) +python3 -m build --sdist --wheel --outdir dist/ # Install pytrexio in the current environment from the aforementioned wheel + +# OLD WAY # --force-reinstall is needed here because build-system pre-installs pytrexio in the environment # but does not install things in the corresponding site-packages directory -python3 -m pip install dist/trexio-*.whl --force-reinstall +#python3 -m pip install dist/trexio-*.whl --force-reinstall + +# NEW WAY +python3 -m pip install dist/trexio-*.whl # Run the command below in the root directory to install the package in 'editable' (-e) mode without dependencies (--no-deps) #python -m pip install -e . --no-deps diff --git a/python/setup.py b/python/setup.py index c7b7a65..2348038 100644 --- a/python/setup.py +++ b/python/setup.py @@ -22,19 +22,16 @@ def parse_setuppy_commands(): do_sdist = parse_setuppy_commands() # this was recommended to solve the problem of the missing numpy header files -# bit it causes `pip install .` to fail with numpy module not found error -#try: -# import numpy -#except ImportError: -# raise Exception("numpy Python package cannot be imported.") -#numpy_includedir = numpy.get_include() +try: + import numpy +except ImportError: + raise Exception("numpy Python package cannot be imported.") -# this does not cause aforementioned issue but the includedir points to system-wide numpy and not to venv-wide -#from distutils.sysconfig import get_python_inc -#numpy_includedir = os.path.join(get_python_inc(plat_specific=1), 'numpy') +numpy_includedir = numpy.get_include() # dirty workaround: get numpy includedir from the environment variable that can be pre-set using set_NUMPY_INCLUDEDIR.sh -numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None) +#numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None) + numpy_isUndefined = numpy_includedir is None or numpy_includedir=="" if numpy_isUndefined and not do_sdist: diff --git a/tools/build_doc.sh b/tools/build_doc.sh index bd932e2..614a537 100755 --- a/tools/build_doc.sh +++ b/tools/build_doc.sh @@ -123,10 +123,6 @@ function main() { then cd ${DOCS} rm -f index.html - # enter Google verificator into the HTML head - local GOOGLE_VERIF="" - LINE_NO=$(($(awk '/meta name="viewport"/{print NR}' README.html) + 1)) - sed -i "$LINE_NO i ${GOOGLE_VERIF}" README.html ln README.html index.html exit 0 diff --git a/tools/prepare_python.sh b/tools/prepare_python.sh index 5381fae..0b22ae6 100755 --- a/tools/prepare_python.sh +++ b/tools/prepare_python.sh @@ -31,8 +31,10 @@ cp ${INCLUDIR}/trexio.h ${PYDIR}/src # fix needed to define HAVE_HDF5 symbol so that Python extension is always compiled with HDF5 (without including config.h) # add "#define HAVE_HDF5 1" line after "#include stdint.h" using awk and sed -LINE_NO=$(($(awk '/stdint.h/{print NR}' ${PYDIR}/src/trexio.h) + 1)) -sed -i "$LINE_NO i #define HAVE_HDF5 1" ${PYDIR}/src/trexio.h +export LINE_NO=$(($(awk '/stdint.h/{print NR}' ${PYDIR}/src/trexio.h) + 1)) +sed -i'' -e "$LINE_NO"'i \ +#define HAVE_HDF5 1 +' -- ${PYDIR}/src/trexio.h # Copy additional info cp ${TREXIO_ROOT}/AUTHORS ${TREXIO_ROOT}/LICENSE ${PYDIR}