From 7651d19f2269a8be1660f7b16f4345f3756ee8ae Mon Sep 17 00:00:00 2001 From: Anthony Scemama Date: Wed, 4 May 2022 16:41:16 +0200 Subject: [PATCH] Out of source build of python --- Makefile.am | 37 +++++++++++++------ python/README.md | 24 ++++++------ python/build_pyqmckl.sh | 33 ----------------- ...all_pyqmckl.sh => manual_install_qmckl.sh} | 4 +- python/pip_install_pyqmckl.sh | 14 ------- python/setup.py | 6 +-- python/src/process_header.py | 17 +++++---- python/src/{pyqmckl.i => qmckl.i} | 0 8 files changed, 52 insertions(+), 83 deletions(-) delete mode 100755 python/build_pyqmckl.sh rename python/{manual_install_pyqmckl.sh => manual_install_qmckl.sh} (82%) delete mode 100755 python/pip_install_pyqmckl.sh rename python/src/{pyqmckl.i => qmckl.i} (100%) diff --git a/Makefile.am b/Makefile.am index 52ccd7f..6d7171a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -65,7 +65,7 @@ AM_CPPFLAGS += -DQMCKL_TEST_DIR=\"$(QMCKL_TEST_DIR)\" lib_LTLIBRARIES = src/libqmckl.la src_libqmckl_la_SOURCES = $(qmckl_h) $(src_qmckl_f) $(C_FILES) $(F_FILES) $(H_PRIVATE_FUNC_FILES) $(H_PRIVATE_TYPE_FILES) -src_libqmckl_la_LDFLAGS = $(LDFLAGS) +src_libqmckl_la_LDFLAGS = $(LDFLAGS) CLEANFILES+=$(test_qmckl_fo) $(src_qmckl_fo) $(test_qmckl_o) $(src_qmckl_o) $(FH_TYPE_FILES) $(FH_FUNC_FILES) @@ -172,26 +172,39 @@ cppcheck.out: $(qmckl_h) setup_py = $(srcdir)/python/setup.py process_header_py = $(srcdir)/python/src/process_header.py -qmckl_py = $(srcdir)/python/qmckl.py test_py = $(srcdir)/python/test/test_api.py -qmckl_wrap_c = $(srcdir)/python/src/qmckl_wrap.c -pyqmckl_i = $(srcdir)/python/src/pyqmckl.i -pyqmckl_include_i = $(srcdir)/python/src/pyqmckl_include.i +qmckl_i = $(srcdir)/python/src/qmckl.i numpy_i = $(srcdir)/python/src/numpy.i +qmckl_wrap_c = python/src/qmckl_wrap.c +qmckl_include_i = python/src/qmckl_include.i +qmckl_py = python/qmckl/qmckl.py + +$(qmckl_include_i): $(qmckl_h) $(process_header_py) + $(MKDIR_P) python/src + python $(process_header_py) $(qmckl_h) + mv qmckl_include.i $(qmckl_include_i) -python-install: $(qmckl_h) $(pyqmckl_i) $(process_header_py) $(setup_py) - cp $(qmckl_h) $(srcdir)/python/src/ - cd $(srcdir)/python/ && \ - ./pip_install_pyqmckl.sh +$(qmckl_py): $(qmckl_i) $(qmckl_include_i) + swig -Iinclude -Ipython/src -python -py3 -builtin -o $(qmckl_wrap_c) $(qmckl_i) + +$(qmckl_wrap_c): $(qmckl_py) + +python-install: $(qmckl_h) $(qmckl_i) $(setup_py) $(qmckl_py) $(qmckl_wrap_c) + $(MKDIR_P) python/src + cd python ; \ + [[ ! -f pyproject.toml ]] && \ + cp $(abs_srcdir)/python/{pyproject.toml,requirements.txt,README.md,setup.py} . ; \ + cp src/qmckl.py . ; \ + pip install . python-test: $(test_py) - cd $(srcdir)/python/test/ && \ + cd $(abs_srcdir)/python/test/ && \ python test_api.py CLEANFILES += $(qmckl_wrap_c) \ - $(qmckl_py) \ - $(pyqmckl_include_i) + $(qmckl_include_i) \ + $(qmckl_py) .PHONY: cppcheck python-test python-install diff --git a/python/README.md b/python/README.md index 9d468ef..8dc41bb 100644 --- a/python/README.md +++ b/python/README.md @@ -12,31 +12,31 @@ ## Manual installation 1. Install the QMCkl library (see upstream instructions) -2. `./manual_install_pyqmckl.sh` which should do the following -3. Copy the produced `_pyqmckl.so` and `pyqmckl.py` files into your working directory and do not forget to `import pyqmckl` in your Python scripts +2. `./manual_install_qmckl.sh` which should do the following +3. Copy the produced `_qmckl.so` and `qmckl.py` files into your working directory and do not forget to `import qmckl` in your Python scripts The second step executes the following under the hood: -1. `./build_pyqmckl.sh` -2. ` -I/usr/include/python3.8 -c -fPIC pyqmckl_wrap.c` to compile the wrapper code into an object file using the `` (replace with your C compiler, e.g. `gcc`) on your machine -3. ` -shared pyqmckl_wrap.o -lqmckl -o _pyqmckl.so` to produce the final C extension (this requires the `qmckl` library to be installed and present in the linking paths together with all its dependencies like `trexio`) +1. `./build_qmckl.sh` +2. ` -I/usr/include/python3.8 -c -fPIC qmckl_wrap.c` to compile the wrapper code into an object file using the `` (replace with your C compiler, e.g. `gcc`) on your machine +3. ` -shared qmckl_wrap.o -lqmckl -o _qmckl.so` to produce the final C extension (this requires the `qmckl` library to be installed and present in the linking paths together with all its dependencies like `trexio`) ## Python-ic installation (recommended) 1. Install the QMCkl library (see upstream instructions) -2. `./pip_install_pyqmckl.sh` +2. `./pip_install_qmckl.sh` -The last step runs `./build_pyqmckl.sh`, copies the result into the `pyqmckl/` directory and -then runs `pip install .` to install the `pyqmckl` Python package in your environment. +The last step runs `./build_qmckl.sh`, copies the result into the `qmckl/` directory and +then runs `pip install .` to install the `qmckl` Python package in your environment. ## SWIG pre-processing -Both aforementioned steps call `build_pyqmckl.sh` script which does the following pre-processing for SWIG +Both aforementioned steps call `build_qmckl.sh` script which does the following pre-processing for SWIG 1. Copy the latest `qmckl.h` file fron `include/` into the `src/` directory -2. `python process_header.py` to generate `pyqmckl_include.i` list of SWIG patterns -3. `swig -python -py3 -builtin -threads -o pyqmckl_wrap.c pyqmckl.i` to generate the SWIG wrapper code in C and `pyqmckl.py` module in Python. -**Note:** for this to work three files have to be present in the working directory: `pyqmckl.i`, `pyqmckl_include.i` and `numpy.i`. +2. `python process_header.py` to generate `qmckl_include.i` list of SWIG patterns +3. `swig -python -py3 -builtin -threads -o qmckl_wrap.c qmckl.i` to generate the SWIG wrapper code in C and `qmckl.py` module in Python. +**Note:** for this to work three files have to be present in the working directory: `qmckl.i`, `qmckl_include.i` and `numpy.i`. diff --git a/python/build_pyqmckl.sh b/python/build_pyqmckl.sh deleted file mode 100755 index 961db80..0000000 --- a/python/build_pyqmckl.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -e -set -x - -#cp ../include/qmckl.h src/ - -cd src/ - -# check if qmckl header exists -if [[ ! -f 'qmckl.h' ]]; then - echo "qmckl.h NOT FOUND in the src/ directory" - exit 1 -fi - -# process the qmckl header file to get patterns for SWIG -python process_header.py - -# check if SWIG files exist -SWIG_LIST='pyqmckl.i pyqmckl_include.i numpy.i' -for file in $SWIG_LIST; do - if [[ ! -f $file ]]; then - echo "$file NOT FOUND in the src/ directory" - exit 1 - fi -done - -# run SWIG interface file to produce the Python wrappers -# `-threads` to release GIL before the call to C functions (for multi-threading) -# `-builtin` to activate Python-ic built-in types -swig -python -py3 -builtin -o qmckl_wrap.c pyqmckl.i - -cd .. diff --git a/python/manual_install_pyqmckl.sh b/python/manual_install_qmckl.sh similarity index 82% rename from python/manual_install_pyqmckl.sh rename to python/manual_install_qmckl.sh index 24f5c6a..ff1fff6 100755 --- a/python/manual_install_pyqmckl.sh +++ b/python/manual_install_qmckl.sh @@ -4,7 +4,7 @@ set -x set -e # swig pre-processing -./build_pyqmckl.sh +./build_qmckl.sh cd src/ @@ -12,7 +12,7 @@ cd src/ cc -c -fPIC `pkg-config --cflags qmckl` -I/usr/include/python3.8 qmckl_wrap.c -o qmckl_wrap.o # link against the previously installed QMCkl library (as detected by pkg-config) -cc -shared pyqmckl_wrap.o `pkg-config --libs qmckl` -o _qmckl.so +cc -shared qmckl_wrap.o `pkg-config --libs qmckl` -o _qmckl.so cd .. diff --git a/python/pip_install_pyqmckl.sh b/python/pip_install_pyqmckl.sh deleted file mode 100755 index 6af48a4..0000000 --- a/python/pip_install_pyqmckl.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -x -set -e - -./build_pyqmckl.sh - -# copy swig-produced pyqmckl.py module into the pyqmckl/ folder -#cp src/qmckl.py qmckl/ -cp src/qmckl.py ./ - -# install using pip -pip install . - diff --git a/python/setup.py b/python/setup.py index 59a7256..6932d61 100644 --- a/python/setup.py +++ b/python/setup.py @@ -14,8 +14,8 @@ with open("README.md", "r") as fh: # Define the name of the Python package MODULE_NAME = "qmckl" -# Define pyqmckl extension module based on SWIG interface file (requires qmckl.h) -pyqmckl_module = Extension(name = "._" + MODULE_NAME, +# Define qmckl extension module based on SWIG interface file (requires qmckl.h) +qmckl_module = Extension(name = "._" + MODULE_NAME, sources = [ join("src", MODULE_NAME + "_wrap.c") ], #include_dirs = [numpy_includedir], #library_dirs = [], @@ -34,7 +34,7 @@ setup(name = MODULE_NAME, description = """Python API of the QMCkl library""", long_description = long_description, long_description_content_type = "text/markdown", - ext_modules = [pyqmckl_module], + ext_modules = [qmckl_module], py_modules = [MODULE_NAME], url = "https://github.com/TREX-CoE/qmckl", license = "BSD", diff --git a/python/src/process_header.py b/python/src/process_header.py index 07a2966..95a3588 100644 --- a/python/src/process_header.py +++ b/python/src/process_header.py @@ -1,6 +1,9 @@ +#!/usr/bin/env python3 import os +import sys +qmckl_h = sys.argv[1] collect = False process = False @@ -13,9 +16,9 @@ numbers = {} qmckl_public_api = [] qmckl_errors = [] -with open("qmckl.h", 'r') as f_in: +with open(qmckl_h, 'r') as f_in: for line in f_in: - + # get the errors but without the type cast because SWIG does not recognize it if '#define' in line and 'qmckl_exit_code' in line: qmckl_errors.append(line.strip().replace('(qmckl_exit_code)','')) @@ -62,11 +65,11 @@ with open("qmckl.h", 'r') as f_in: # print(pattern) continue - # if size_max is not provided then the function should deal with numbers or string + # if size_max is not provided then the function should deal with numbers or string #elif 'num' in line and 'get' in func_name: elif ';' in line and 'get' in func_name: - # special case - if 'size_max' in line: + # special case + if 'size_max' in line: continue #print(line) @@ -145,7 +148,7 @@ processed = list(arrays.keys()) + list(numbers.keys()) # print(v) -with open("pyqmckl_include.i", 'w') as f_out: +with open("qmckl_include.i", 'w') as f_out: # write the list of errors as constants without the type cast for e in qmckl_errors: @@ -154,7 +157,7 @@ with open("pyqmckl_include.i", 'w') as f_out: swig_type = '' for v in numbers.values(): - + if 'int' in v['datatype']: swig_type = 'int' elif 'float' in v['datatype'] or 'double' in v['datatype']: diff --git a/python/src/pyqmckl.i b/python/src/qmckl.i similarity index 100% rename from python/src/pyqmckl.i rename to python/src/qmckl.i