1
0
mirror of https://github.com/TREX-CoE/qmckl.git synced 2024-06-18 11:15:38 +02:00

Merge pull request #80 from q-posev/add-python-api

Add python API (pyqmckl)
This commit is contained in:
Anthony Scemama 2022-05-03 17:02:11 +02:00 committed by GitHub
commit 4edc59b6ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 51242 additions and 1 deletions

View File

@ -65,6 +65,7 @@ AM_CPPFLAGS += -DQMCKL_TEST_DIR=\"$(QMCKL_TEST_DIR)\"
lib_LTLIBRARIES = src/libqmckl.la
src_libqmckl_la_SOURCES = $(qmckl_h) $(src_qmckl_f) $(C_FILES) $(F_FILES) $(H_PRIVATE_FUNC_FILES) $(H_PRIVATE_TYPE_FILES)
src_libqmckl_la_LDFLAGS = $(LDFLAGS)
CLEANFILES+=$(test_qmckl_fo) $(src_qmckl_fo) $(test_qmckl_o) $(src_qmckl_o) $(FH_TYPE_FILES) $(FH_FUNC_FILES)
@ -169,6 +170,29 @@ cppcheck.out: $(qmckl_h)
--language=c --std=c99 -rp --platform=unix64 \
-I$(srcdir)/include -I$(top_builddir)/include *.c *.h 2>../$@
.PHONY: cppcheck
setup_py = $(srcdir)/python/setup.py
process_header_py = $(srcdir)/python/src/process_header.py
pyqmckl_py = $(srcdir)/python/pyqmckl/pyqmckl.py
test_py = $(srcdir)/python/test/test_api.py
pyqmckl_wrap_c = $(srcdir)/python/src/pyqmckl_wrap.c
pyqmckl_i = $(srcdir)/python/src/pyqmckl.i
pyqmckl_include_i = $(srcdir)/python/src/pyqmckl_include.i
numpy_i = $(srcdir)/python/src/numpy.i
python-install: $(qmckl_h) $(pyqmckl_i) $(process_header_py) $(setup_py)
cp $(qmckl_h) python/src/
cd python/ && \
./pip_install_pyqmckl.sh
python-test: $(test_py)
cd python/test/ && \
python test_api.py
CLEANFILES += $(pyqmckl_wrap_c) \
$(pyqmckl_include_i) \
$(pyqmckl_py)
.PHONY: cppcheck python-test python-install
endif

View File

@ -317,6 +317,8 @@ if test "x${QMCKL_DEVEL}" != "x"; then
HAS_CPPCHECK=1
fi
AX_PKG_SWIG(4.0.0, [], AC_MSG_WARN([SWIG is required to build Python API.]) )
fi
# Enable Verificarlo tests

139
m4/ax_pkg_swig.m4 Normal file
View File

@ -0,0 +1,139 @@
# ===========================================================================
# https://www.gnu.org/software/autoconf-archive/ax_pkg_swig.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_PKG_SWIG([major.minor.micro], [action-if-found], [action-if-not-found])
#
# DESCRIPTION
#
# This macro searches for a SWIG installation on your system. If found,
# then SWIG is AC_SUBST'd; if not found, then $SWIG is empty. If SWIG is
# found, then SWIG_LIB is set to the SWIG library path, and AC_SUBST'd.
#
# You can use the optional first argument to check if the version of the
# available SWIG is greater than or equal to the value of the argument. It
# should have the format: N[.N[.N]] (N is a number between 0 and 999. Only
# the first N is mandatory.) If the version argument is given (e.g.
# 1.3.17), AX_PKG_SWIG checks that the swig package is this version number
# or higher.
#
# As usual, action-if-found is executed if SWIG is found, otherwise
# action-if-not-found is executed.
#
# In configure.in, use as:
#
# AX_PKG_SWIG(1.3.17, [], [ AC_MSG_ERROR([SWIG is required to build..]) ])
# AX_SWIG_ENABLE_CXX
# AX_SWIG_MULTI_MODULE_SUPPORT
# AX_SWIG_PYTHON
#
# LICENSE
#
# Copyright (c) 2008 Sebastian Huber <sebastian-huber@web.de>
# Copyright (c) 2008 Alan W. Irwin
# Copyright (c) 2008 Rafael Laboissiere <rafael@laboissiere.net>
# Copyright (c) 2008 Andrew Collier
# Copyright (c) 2011 Murray Cumming <murrayc@openismus.com>
# Copyright (c) 2021 Vincent Danjean <Vincent.Danjean@ens-lyon.org>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.
#
# As a special exception, the respective Autoconf Macro's copyright owner
# gives unlimited permission to copy, distribute and modify the configure
# scripts that are the output of Autoconf when processing the Macro. You
# need not follow the terms of the GNU General Public License when using
# or distributing such scripts, even though portions of the text of the
# Macro appear in them. The GNU General Public License (GPL) does govern
# all other use of the material that constitutes the Autoconf Macro.
#
# This special exception to the GPL applies to versions of the Autoconf
# Macro released by the Autoconf Archive. When you make and distribute a
# modified version of the Autoconf Macro, you may extend this special
# exception to the GPL to apply to your modified version as well.
#serial 14
AC_DEFUN([AX_PKG_SWIG],[
# Find path to the "swig" executable.
AC_PATH_PROGS([SWIG],[swig swig3.0 swig2.0])
if test -z "$SWIG" ; then
m4_ifval([$3],[$3],[:])
elif test -z "$1" ; then
m4_ifval([$2],[$2],[:])
else
AC_MSG_CHECKING([SWIG version])
[swig_version=`$SWIG -version 2>&1 | grep 'SWIG Version' | sed 's/.*\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/g'`]
AC_MSG_RESULT([$swig_version])
if test -n "$swig_version" ; then
# Calculate the required version number components
[required=$1]
[required_major=`echo $required | sed 's/[^0-9].*//'`]
if test -z "$required_major" ; then
[required_major=0]
fi
[required=`echo $required. | sed 's/[0-9]*[^0-9]//'`]
[required_minor=`echo $required | sed 's/[^0-9].*//'`]
if test -z "$required_minor" ; then
[required_minor=0]
fi
[required=`echo $required. | sed 's/[0-9]*[^0-9]//'`]
[required_patch=`echo $required | sed 's/[^0-9].*//'`]
if test -z "$required_patch" ; then
[required_patch=0]
fi
# Calculate the available version number components
[available=$swig_version]
[available_major=`echo $available | sed 's/[^0-9].*//'`]
if test -z "$available_major" ; then
[available_major=0]
fi
[available=`echo $available | sed 's/[0-9]*[^0-9]//'`]
[available_minor=`echo $available | sed 's/[^0-9].*//'`]
if test -z "$available_minor" ; then
[available_minor=0]
fi
[available=`echo $available | sed 's/[0-9]*[^0-9]//'`]
[available_patch=`echo $available | sed 's/[^0-9].*//'`]
if test -z "$available_patch" ; then
[available_patch=0]
fi
# Convert the version tuple into a single number for easier comparison.
# Using base 100 should be safe since SWIG internally uses BCD values
# to encode its version number.
required_swig_vernum=`expr $required_major \* 10000 \
\+ $required_minor \* 100 \+ $required_patch`
available_swig_vernum=`expr $available_major \* 10000 \
\+ $available_minor \* 100 \+ $available_patch`
if test $available_swig_vernum -lt $required_swig_vernum; then
AC_MSG_WARN([SWIG version >= $1 is required. You have $swig_version.])
SWIG=''
m4_ifval([$3],[$3],[])
else
AC_MSG_CHECKING([for SWIG library])
SWIG_LIB=`$SWIG -swiglib`
AC_MSG_RESULT([$SWIG_LIB])
m4_ifval([$2],[$2],[])
fi
else
AC_MSG_WARN([cannot determine SWIG version])
SWIG=''
m4_ifval([$3],[$3],[])
fi
fi
AC_SUBST([SWIG_LIB])
])

42
python/README.md Normal file
View File

@ -0,0 +1,42 @@
# Python API of the QMCkl library
## Requirements
- `setuptools`
- `numpy`
- `swig` (>= 4.0)
## Manual installation
1. Install the QMCkl library (see upstream instructions)
2. `./manual_install_pyqmckl.sh` which should do the following
3. Copy the produced `_pyqmckl.so` and `pyqmckl.py` files into your working directory and do not forget to `import pyqmckl` in your Python scripts
The second step executes the following under the hood:
1. `./build_pyqmckl.sh`
2. `<c-compiler> -I/usr/include/python3.8 -c -fPIC pyqmckl_wrap.c` to compile the wrapper code into an object file using the `<c-compiler>` (replace with your C compiler, e.g. `gcc`) on your machine
3. `<c-compiler> -shared pyqmckl_wrap.o -lqmckl -o _pyqmckl.so` to produce the final C extension (this requires the `qmckl` library to be installed and present in the linking paths together with all its dependencies like `trexio`)
## Python-ic installation (recommended)
1. Install the QMCkl library (see upstream instructions)
2. `./pip_install_pyqmckl.sh`
The last step runs `./build_pyqmckl.sh`, copies the result into the `pyqmckl/` directory and
then runs `pip install .` to install the `pyqmckl` Python package in your environment.
## SWIG pre-processing
Both aforementioned steps call `build_pyqmckl.sh` script which does the following pre-processing for SWIG
1. Copy the latest `qmckl.h` file fron `include/` into the `src/` directory
2. `python process_header.py` to generate `pyqmckl_include.i` list of SWIG patterns
3. `swig -python -py3 -builtin -threads -o pyqmckl_wrap.c pyqmckl.i` to generate the SWIG wrapper code in C and `pyqmckl.py` module in Python.
**Note:** for this to work three files have to be present in the working directory: `pyqmckl.i`, `pyqmckl_include.i` and `numpy.i`.

31
python/build_pyqmckl.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
set -e
set -x
cp ../include/qmckl.h src/
cd src/
# check if qmckl header exists
if [[ ! -f 'qmckl.h' ]]; then
echo "qmckl.h NOT FOUND"
exit 1
fi
# process the qmckl header file to get patterns for SWIG
python process_header.py
# check if SWIG files exist
SWIG_LIST='pyqmckl.i pyqmckl_include.i numpy.i'
for file in $SWIG_LIST; do
if [[ ! -f $file ]]; then
echo "$file NOT FOUND"
exit 1
fi
done
# run SWIG interface file to produce the Python wrappers
swig -python -py3 -builtin -threads -o pyqmckl_wrap.c pyqmckl.i
cd ..

View File

@ -0,0 +1,25 @@
#!/bin/bash
set -x
set -e
# swig pre-processing
./build_pyqmckl.sh
cd src/
# compile the wrapper code
cc -c -fPIC `pkg-config --cflags qmckl` -I/usr/include/python3.8 pyqmckl_wrap.c -o pyqmckl_wrap.o
# link against the previously installed QMCkl library (as detected by pkg-config)
cc -shared pyqmckl_wrap.o `pkg-config --libs qmckl` -o _pyqmckl.so
cd ..
# copy the produced files into the test dir
cp src/_pyqmckl.so src/pyqmckl.py test/
# run tests
cd test/
python test_api.py
cd ..

13
python/pip_install_pyqmckl.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
set -x
set -e
./build_pyqmckl.sh
# copy swig-produced pyqmckl.py module into the pyqmckl/ folder
cp src/pyqmckl.py pyqmckl/
# install using pip
pip install .

7
python/pyproject.toml Normal file
View File

@ -0,0 +1,7 @@
[build-system]
requires = [
"setuptools>=42",
"wheel",
"numpy>=1.17.3"
]
build-backend = "setuptools.build_meta"

View File

@ -0,0 +1,2 @@
from .pyqmckl import *
from ._version import __version__

View File

@ -0,0 +1 @@
__version__ = "0.2.0"

2
python/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
setuptools>=42
numpy>=1.17.3

73
python/setup.py Normal file
View File

@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""
setup.py file for pyqmckl package
"""
from setuptools import setup, Extension
from os.path import join
# Read the long description
with open("README.md", "r") as fh:
long_description = fh.read()
# Read the version string from the file
VERSIONFILE = "pyqmckl/_version.py"
try:
exec(open(VERSIONFILE).read())
except:
raise IOError("Could not open the version file %s." % (VERSIONFILE, ))
version_r = __version__
if not version_r:
raise RuntimeError("Unable to find a version string in %s." % (VERSIONFILE, ))
# Define the name of the Python package
mod_name = 'pyqmckl'
# Define pyqmckl extension module based on SWIG interface file (requires qmckl.h)
pyqmckl_module = Extension(name = mod_name + '._' + mod_name,
sources = [ join('src', mod_name + '_wrap.c') ],
#include_dirs = [numpy_includedir],
#library_dirs = [],
#runtime_library_dirs = [],
libraries = ['qmckl'],
extra_compile_args = ['-Wall'],
#extra_link_args = [h5_ldflags],
#swig_opts = ['-py3' , '-builtin'],
depends = [ join('src', 'qmckl.h') ],
language = 'c'
)
setup(name = mod_name,
version = version_r,
author = "TREX-CoE",
author_email = "posenitskiy@irsamc.ups-tlse.fr",
description = """Python API of the QMCkl library""",
long_description = long_description,
long_description_content_type = "text/markdown",
ext_modules = [pyqmckl_module],
py_modules = [mod_name],
packages = [mod_name],
url = 'https://github.com/TREX-CoE/qmckl',
license = 'BSD',
classifiers=[
"Intended Audience :: Science/Research",
"Intended Audience :: Developers",
"Topic :: Scientific/Engineering",
"Programming Language :: C",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: Implementation :: CPython",
"License :: OSI Approved :: BSD License",
"Operating System :: POSIX",
"Operating System :: Unix",
"Operating System :: MacOS"
],
python_requires = ">=3.0",
install_requires = ['numpy>=1.17.3']
)

3183
python/src/numpy.i Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,175 @@
import os
collect = False
process = False
get_name = False
block = []
res_str = ''
func_name = ''
arrays = {}
numbers = {}
qmckl_public_api = []
with open("qmckl.h", 'r') as f_in:
for line in f_in:
if get_name:
words = line.strip().split()
if '(' in words[0]:
func_name = words[0].split('(')[0]
else:
func_name = words[0]
if 'get' in func_name or 'set' in func_name:
qmckl_public_api.append(func_name)
get_name = False
if 'qmckl_exit_code' in line:
words = line.strip().split()
if len(words) > 1 and 'qmckl_exit_code' in words[0]:
# this means that the function name is on the same line as `qmckl_exit_code`
func_name = words[1].split('(')[0]
if 'get' in func_name or 'set' in func_name:
qmckl_public_api.append(func_name)
elif len(words) == 1:
# this means that the function name is the first element on the next line
get_name = True
#continue # do not `continue` here otherwise collect is not True for some functions
# process functions - oneliners (for arrays)
if 'size_max' in line and ';' in line:
tmp_list = line.split(',')
for i,s in enumerate(tmp_list):
if 'size_max' in s:
end_str = tmp_list[i].replace(';','').replace('\n','')
pattern = f"({tmp_list[i-1]} ,{end_str}"
datatype = tmp_list[i-1].replace('const','').replace('*','').split()[0]
arrays[func_name] = {
'datatype' : datatype,
'pattern' : pattern
}
#if 'qmckl_get_jastrow_type_nucl_vector' in func_name:
# print(line)
# print(pattern)
continue
# if size_max is not provided then the function should deal with numbers or string
#elif 'num' in line and 'get' in func_name:
elif ';' in line and 'get' in func_name:
# special case
if 'size_max' in line:
continue
#print(line)
tmp_str = line.split(',')[-1].strip()
pattern = tmp_str.replace(')','').replace(';','')
datatype = pattern.replace('const','').replace('*','').split()[0]
numbers[func_name] = {
'datatype' : datatype,
'pattern' : pattern
}
continue
# for multilne functions - append line by line to the list
else:
block.append(line)
collect = True
continue
# if size_max is encountered within the multiline function
if 'size_max' in line and collect:
#if 'qmckl_get_electron_rescale_factor_en' in func_name:
# print("LOL")
# this will not work for 2-line functions where array argument is on the same line as
# func name and size_max argument is on the next line
if not 'qmckl_exit_code' in block[-1] and not '*/' in line:
pattern = '(' + block[-1].strip() + line.strip().replace(';','')
datatype = pattern.replace('const','').replace('*','').replace('(','').split()[0]
collect = False
block = []
arrays[func_name] = {
'datatype' : datatype,
'pattern' : pattern
}
continue
#if 'num' in line and 'get' in func_name and not 'qmckl_get' in line and collect:
if 'get' in func_name and not 'qmckl_get' in line and collect and ';' in line:
#print(func_name)
#print(line)
pattern = line.replace(';','').replace(')','').strip()
datatype = pattern.replace('const','').replace('*','').split()[0]
collect = False
block = []
numbers[func_name] = {
'datatype' : datatype,
'pattern' : pattern
}
continue
# stop/continue multiline function analyzer
if collect and ')' in line:
collect = False
block = []
continue
else:
block.append(line)
continue
# remove buggy qmckl_get_electron_rescale_factor_en key
#arrays.pop('qmckl_get_electron_rescale_factor_en')
processed = list(arrays.keys()) + list(numbers.keys())
#for pub_func in qmckl_public_api:
#if pub_func not in processed and 'set' not in pub_func:
#print("TODO", pub_func)
#print(v['datatype'])
#for k,v in numbers.items():
# print(v)
with open("pyqmckl_include.i", 'w') as f_out:
swig_type = ''
for v in numbers.values():
if 'int' in v['datatype']:
swig_type = 'int'
elif 'float' in v['datatype'] or 'double' in v['datatype']:
swig_type = 'float'
elif 'char' in v['datatype'] or 'bool' in v['datatype']:
#print('SWIG, skipping', v['datatype'], v['pattern'])
continue
else:
raise TypeError(f"Unknown datatype for swig conversion: {v['datatype']}")
f_out.write(f"%apply {swig_type} *OUTPUT {{ {v['pattern']} }};\n")
for k,v in arrays.items():
if 'char' in v['datatype']:
#print("String type", k, v)
pass
if len(v['pattern'].split(',')) != 2:
print('Problemo', k, v)
continue
if 'get' in k:
f_out.write(f"%apply ( {v['datatype']}* ARGOUT_ARRAY1 , int64_t DIM1 ) {{ {v['pattern']} }};\n")
elif 'set' in k:
f_out.write(f"%apply ( {v['datatype']}* IN_ARRAY1 , int64_t DIM1 ) {{ {v['pattern']} }};\n")
#else:
#print("HOW-TO ?", k)

66
python/src/pyqmckl.i Normal file
View File

@ -0,0 +1,66 @@
%module pyqmckl
/* Define SWIGWORDSIZE in order to properly align long integers on 64-bit system */
#define SWIGWORDSIZE64
%{
#define SWIG_FILE_WITH_INIT
/* Include the headers in the wrapper code */
#include "qmckl.h"
%}
/* Include stdint to recognize types from stdint.h */
%include <stdint.i>
/* Include typemaps to play with input/output re-casting (e.g. C pointers) */
%include typemaps.i
%apply int *OUTPUT { qmckl_exit_code *exit_code};
/* Avoid passing file_name length as an additiona argument */
%apply (char *STRING, int LENGTH) { (const char* file_name, const int64_t size_max) };
/* For functions that return strings */
%include <cstring.i>
%cstring_bounded_output(char* function_name, 1024);
%cstring_bounded_output(char* message, 1024);
/* This block is needed make SWIG convert NumPy arrays to/from from the C pointer and size_max argument.
NOTE: `numpy.i` interface file is not part of SWIG but it is included in the numpy distribution (under numpy/tools/swig/numpy.i)
*/
%include "numpy.i"
%init %{
import_array();
%}
/* Typemaps below change the type of numpy array dimensions from int to int64_t */
%numpy_typemaps(double, NPY_DOUBLE, int64_t)
%numpy_typemaps(float, NPY_FLOAT, int64_t)
%numpy_typemaps(int32_t, NPY_INT32, int64_t)
%numpy_typemaps(int64_t, NPY_INT64, int64_t)
/* Include typemaps generated by the process_header.py script */
%include "pyqmckl_include.i"
/* exception.i is a generic (language-independent) module */
%include "exception.i"
/* Error handling
TODO: the sizeof() check below if a dummy workaround
It is good to skip exception raise for functions like context_create and others, but might fail
if sizeof(result) == sizeof(qmckl_exit_code), e.g. for functions that return non-zero integers or floats
*/
%exception {
$action
if (result != 0 && sizeof(result) == sizeof(qmckl_exit_code)) {
SWIG_exception_fail(SWIG_RuntimeError, qmckl_string_of_error(result));
}
}
/* The exception handling above does not work for void functions like lock/unlock so exclude them for now */
%ignore qmckl_lock;
%ignore qmckl_unlock;
/* Parse the header files to generate wrappers */
%include "qmckl.h"

Binary file not shown.

47404
python/test/data/data.py Normal file

File diff suppressed because it is too large Load Diff

52
python/test/test_api.py Normal file
View File

@ -0,0 +1,52 @@
"""
This is the test of the Python API of the QMCkl library.
It is the `bench_mos.c` C code adapted from the `qmckl_bench`
repo and translated into Python with some modifications.
"""
from os.path import join
import time
import pyqmckl as pq
from data.data import coord
walk_num = 100
elec_num = 158
ITERMAX = 10
ctx = pq.qmckl_context_create()
fname = join('data', 'Alz_small.h5')
rc = pq.qmckl_trexio_read(ctx, fname)
assert rc==0
print(pq.qmckl_string_of_error(rc))
rc = pq.qmckl_set_electron_walk_num(ctx, walk_num)
assert rc==0
rc, mo_num = pq.qmckl_get_mo_basis_mo_num(ctx)
assert rc==0
rc = pq.qmckl_set_electron_coord(ctx, 'T', coord)
assert rc==0
size_max = 5*walk_num*elec_num*mo_num
rc, mo_vgl = pq.qmckl_get_mo_basis_mo_vgl(ctx, size_max)
assert rc==0
start = time.clock_gettime_ns(time.CLOCK_REALTIME)
for _ in range(ITERMAX):
rc, mo_vgl_in = pq.qmckl_get_mo_basis_mo_vgl_inplace(ctx, size_max)
assert rc==0
end = time.clock_gettime_ns(time.CLOCK_REALTIME)
print(f'Time for the calculation of 1 step : {(end-start)*.000001/ITERMAX} ms')