From 79d374bdffd319ab7a017804f58afb38b13646e5 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 19 Jul 2021 17:58:50 +0200 Subject: [PATCH 001/107] [WIP] Python interface generated by SWIG --- Makefile.am | 10 ++++++ src/pytrexio.i | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 src/pytrexio.i diff --git a/Makefile.am b/Makefile.am index ed7067d..9fab4fd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -179,6 +179,16 @@ cppcheck.out: $(trexio_h) --language=c --std=c99 -rp --platform=unix64 \ -I../include *.c *.h 2>../$@ + +python: $(ORG_FILES) src/pytrexio.i + cd src/ && \ + swig -python -o pytrexio_wrap.c pytrexio.i && \ + $(CC) $(CPPFLAGS) -I/usr/include/python3.8 \ + -fPIC -c trexio.c trexio_hdf5.c trexio_text.c pytrexio_wrap.c && \ + $(CC) -shared trexio.o trexio_hdf5.o trexio_text.o pytrexio_wrap.o \ + -L/usr/lib/x86_64-linux-gnu/hdf5/serial/ -lz -lm -lhdf5 -lhdf5_hl -o _pytrexio.so + + .PHONY: cppcheck endif diff --git a/src/pytrexio.i b/src/pytrexio.i new file mode 100644 index 0000000..cc88263 --- /dev/null +++ b/src/pytrexio.i @@ -0,0 +1,89 @@ +%module pytrexio +/* Define SWIGWORDSIZE in order to properly align long integers on 64-bit system */ +#define SWIGWORDSIZE64 +%{ +/* Include the headers in the wrapper code */ +#include "trexio.h" +#include "trexio_s.h" +#include "trexio_private.h" +#include "trexio_text.h" +#include "trexio_hdf5.h" +%} + +/* Include stdint to recognize types from stdint.h */ +%include +/* Include carrays to work with C-like arrays */ +%include "carrays.i" +/* Include classes that correspond to integer and float arrays */ +%array_class(double, doubleArray); +%array_class(float, floatArray); +%array_class(int32_t, int32Array); +%array_class(int64_t, int64Array); +/* Include typemaps to play with input/output re-casting + Useful when working with C pointers +*/ +%include typemaps.i +/* Redefine the int32_t* and int64_t* num to be output + Useful for TREXIO read_num functions where the + num variable is modified by address +*/ +%apply int *OUTPUT { int32_t* const num}; +%apply int *OUTPUT { int64_t* const num}; +/* Does not work for arrays (SIGSEGV) +%apply double *OUTPUT { double* const dataset }; +*/ +/* TODO: Redefine trexio_exit_code and back_end_t types to gain access to the list + of back ends and exit codes. Currently: hard-coded back ends +*/ +/*%typemap(out) back_end_t { + // $1 is what we got from our C or C++ call + $result = PyInt_FromLong((long) $1); + // $result is what gets given back to Python and we are responsible for setting it +}*/ +%constant int TREXIO_HDF5 = 0; +%constant int TREXIO_TEXT = 1; + +/* This tells SWIG to treat char ** as a special case */ +%typemap(in) char ** { + /* Check if is a list */ + if (PyList_Check($input)) { + int size = PyList_Size($input); + Py_ssize_t i = 0; + $1 = (char **) malloc((size+1)*sizeof(char *)); + for (i = 0; i < size; i++) { + PyObject *o = PyList_GetItem($input, i); + if (PyUnicode_Check(o)) { + $1[i] = PyUnicode_AsUTF8(PyList_GetItem($input,i)); + } else { + PyErr_Format(PyExc_TypeError, "list must contain strings. %d/%d element was not string.", i, size); + free($1); + return NULL; + } + } + $1[i] = 0; + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + return NULL; + } +} +/* This cleans up the char ** array we malloc-ed before */ +%typemap(freearg) char ** { + free((char *) $1); +} +/* Now a test function +%inline %{ +int print_args(char ** argv) { + int i = 0; + while (argv[i]) { + printf("argv[%d] = %s\n", i, argv[i]); + i++; + } + return i; +} +%}*/ +/* Parse the header files to generate wrappers */ +%include "trexio.h" +%include "trexio_s.h" +%include "trexio_private.h" +%include "trexio_text.h" +%include "trexio_hdf5.h" From d527561440599a6e8d5604b46de6ebd185d15743 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 19 Jul 2021 17:59:26 +0200 Subject: [PATCH 002/107] add test of basic I/O for Python --- src/test.py | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/test.py diff --git a/src/test.py b/src/test.py new file mode 100644 index 0000000..147af1a --- /dev/null +++ b/src/test.py @@ -0,0 +1,110 @@ +import os +import shutil + +from pytrexio import * + +#=========================================================# +#======== SETUP THE BACK END AND OUTPUT FILE NAME ========# +#=========================================================# + +# 0: TREXIO_HDF5 ; 1: TREXIO_TEXT +TEST_TREXIO_BACKEND = 1 +OUTPUT_FILENAME_TEXT = 'test_py_swig.dir' +OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' + + +if TEST_TREXIO_BACKEND == 0: + output_filename = OUTPUT_FILENAME_HDF5 +elif TEST_TREXIO_BACKEND == 1: + output_filename = OUTPUT_FILENAME_TEXT +else: + raise ValueError ('Specify one of the supported back end as TEST_TREXIO_BACKEND') + + +try: + if TEST_TREXIO_BACKEND == 0: + os.remove(output_filename) + if TEST_TREXIO_BACKEND == 1: + shutil.rmtree(output_filename) +except: + print (f'Test file {OUTPUT_FILENAME} does not exist') + +#=========================================================# +#============ WRITE THE DATA IN THE TEST FILE ============# +#=========================================================# + +test_file = trexio_open(output_filename, 'w', TEST_TREXIO_BACKEND) + +nucleus_num = 12 + +rc = trexio_write_nucleus_num(test_file, nucleus_num) +assert rc==0 + +charges = doubleArray(nucleus_num) +for i in range(nucleus_num): + if i < nucleus_num/2: + charges[i] = 6. + else: + charges[i] = 1. + +rc = trexio_write_nucleus_charge(test_file, charges) +assert rc==0 + +point_group = 'B3U' + +rc = trexio_write_nucleus_point_group(test_file, point_group, 10) +assert rc==0 + +labels = [ + 'C', + 'C', + 'C', + 'C', + 'C', + 'C', + 'H', + 'H', + 'H', + 'H', + 'H', + 'H'] + +rc = trexio_write_nucleus_label(test_file, labels, 10) +assert rc==0 + +rc = trexio_close(test_file) +assert rc==0 + +#==========================================================# +#============ READ THE DATA FROM THE TEST FILE ============# +#==========================================================# + +test_file2 = trexio_open(output_filename, 'r', TEST_TREXIO_BACKEND) + +result = trexio_read_nucleus_num(test_file2) +assert result[0]==0 +assert result[1]==nucleus_num +#print(result) + +charges2 = doubleArray(nucleus_num) +for i in range(nucleus_num): + charges2[i] = -1. + +rc = trexio_read_nucleus_charge(test_file2, charges2) +assert rc==0 +for i in range(nucleus_num): + assert charges2[i]==charges[i] + +labels2 = ['' for i in range(nucleus_num)] +print(labels2) +rc = trexio_read_nucleus_label(test_file2, labels2, 10) +print(labels2) +assert rc==0 +for i in range(nucleus_num): + assert labels2[i]==labels[i] + +rc = trexio_close(test_file2) +assert rc==0 + +#==========================================================# + From 9fa2bc28351b8a5a11f19ec8649cb57a4ba7a00e Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 10:39:05 +0200 Subject: [PATCH 003/107] disable type casting in constant definitions to access them in the target language required because SWIG cannot handle definitions with type casting --- src/templates_front/templator_front.org | 58 ++++++++++++------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index d619fb6..cb8241a 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -159,7 +159,7 @@ typedef int32_t trexio_exit_code; result = [ "#+begin_src c :tangle prefix_front.h :exports none" ] for (text, code,_) in table: text=text.replace("~","") - result += [ f"#define {text:30s} ((trexio_exit_code) {code:d})" ] + result += [ f"#define {text:30s} {code:d} //((trexio_exit_code) {code:d})" ] result += [ "#+end_src" ] result += [ "" ] @@ -178,31 +178,31 @@ return '\n'.join(result) #+RESULTS: :results: #+begin_src c :tangle prefix_front.h - #define TREXIO_FAILURE ((trexio_exit_code) -1) - #define TREXIO_SUCCESS ((trexio_exit_code) 0) - #define TREXIO_INVALID_ARG_1 ((trexio_exit_code) 1) - #define TREXIO_INVALID_ARG_2 ((trexio_exit_code) 2) - #define TREXIO_INVALID_ARG_3 ((trexio_exit_code) 3) - #define TREXIO_INVALID_ARG_4 ((trexio_exit_code) 4) - #define TREXIO_INVALID_ARG_5 ((trexio_exit_code) 5) - #define TREXIO_END ((trexio_exit_code) 6) - #define TREXIO_READONLY ((trexio_exit_code) 7) - #define TREXIO_ERRNO ((trexio_exit_code) 8) - #define TREXIO_INVALID_ID ((trexio_exit_code) 9) - #define TREXIO_ALLOCATION_FAILED ((trexio_exit_code) 10) - #define TREXIO_HAS_NOT ((trexio_exit_code) 11) - #define TREXIO_INVALID_NUM ((trexio_exit_code) 12) - #define TREXIO_ATTR_ALREADY_EXISTS ((trexio_exit_code) 13) - #define TREXIO_DSET_ALREADY_EXISTS ((trexio_exit_code) 14) - #define TREXIO_OPEN_ERROR ((trexio_exit_code) 15) - #define TREXIO_LOCK_ERROR ((trexio_exit_code) 16) - #define TREXIO_UNLOCK_ERROR ((trexio_exit_code) 17) - #define TREXIO_FILE_ERROR ((trexio_exit_code) 18) - #define TREXIO_GROUP_READ_ERROR ((trexio_exit_code) 19) - #define TREXIO_GROUP_WRITE_ERROR ((trexio_exit_code) 20) - #define TREXIO_ELEM_READ_ERROR ((trexio_exit_code) 21) - #define TREXIO_ELEM_WRITE_ERROR ((trexio_exit_code) 22) - #define TREXIO_INVALID_STR_LEN ((trexio_exit_code) 30) + #define TREXIO_FAILURE -1 //((trexio_exit_code) -1) + #define TREXIO_SUCCESS 0 //((trexio_exit_code) 0) + #define TREXIO_INVALID_ARG_1 1 //((trexio_exit_code) 1) + #define TREXIO_INVALID_ARG_2 2 //((trexio_exit_code) 2) + #define TREXIO_INVALID_ARG_3 3 //((trexio_exit_code) 3) + #define TREXIO_INVALID_ARG_4 4 //((trexio_exit_code) 4) + #define TREXIO_INVALID_ARG_5 5 //((trexio_exit_code) 5) + #define TREXIO_END 6 //((trexio_exit_code) 6) + #define TREXIO_READONLY 7 //((trexio_exit_code) 7) + #define TREXIO_ERRNO 8 //((trexio_exit_code) 8) + #define TREXIO_INVALID_ID 9 //((trexio_exit_code) 9) + #define TREXIO_ALLOCATION_FAILED 10 //((trexio_exit_code) 10) + #define TREXIO_HAS_NOT 11 //((trexio_exit_code) 11) + #define TREXIO_INVALID_NUM 12 //((trexio_exit_code) 12) + #define TREXIO_ATTR_ALREADY_EXISTS 13 //((trexio_exit_code) 13) + #define TREXIO_DSET_ALREADY_EXISTS 14 //((trexio_exit_code) 14) + #define TREXIO_OPEN_ERROR 15 //((trexio_exit_code) 15) + #define TREXIO_LOCK_ERROR 16 //((trexio_exit_code) 16) + #define TREXIO_UNLOCK_ERROR 17 //((trexio_exit_code) 17) + #define TREXIO_FILE_ERROR 18 //((trexio_exit_code) 18) + #define TREXIO_GROUP_READ_ERROR 19 //((trexio_exit_code) 19) + #define TREXIO_GROUP_WRITE_ERROR 20 //((trexio_exit_code) 20) + #define TREXIO_ELEM_READ_ERROR 21 //((trexio_exit_code) 21) + #define TREXIO_ELEM_WRITE_ERROR 22 //((trexio_exit_code) 22) + #define TREXIO_INVALID_STR_LEN 30 //((trexio_exit_code) 30) #+end_src #+begin_src f90 :tangle prefix_fortran.f90 @@ -389,10 +389,10 @@ end interface #+begin_src c :tangle prefix_front.h typedef int32_t back_end_t; -#define TREXIO_HDF5 ( (back_end_t) 0 ) -#define TREXIO_TEXT ( (back_end_t) 1 ) +#define TREXIO_HDF5 0 //( (back_end_t) 0 ) +#define TREXIO_TEXT 1 //( (back_end_t) 1 ) /*#define TREXIO_JSON ( (back_end_t) 2 )*/ -#define TREXIO_INVALID_BACK_END ( (back_end_t) 2 ) +#define TREXIO_INVALID_BACK_END 2 //( (back_end_t) 2 ) #define TREXIO_DELIM "\n" #+end_src From e06e0b374548ac3796472a25c1aa4455b154c0e1 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 10:42:25 +0200 Subject: [PATCH 004/107] commented brute force patch for re-defining constants in the target language --- src/pytrexio.i | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index cc88263..e6ed4ce 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -32,17 +32,13 @@ /* Does not work for arrays (SIGSEGV) %apply double *OUTPUT { double* const dataset }; */ -/* TODO: Redefine trexio_exit_code and back_end_t types to gain access to the list - of back ends and exit codes. Currently: hard-coded back ends +/* TREXIO back ends and exit codes can be redefines in the SWIG target language using %constant. + TREXIO back ends and exit codes can be redefines in the SWIG target languageProbably not a good idea since this overwrite existing macros definitions. */ -/*%typemap(out) back_end_t { - // $1 is what we got from our C or C++ call - $result = PyInt_FromLong((long) $1); - // $result is what gets given back to Python and we are responsible for setting it -}*/ +/* %constant int TREXIO_HDF5 = 0; %constant int TREXIO_TEXT = 1; - +*/ /* This tells SWIG to treat char ** as a special case */ %typemap(in) char ** { /* Check if is a list */ From 88e777f68b7aa737d46a4ba65f099c33daa4d76f Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 11:36:23 +0200 Subject: [PATCH 005/107] resolve complaint about undefined TREXIO_MAX_FILENAME_LENGTH and remove unused assert header --- src/templates_front/templator_front.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index cb8241a..817d687 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -72,8 +72,8 @@ typedef int32_t trexio_exit_code; #define _TREXIO_S_H #include "trexio.h" +#include "trexio_private.h" #include -#include #include #+end_src From 6f8279ed80f94e28ab03969a152811a2723f0f5e Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 11:40:13 +0200 Subject: [PATCH 006/107] improved make python rule --- Makefile.am | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9fab4fd..d4f7b44 100644 --- a/Makefile.am +++ b/Makefile.am @@ -180,13 +180,15 @@ cppcheck.out: $(trexio_h) -I../include *.c *.h 2>../$@ -python: $(ORG_FILES) src/pytrexio.i +python: $(ORG_FILES) $(trexio_h) src/pytrexio.i + cp $(trexio_h) src/ cd src/ && \ swig -python -o pytrexio_wrap.c pytrexio.i && \ - $(CC) $(CPPFLAGS) -I/usr/include/python3.8 \ - -fPIC -c trexio.c trexio_hdf5.c trexio_text.c pytrexio_wrap.c && \ + $(CC) $(CPPFLAGS) -I/usr/include/python3.8 -fPIC -Wno-discarded-qualifiers \ + -c trexio.c trexio_hdf5.c trexio_text.c pytrexio_wrap.c && \ $(CC) -shared trexio.o trexio_hdf5.o trexio_text.o pytrexio_wrap.o \ - -L/usr/lib/x86_64-linux-gnu/hdf5/serial/ -lz -lm -lhdf5 -lhdf5_hl -o _pytrexio.so + $(LDFLAGS) $(LIBS) -o _pytrexio.so + rm -f -- src/trexio.h .PHONY: cppcheck From ac281106965b56619723bb315ee49d5a1539451b Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 12:06:44 +0200 Subject: [PATCH 007/107] add include guard to the trexio_private.h file --- src/templates_front/templator_front.org | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 817d687..73f2046 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -77,6 +77,12 @@ typedef int32_t trexio_exit_code; #include #+end_src + #+begin_src c :tangle trexio_private.h :noweb yes +<
> +#ifndef _TREXIO_PRIVATE_H +#define _TREXIO_PRIVATE_H + #+end_src + * Coding conventions - integer types will be defined using types given in ~stdint.h~ @@ -2231,6 +2237,10 @@ contains #endif #+end_src + #+begin_src c :tangle trexio_private.h +#endif + #+end_src + #+begin_src f90 :tangle suffix_fortran.f90 end module trexio #+end_src From 46eec8977469b863bd62d5cec55e188c2c036f4d Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 12:08:18 +0200 Subject: [PATCH 008/107] remove unsafe and unused file_name field from trexio_hdf5_s --- src/templates_hdf5/templator_hdf5.org | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/templates_hdf5/templator_hdf5.org b/src/templates_hdf5/templator_hdf5.org index 7550c39..6d16cf9 100644 --- a/src/templates_hdf5/templator_hdf5.org +++ b/src/templates_hdf5/templator_hdf5.org @@ -64,9 +64,8 @@ #+begin_src c :tangle struct_hdf5.h typedef struct trexio_hdf5_s { trexio_t parent ; - hid_t file_id; - hid_t $group$_group; - const char* file_name; + hid_t file_id; + hid_t $group$_group; } trexio_hdf5_t; #+end_src From 678f90f86af9135eec800fa4d56246829125782f Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 12:31:21 +0200 Subject: [PATCH 009/107] add a note on alternative way to re-define ill-defined constants --- src/pytrexio.i | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index e6ed4ce..ae607b9 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -32,12 +32,14 @@ /* Does not work for arrays (SIGSEGV) %apply double *OUTPUT { double* const dataset }; */ -/* TREXIO back ends and exit codes can be redefines in the SWIG target language using %constant. - TREXIO back ends and exit codes can be redefines in the SWIG target languageProbably not a good idea since this overwrite existing macros definitions. +/* TREXIO back ends and exit codes can be redefined in the SWIG target language using %ignore and further #define statements + (instead of disabling the type cast in the trexio.h file) */ /* -%constant int TREXIO_HDF5 = 0; -%constant int TREXIO_TEXT = 1; +%ignore TREXIO_HDF5; // Ignore a macro in the header file +%ignore TREXIO_TEST; // Ignore a macro in the header file +#define TREXIO_HDF5 0 +#define TREXIO_TEXT 0 */ /* This tells SWIG to treat char ** as a special case */ %typemap(in) char ** { From 836603fd32960d915cbd137c26b927e889213d76 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 14:22:54 +0200 Subject: [PATCH 010/107] do no wrap-up the code from back ends and private header --- src/pytrexio.i | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index ae607b9..f928654 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -3,11 +3,8 @@ #define SWIGWORDSIZE64 %{ /* Include the headers in the wrapper code */ -#include "trexio.h" #include "trexio_s.h" -#include "trexio_private.h" -#include "trexio_text.h" -#include "trexio_hdf5.h" +#include "trexio.h" %} /* Include stdint to recognize types from stdint.h */ @@ -80,8 +77,5 @@ int print_args(char ** argv) { } %}*/ /* Parse the header files to generate wrappers */ -%include "trexio.h" %include "trexio_s.h" -%include "trexio_private.h" -%include "trexio_text.h" -%include "trexio_hdf5.h" +%include "trexio.h" From 4b651dbe1f958eecbabb2d7e208c7bf837085409 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 20 Jul 2021 14:57:48 +0200 Subject: [PATCH 011/107] better make for Python --- Makefile.am | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index d4f7b44..c7b2f1e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -180,7 +180,13 @@ cppcheck.out: $(trexio_h) -I../include *.c *.h 2>../$@ -python: $(ORG_FILES) $(trexio_h) src/pytrexio.i +python: src/_pytrexio.so + +python-test: src/test.py src/_pytrexio.so + cd src/ && python3 test.py + $(RM) -r -- src/__pycache__ + +src/_pytrexio.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i cp $(trexio_h) src/ cd src/ && \ swig -python -o pytrexio_wrap.c pytrexio.i && \ @@ -188,10 +194,12 @@ python: $(ORG_FILES) $(trexio_h) src/pytrexio.i -c trexio.c trexio_hdf5.c trexio_text.c pytrexio_wrap.c && \ $(CC) -shared trexio.o trexio_hdf5.o trexio_text.o pytrexio_wrap.o \ $(LDFLAGS) $(LIBS) -o _pytrexio.so - rm -f -- src/trexio.h + $(RM) -- src/trexio.h -.PHONY: cppcheck +CLEANFILES += src/pytrexio_wrap.c src/pytrexio.py src/_pytrexio.so + +.PHONY: cppcheck python python-test endif From 8c6289a8bae512aed10db7a7a75a984e52fb181e Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 26 Jul 2021 18:43:15 +0200 Subject: [PATCH 012/107] build Python module using pythonic distutils --- Makefile.am | 31 ++++++++++++++++++++++--------- src/setup.py | 27 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 src/setup.py diff --git a/Makefile.am b/Makefile.am index c7b2f1e..bc3e06e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -180,24 +180,37 @@ cppcheck.out: $(trexio_h) -I../include *.c *.h 2>../$@ -python: src/_pytrexio.so +python: src/_pytrexio*.so -python-test: src/test.py src/_pytrexio.so +python-test: src/test.py src/_pytrexio*.so cd src/ && python3 test.py $(RM) -r -- src/__pycache__ -src/_pytrexio.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i +# Advanced compilation using Python-native distutils +# +# swig -python -threads pytrexio.i ----> Add thread support for all the interface +# +src/_pytrexio*.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i cp $(trexio_h) src/ cd src/ && \ - swig -python -o pytrexio_wrap.c pytrexio.i && \ - $(CC) $(CPPFLAGS) -I/usr/include/python3.8 -fPIC -Wno-discarded-qualifiers \ - -c trexio.c trexio_hdf5.c trexio_text.c pytrexio_wrap.c && \ - $(CC) -shared trexio.o trexio_hdf5.o trexio_text.o pytrexio_wrap.o \ - $(LDFLAGS) $(LIBS) -o _pytrexio.so + swig -python -py3 -o pytrexio_wrap.c pytrexio.i && \ + python3 setup.py build_ext --inplace --swig-opts="-modern" $(RM) -- src/trexio.h +# Manual compilation of Python module +# +#src/_pytrexio.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i +# cp $(trexio_h) src/ +# cd src/ && \ +# swig -python -py3 -o pytrexio_wrap.c pytrexio.i && \ +# $(CC) $(CPPFLAGS) -I/usr/include/python3.8 -fPIC -Wno-discarded-qualifiers \ +# -c trexio.c trexio_hdf5.c trexio_text.c pytrexio_wrap.c && \ +# $(CC) -shared trexio.o trexio_hdf5.o trexio_text.o pytrexio_wrap.o \ +# $(LDFLAGS) $(LIBS) -o _pytrexio.so +# $(RM) -- src/trexio.h +# -CLEANFILES += src/pytrexio_wrap.c src/pytrexio.py src/_pytrexio.so +CLEANFILES += src/pytrexio_wrap.c src/pytrexio.py src/_pytrexio*.so .PHONY: cppcheck python python-test diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..ca9335d --- /dev/null +++ b/src/setup.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +""" +setup.py file for pytrexio +""" + +from distutils.core import setup, Extension + + +pytrexio_module = Extension('_pytrexio', + sources=['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'], + include_dirs=['/usr/include/hdf5/serial'], + #runtime_library_dirs=['/usr/lib/x86_64-linux-gnu/hdf5/serial'], + libraries=['hdf5', 'hdf5_hl'], + #extra_compile_args=['-I/usr/include/hdf5/serial'], + extra_link_args=['-L/usr/lib/x86_64-linux-gnu/hdf5/serial'] + ) + +setup (name = 'pytrexio', + version = '0.1', + author = "TREX-CoE", + description = """Python API of the TREXIO library""", + ext_modules = [pytrexio_module], + py_modules = ["pytrexio"], + url='https://github.com/TREX-CoE/trexio', + packages=['distutils', 'distutils.command'], + ) From 12c2a3a01412f729c559ad409de12a4fc34a2a7a Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 26 Jul 2021 18:43:49 +0200 Subject: [PATCH 013/107] update gitignore --- .gitignore | 5 +++++ src/.gitignore | 13 +++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 859e3f8..1347eb2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,11 @@ */*.la */*.lo aclocal.m4 +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 autom4te.cache/ compile config.guess diff --git a/src/.gitignore b/src/.gitignore index c485d0b..2fb239d 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -15,20 +15,21 @@ templates_text/*.h templates_text/*.dump templates_text/populated/ -libtrexio.so +*.so *.o *.h *.dump *.cppcheck + trexio.c trexio_text.c trexio_hdf5.c trexio_f.f90 trexio.mod -test_c -test_f -*.h5 -trexio_test/ -trexio_test_fort/ +pytrexio.py +pytrexio_wrap.c + +*.h5 +*.dir/ From 1f98604d58c19abab8c17fc249dd8f63b482c13c Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 26 Jul 2021 18:49:26 +0200 Subject: [PATCH 014/107] add pattern matching for arrays of strings + comments --- src/pytrexio.i | 74 +++++++++++++++++++------ src/templates_front/templator_front.org | 38 ++++++------- src/test.py | 15 +++-- 3 files changed, 86 insertions(+), 41 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index f928654..c8eb1c8 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -2,6 +2,7 @@ /* Define SWIGWORDSIZE in order to properly align long integers on 64-bit system */ #define SWIGWORDSIZE64 %{ +#define SWIG_FILE_WITH_INIT /* Include the headers in the wrapper code */ #include "trexio_s.h" #include "trexio.h" @@ -9,7 +10,7 @@ /* Include stdint to recognize types from stdint.h */ %include -/* Include carrays to work with C-like arrays */ +/* Include carrays to work with C pointers to arrays */ %include "carrays.i" /* Include classes that correspond to integer and float arrays */ %array_class(double, doubleArray); @@ -20,6 +21,7 @@ Useful when working with C pointers */ %include typemaps.i + /* Redefine the int32_t* and int64_t* num to be output Useful for TREXIO read_num functions where the num variable is modified by address @@ -29,8 +31,14 @@ /* Does not work for arrays (SIGSEGV) %apply double *OUTPUT { double* const dataset }; */ -/* TREXIO back ends and exit codes can be redefined in the SWIG target language using %ignore and further #define statements - (instead of disabling the type cast in the trexio.h file) +/* This enables access to trexio_[...]_read_dset_str_low set of functions + in order to return one long string with TREXIO_DELIM delimeter as 2-nd argument of output tuple + */ +%include +%cstring_bounded_output(char* dset_out, 1024); + +/* [WIP] TREXIO back ends and exit codes can be redefined in the SWIG target language + using %ignore and further #define statements (instead of disabling the type cast in the trexio.h file) */ /* %ignore TREXIO_HDF5; // Ignore a macro in the header file @@ -38,8 +46,11 @@ #define TREXIO_HDF5 0 #define TREXIO_TEXT 0 */ -/* This tells SWIG to treat char ** as a special case */ -%typemap(in) char ** { +/* This tells SWIG to treat char ** dset_in pattern as a special case + Enables access to trexio_[...]_write_dset_str set of functions directly, i.e. + by converting input list of strings from Python into char ** of C +*/ +%typemap(in) char ** dset_in { /* Check if is a list */ if (PyList_Check($input)) { int size = PyList_Size($input); @@ -62,20 +73,51 @@ } } /* This cleans up the char ** array we malloc-ed before */ -%typemap(freearg) char ** { +%typemap(freearg) char ** dset_in { free((char *) $1); } -/* Now a test function -%inline %{ -int print_args(char ** argv) { - int i = 0; - while (argv[i]) { - printf("argv[%d] = %s\n", i, argv[i]); - i++; - } - return i; + +/* [WIP] This is an attempt to make SWIG treat char ** dset_out as a special case + In order to return list of string to Python from C-native char ** dset_out, + which is modified (but not allocated) within the trexio_[...}read_dset_str function +*/ +%typemap(in, numinputs=0) char ** dset_out (char * temp) { + /*temp = (char *) malloc(1028*sizeof(char));*/ + $1 = &temp; } -%}*/ + +%typemap(argout) char ** dset_out { + + Py_ssize_t i = 0; + Py_ssize_t mysize = 12; + PyObject *o_res = PyList_New(mysize); + PyObject *o; + for (i = 0; i < mysize; i++) { + //printf("%s\n", $1[i]); + o = PyString_FromString($1[i]); + PyList_SetItem(o_res, i, o); + } + + PyObject *o2, *o3; + + if ((!$result) || ($result == Py_None)) { + $result = o_res; + } else { + if (!PyTuple_Check($result)) { + PyObject *o2 = $result; + $result = PyTuple_New(1); + PyTuple_SetItem($result, 0, o2); + } + o3 = PyTuple_New(1); + PyTuple_SetItem(o3, 0, o_res); + o2 = $result; + $result = PySequence_Concat(o2, o3); + Py_DECREF(o2); + Py_DECREF(o3); + Py_DECREF(o_res); + } +} + /* Parse the header files to generate wrappers */ %include "trexio_s.h" %include "trexio.h" diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 378d4b1..be10704 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -1643,19 +1643,19 @@ trexio_read_chunk_ao_2e_int_eri_value_64(trexio_t* const file, #+begin_src c :tangle hrw_dset_str_front.h :exports none trexio_exit_code trexio_has_$group_dset$(trexio_t* const file); -trexio_exit_code trexio_read_$group_dset$_low(trexio_t* const file, char* dset, const uint32_t max_str_len); -trexio_exit_code trexio_write_$group_dset$_low(trexio_t* const file, const char* dset, const uint32_t max_str_len); -trexio_exit_code trexio_read_$group_dset$(trexio_t* const file, char** dset, const uint32_t max_str_len); -trexio_exit_code trexio_write_$group_dset$(trexio_t* const file, const char** dset, const uint32_t max_str_len); +trexio_exit_code trexio_read_$group_dset$_low(trexio_t* const file, char* dset_out, const uint32_t max_str_len); +trexio_exit_code trexio_write_$group_dset$_low(trexio_t* const file, const char* dset_in, const uint32_t max_str_len); +trexio_exit_code trexio_read_$group_dset$(trexio_t* const file, char** dset_out, const uint32_t max_str_len); +trexio_exit_code trexio_write_$group_dset$(trexio_t* const file, const char** dset_in, const uint32_t max_str_len); #+end_src #+begin_src c :tangle read_dset_str_front.c trexio_exit_code -trexio_read_$group_dset$_low (trexio_t* const file, char* dset, const uint32_t max_str_len) +trexio_read_$group_dset$_low (trexio_t* const file, char* dset_out, const uint32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; - if (dset == NULL) return TREXIO_INVALID_ARG_2; + if (dset_out == NULL) return TREXIO_INVALID_ARG_2; if (max_str_len <= 0) return TREXIO_INVALID_ARG_3; trexio_exit_code rc; @@ -1673,15 +1673,15 @@ trexio_read_$group_dset$_low (trexio_t* const file, char* dset, const uint32_t m switch (file->back_end) { case TREXIO_TEXT: - return trexio_text_read_$group_dset$(file, dset, rank, dims, max_str_len); + return trexio_text_read_$group_dset$(file, dset_out, rank, dims, max_str_len); break; case TREXIO_HDF5: - return trexio_hdf5_read_$group_dset$(file, dset, rank, dims, max_str_len); + return trexio_hdf5_read_$group_dset$(file, dset_out, rank, dims, max_str_len); break; /* case TREXIO_JSON: - rc = trexio_json_read_$group_dset$(file, dset, rank, dims); + rc = trexio_json_read_$group_dset$(file, dset_out, rank, dims); break; ,*/ } @@ -1690,11 +1690,11 @@ trexio_read_$group_dset$_low (trexio_t* const file, char* dset, const uint32_t m } trexio_exit_code -trexio_read_$group_dset$ (trexio_t* const file, char** dset, const uint32_t max_str_len) +trexio_read_$group_dset$ (trexio_t* const file, char** dset_out, const uint32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; - if (dset == NULL) return TREXIO_INVALID_ARG_2; + if (dset_out == NULL) return TREXIO_INVALID_ARG_2; if (max_str_len <= 0) return TREXIO_INVALID_ARG_3; assert(file->back_end < TREXIO_INVALID_BACK_END); @@ -1725,8 +1725,8 @@ trexio_read_$group_dset$ (trexio_t* const file, char** dset, const uint32_t max_ return TREXIO_FAILURE; } - strcpy(dset[i], ""); - strcat(dset[i], pch); + strcpy(dset_out[i], ""); + strcat(dset_out[i], pch); } @@ -1738,11 +1738,11 @@ trexio_read_$group_dset$ (trexio_t* const file, char** dset, const uint32_t max_ #+begin_src c :tangle write_dset_str_front.c trexio_exit_code -trexio_write_$group_dset$_low (trexio_t* const file, const char* dset, const uint32_t max_str_len) +trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const uint32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; - if (dset == NULL) return TREXIO_INVALID_ARG_2; + if (dset_in == NULL) return TREXIO_INVALID_ARG_2; if (max_str_len <= 0) return TREXIO_INVALID_ARG_3; if (trexio_has_$group_dset$(file) == TREXIO_SUCCESS) return TREXIO_DSET_ALREADY_EXISTS; @@ -1769,7 +1769,7 @@ trexio_write_$group_dset$_low (trexio_t* const file, const char* dset, const uin /* parse the string using strtok */ for(uint64_t i=0; i Date: Mon, 26 Jul 2021 19:27:45 +0200 Subject: [PATCH 015/107] replace some integers with TREXIO-native constants --- src/test.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/test.py b/src/test.py index ac7ae40..83ea513 100644 --- a/src/test.py +++ b/src/test.py @@ -8,23 +8,23 @@ from pytrexio import * #=========================================================# # 0: TREXIO_HDF5 ; 1: TREXIO_TEXT -TEST_TREXIO_BACKEND = 1 +TEST_TREXIO_BACKEND = TREXIO_TEXT OUTPUT_FILENAME_TEXT = 'test_py_swig.dir' OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' -if TEST_TREXIO_BACKEND == 0: +if TEST_TREXIO_BACKEND == TREXIO_HDF5: output_filename = OUTPUT_FILENAME_HDF5 -elif TEST_TREXIO_BACKEND == 1: +elif TEST_TREXIO_BACKEND == TREXIO_TEXT: output_filename = OUTPUT_FILENAME_TEXT else: - raise ValueError ('Specify one of the supported back end as TEST_TREXIO_BACKEND') + raise ValueError ('Specify one of the supported back ends as TEST_TREXIO_BACKEND') try: - if TEST_TREXIO_BACKEND == 0: + if TEST_TREXIO_BACKEND == TREXIO_HDF5: os.remove(output_filename) - if TEST_TREXIO_BACKEND == 1: + elif TEST_TREXIO_BACKEND == TREXIO_TEXT: shutil.rmtree(output_filename) except: print (f'Test file {output_filename} does not exist') @@ -38,7 +38,7 @@ test_file = trexio_open(output_filename, 'w', TEST_TREXIO_BACKEND) nucleus_num = 12 rc = trexio_write_nucleus_num(test_file, nucleus_num) -assert rc==0 +assert rc==TREXIO_SUCCESS charges = doubleArray(nucleus_num) for i in range(nucleus_num): @@ -48,12 +48,12 @@ for i in range(nucleus_num): charges[i] = 1. rc = trexio_write_nucleus_charge(test_file, charges) -assert rc==0 +assert rc==TREXIO_SUCCESS point_group = 'B3U' rc = trexio_write_nucleus_point_group(test_file, point_group, 10) -assert rc==0 +assert rc==TREXIO_SUCCESS labels = [ 'C', @@ -70,10 +70,10 @@ labels = [ 'H'] rc = trexio_write_nucleus_label(test_file, labels, 10) -assert rc==0 +assert rc==TREXIO_SUCCESS rc = trexio_close(test_file) -assert rc==0 +assert rc==TREXIO_SUCCESS #==========================================================# #============ READ THE DATA FROM THE TEST FILE ============# @@ -91,7 +91,7 @@ for i in range(nucleus_num): charges2[i] = -1. rc = trexio_read_nucleus_charge(test_file2, charges2) -assert rc==0 +assert rc==TREXIO_SUCCESS for i in range(nucleus_num): assert charges2[i]==charges[i] @@ -99,15 +99,15 @@ for i in range(nucleus_num): #rc, label_2d = trexio_read_nucleus_label(test_file2, 10) # [WIP]: currently only low-level routines (return one long string instead of an array of strings) work rc, labels_1d = trexio_read_nucleus_label_low(test_file2, 10) -assert rc==0 +assert rc==TREXIO_SUCCESS -labels_2d = [label for label in labels_1d.split('\n') if label] +labels_2d = [label for label in labels_1d.split(TREXIO_DELIM) if label] print(labels_2d) for i in range(nucleus_num): assert labels_2d[i]==labels[i] rc = trexio_close(test_file2) -assert rc==0 +assert rc==TREXIO_SUCCESS #==========================================================# From 171cf1662b7f1223de6f5457185b511d7cbcca8a Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 26 Jul 2021 19:28:56 +0200 Subject: [PATCH 016/107] ignore __pycache__ --- src/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/.gitignore b/src/.gitignore index 2fb239d..977c220 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -32,4 +32,4 @@ pytrexio_wrap.c *.h5 *.dir/ - +__pycache__/ From 9f4176a8b6b7f995ae5cc9e7e8a5982ecb53372a Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 27 Jul 2021 11:06:06 +0200 Subject: [PATCH 017/107] resolve major compiler warnings produced by distutils --- src/setup.py | 2 +- src/templates_front/templator_front.org | 12 +++++++----- src/templates_text/templator_text.org | 9 ++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/setup.py b/src/setup.py index ca9335d..901902a 100644 --- a/src/setup.py +++ b/src/setup.py @@ -12,7 +12,7 @@ pytrexio_module = Extension('_pytrexio', include_dirs=['/usr/include/hdf5/serial'], #runtime_library_dirs=['/usr/lib/x86_64-linux-gnu/hdf5/serial'], libraries=['hdf5', 'hdf5_hl'], - #extra_compile_args=['-I/usr/include/hdf5/serial'], + extra_compile_args=['-Wno-discarded-qualifiers'], extra_link_args=['-L/usr/lib/x86_64-linux-gnu/hdf5/serial'] ) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index be10704..eafc103 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -545,6 +545,10 @@ trexio_open(const char* file_name, const char mode, result->mode = mode; result->one_based = false; // Need to be flipped in Fortran interface int irc = pthread_mutex_init ( &(result->thread_lock), NULL); + if (irc != 0) { + free(result); + return NULL; + } assert (irc == 0); trexio_exit_code rc; @@ -1717,7 +1721,7 @@ trexio_read_$group_dset$ (trexio_t* const file, char** dset_out, const uint32_t } char * pch; - for (uint64_t i=0; i < dset_dim; i++) { + for (uint64_t i=0; i < (uint64_t) dset_dim; i++) { pch = i == 0 ? strtok(str_compiled, TREXIO_DELIM) : strtok(NULL, TREXIO_DELIM) ; if (pch == NULL) { @@ -1777,7 +1781,7 @@ trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const return TREXIO_FAILURE; } - pch_len = strlen(pch); + pch_len = strlen(pch) + 1; if (pch_len > max_str_len) { FREE(dset_str[0]); @@ -1837,7 +1841,7 @@ trexio_write_$group_dset$ (trexio_t* const file, const char** dset_in, const uin if (str_compiled == NULL) return TREXIO_ALLOCATION_FAILED; strcpy(str_compiled, ""); - for (uint64_t i=0; i < dset_dim; i++) { + for (uint64_t i=0; i < (uint64_t) dset_dim; i++) { strcat(str_compiled, dset_in[i]); strcat(str_compiled, TREXIO_DELIM); } @@ -1994,8 +1998,6 @@ trexio_read_$group_str$ (trexio_t* const file, char* const str, const uint32_t m if (str == NULL) return TREXIO_INVALID_ARG_2; if (max_str_len <= 0) return TREXIO_INVALID_ARG_3; - trexio_exit_code rc = TREXIO_FAILURE; - switch (file->back_end) { case TREXIO_TEXT: diff --git a/src/templates_text/templator_text.org b/src/templates_text/templator_text.org index aee64ba..27e0af7 100644 --- a/src/templates_text/templator_text.org +++ b/src/templates_text/templator_text.org @@ -456,8 +456,6 @@ trexio_text_read_$group$ (trexio_text_t* const file) } // END REPEAT GROUP_DSET_NUM - size_t tmp_len; - // START REPEAT GROUP_DSET_STR /* Allocate arrays */ $group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$); @@ -485,7 +483,8 @@ trexio_text_read_$group$ (trexio_text_t* const file) ,*/ char* tmp_$group_dset$; if(size_$group_dset$ != 0) tmp_$group_dset$ = CALLOC(size_$group_dset$*32, char); - tmp_len = 0; + + size_t tmp_$group_dset$_len = 0; for (uint64_t i=0 ; i$group_dset$[i] = tmp_$group_dset$; /* conventional fcanf with "%s" only return the string before the first space character @@ -501,8 +500,8 @@ trexio_text_read_$group$ (trexio_text_t* const file) return NULL; } - tmp_len = strlen($group$->$group_dset$[i]); - tmp_$group_dset$ += tmp_len + 1; + tmp_$group_dset$_len = strlen($group$->$group_dset$[i]); + tmp_$group_dset$ += tmp_$group_dset$_len + 1; } // END REPEAT GROUP_DSET_STR From da5a990acc74d355574e349da78662c1c13d759f Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 28 Jul 2021 10:19:00 +0200 Subject: [PATCH 018/107] add memory-safe API for numerical datasets --- src/templates_front/templator_front.org | 187 ++++++++++++++++++------ tests/io_safe_dset_float_hdf5.c | 165 +++++++++++++++++++++ tests/io_safe_dset_float_text.c | 165 +++++++++++++++++++++ 3 files changed, 471 insertions(+), 46 deletions(-) create mode 100644 tests/io_safe_dset_float_hdf5.c create mode 100644 tests/io_safe_dset_float_text.c diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index eafc103..f6d3072 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -152,6 +152,7 @@ typedef int32_t trexio_exit_code; | ~TREXIO_GROUP_WRITE_ERROR~ | 20 | 'Error writing group' | | ~TREXIO_ELEM_READ_ERROR~ | 21 | 'Error reading element' | | ~TREXIO_ELEM_WRITE_ERROR~ | 22 | 'Error writing element' | + | ~TREXIO_UNSAFE_ARRAY_DIM~ | 23 | 'Access to memory beyond allocated' | | ~TREXIO_INVALID_STR_LEN~ | 30 | 'Invalid max_str_len' | # We need to force Emacs not to indent the Python code: @@ -165,7 +166,7 @@ typedef int32_t trexio_exit_code; result = [ "#+begin_src c :tangle prefix_front.h :exports none" ] for (text, code,_) in table: text=text.replace("~","") - result += [ f"#define {text:30s} {code:d} //((trexio_exit_code) {code:d})" ] + result += [ f"#define {text:30s} {code:d} //((trexio_exit_code) {code:d})" ] result += [ "#+end_src" ] result += [ "" ] @@ -182,36 +183,37 @@ return '\n'.join(result) #+RESULTS: - :results: - #+begin_src c :tangle prefix_front.h - #define TREXIO_FAILURE -1 //((trexio_exit_code) -1) - #define TREXIO_SUCCESS 0 //((trexio_exit_code) 0) - #define TREXIO_INVALID_ARG_1 1 //((trexio_exit_code) 1) - #define TREXIO_INVALID_ARG_2 2 //((trexio_exit_code) 2) - #define TREXIO_INVALID_ARG_3 3 //((trexio_exit_code) 3) - #define TREXIO_INVALID_ARG_4 4 //((trexio_exit_code) 4) - #define TREXIO_INVALID_ARG_5 5 //((trexio_exit_code) 5) - #define TREXIO_END 6 //((trexio_exit_code) 6) - #define TREXIO_READONLY 7 //((trexio_exit_code) 7) - #define TREXIO_ERRNO 8 //((trexio_exit_code) 8) - #define TREXIO_INVALID_ID 9 //((trexio_exit_code) 9) - #define TREXIO_ALLOCATION_FAILED 10 //((trexio_exit_code) 10) - #define TREXIO_HAS_NOT 11 //((trexio_exit_code) 11) - #define TREXIO_INVALID_NUM 12 //((trexio_exit_code) 12) - #define TREXIO_ATTR_ALREADY_EXISTS 13 //((trexio_exit_code) 13) - #define TREXIO_DSET_ALREADY_EXISTS 14 //((trexio_exit_code) 14) - #define TREXIO_OPEN_ERROR 15 //((trexio_exit_code) 15) - #define TREXIO_LOCK_ERROR 16 //((trexio_exit_code) 16) - #define TREXIO_UNLOCK_ERROR 17 //((trexio_exit_code) 17) - #define TREXIO_FILE_ERROR 18 //((trexio_exit_code) 18) - #define TREXIO_GROUP_READ_ERROR 19 //((trexio_exit_code) 19) - #define TREXIO_GROUP_WRITE_ERROR 20 //((trexio_exit_code) 20) - #define TREXIO_ELEM_READ_ERROR 21 //((trexio_exit_code) 21) - #define TREXIO_ELEM_WRITE_ERROR 22 //((trexio_exit_code) 22) - #define TREXIO_INVALID_STR_LEN 30 //((trexio_exit_code) 30) + :RESULTS: + #+begin_src c :tangle prefix_front.h :exports none + #define TREXIO_FAILURE -1 //((trexio_exit_code) -1) + #define TREXIO_SUCCESS 0 //((trexio_exit_code) 0) + #define TREXIO_INVALID_ARG_1 1 //((trexio_exit_code) 1) + #define TREXIO_INVALID_ARG_2 2 //((trexio_exit_code) 2) + #define TREXIO_INVALID_ARG_3 3 //((trexio_exit_code) 3) + #define TREXIO_INVALID_ARG_4 4 //((trexio_exit_code) 4) + #define TREXIO_INVALID_ARG_5 5 //((trexio_exit_code) 5) + #define TREXIO_END 6 //((trexio_exit_code) 6) + #define TREXIO_READONLY 7 //((trexio_exit_code) 7) + #define TREXIO_ERRNO 8 //((trexio_exit_code) 8) + #define TREXIO_INVALID_ID 9 //((trexio_exit_code) 9) + #define TREXIO_ALLOCATION_FAILED 10 //((trexio_exit_code) 10) + #define TREXIO_HAS_NOT 11 //((trexio_exit_code) 11) + #define TREXIO_INVALID_NUM 12 //((trexio_exit_code) 12) + #define TREXIO_ATTR_ALREADY_EXISTS 13 //((trexio_exit_code) 13) + #define TREXIO_DSET_ALREADY_EXISTS 14 //((trexio_exit_code) 14) + #define TREXIO_OPEN_ERROR 15 //((trexio_exit_code) 15) + #define TREXIO_LOCK_ERROR 16 //((trexio_exit_code) 16) + #define TREXIO_UNLOCK_ERROR 17 //((trexio_exit_code) 17) + #define TREXIO_FILE_ERROR 18 //((trexio_exit_code) 18) + #define TREXIO_GROUP_READ_ERROR 19 //((trexio_exit_code) 19) + #define TREXIO_GROUP_WRITE_ERROR 20 //((trexio_exit_code) 20) + #define TREXIO_ELEM_READ_ERROR 21 //((trexio_exit_code) 21) + #define TREXIO_ELEM_WRITE_ERROR 22 //((trexio_exit_code) 22) + #define TREXIO_UNSAFE_ARRAY_DIM 23 //((trexio_exit_code) 23) + #define TREXIO_INVALID_STR_LEN 30 //((trexio_exit_code) 30) #+end_src - #+begin_src f90 :tangle prefix_fortran.f90 + #+begin_src f90 :tangle prefix_fortran.f90 :exports none integer(trexio_exit_code), parameter :: TREXIO_FAILURE = -1 integer(trexio_exit_code), parameter :: TREXIO_SUCCESS = 0 integer(trexio_exit_code), parameter :: TREXIO_INVALID_ARG_1 = 1 @@ -236,9 +238,10 @@ return '\n'.join(result) integer(trexio_exit_code), parameter :: TREXIO_GROUP_WRITE_ERROR = 20 integer(trexio_exit_code), parameter :: TREXIO_ELEM_READ_ERROR = 21 integer(trexio_exit_code), parameter :: TREXIO_ELEM_WRITE_ERROR = 22 + integer(trexio_exit_code), parameter :: TREXIO_UNSAFE_ARRAY_DIM = 23 integer(trexio_exit_code), parameter :: TREXIO_INVALID_STR_LEN = 30 #+end_src - :end: + :END: The ~trexio_string_of_error~ converts an exit code into a string. The string is assumed to be large enough to contain the error message @@ -269,9 +272,7 @@ result = [] for (text, code, message) in table: text = text.replace("~","") message = message.replace("'",'"') - result += [ f"""case {text}: - return {message}; - break;""" ] + result += [ f"""case {text}:\n return {message};\n break;""" ] return '\n'.join(result) #+end_src @@ -320,8 +321,11 @@ return '\n'.join(result) case TREXIO_INVALID_NUM: return "Invalid dimensions"; break; - case TREXIO_NUM_ALREADY_EXISTS: - return "Variable already exists"; + case TREXIO_ATTR_ALREADY_EXISTS: + return "Attribute (num/str) already exists"; + break; + case TREXIO_DSET_ALREADY_EXISTS: + return "Dataset already exists"; break; case TREXIO_OPEN_ERROR: return "Error opening file"; @@ -347,6 +351,12 @@ return '\n'.join(result) case TREXIO_ELEM_WRITE_ERROR: return "Error writing element"; break; + case TREXIO_UNSAFE_ARRAY_DIM: + return "Access to memory beyond allocated"; + break; + case TREXIO_INVALID_STR_LEN: + return "Invalid max_str_len"; + break; #+end_example # Source @@ -1113,15 +1123,17 @@ end interface This section concerns API calls related to datasets. - | Function name | Description | Precision | - |--------------------------------+-------------------------------------+-----------| - | ~trexio_has_$group_dset$~ | Check if a dataset exists in a file | --- | - | ~trexio_read_$group_dset$~ | Read a dataset | Double | - | ~trexio_write_$group_dset$~ | Write a dataset | Double | - | ~trexio_read_$group_dset$_32~ | Read a dataset | Single | - | ~trexio_write_$group_dset$_32~ | Write a dataset | Single | - | ~trexio_read_$group_dset$_64~ | Read a dataset | Double | - | ~trexio_write_$group_dset$_64~ | Write a dataset | Double | + | Function name | Description | Precision | + |----------------------------------+--------------------------------------+-----------------------------| + | ~trexio_has_$group_dset$~ | Check if a dataset exists in a file | --- | + | ~trexio_read_$group_dset$~ | Read a dataset in default precision | Double/Single for float/int | + | ~trexio_write_$group_dset$~ | Write a dataset in default precision | Double/Single for float/int | + | ~trexio_read_safe_$group_dset$~ | Read a bounded dataset | Double | + | ~trexio_write_safe_$group_dset$~ | Write a bounded dataset | Double | + | ~trexio_read_$group_dset$_32~ | Read a dataset in single precision | Single | + | ~trexio_write_$group_dset$_32~ | Write a dataset in single precision | Single | + | ~trexio_read_$group_dset$_64~ | Read a dataset in double precision | Double | + | ~trexio_write_$group_dset$_64~ | Write a dataset in double precision | Double | *** C templates for front end @@ -1131,6 +1143,8 @@ end interface Suffixes ~_32~ and ~_64~ correspond to API calls dealing with single and double precision, respectively. The basic (non-suffixed) API call on datasets deals with double precision (see Table above). +**** Function declarations + #+begin_src c :tangle hrw_dset_data_front.h :exports none trexio_exit_code trexio_has_$group_dset$(trexio_t* const file); @@ -1140,8 +1154,13 @@ trexio_exit_code trexio_read_$group_dset$_32(trexio_t* const file, $group_dset_d trexio_exit_code trexio_write_$group_dset$_32(trexio_t* const file, const $group_dset_dtype_single$* $group_dset$); trexio_exit_code trexio_read_$group_dset$_64(trexio_t* const file, $group_dset_dtype_double$* const $group_dset$); trexio_exit_code trexio_write_$group_dset$_64(trexio_t* const file, const $group_dset_dtype_double$* $group_dset$); +trexio_exit_code trexio_read_safe_$group_dset$(trexio_t* const file, $group_dset_dtype_default$* const dset_out, const uint64_t dim_out); +trexio_exit_code trexio_write_safe_$group_dset$(trexio_t* const file, const $group_dset_dtype_default$* dset_in, const uint64_t dim_in); #+end_src +**** Source code for double precision functions + + #+begin_src c :tangle read_dset_data_64_front.c trexio_exit_code trexio_read_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* const $group_dset$) @@ -1155,7 +1174,6 @@ trexio_read_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* co /* Error handling for this call is added by the generator */ rc = trexio_read_$group_dset_dim$_64(file, &($group_dset_dim$)); - if (rc != TREXIO_SUCCESS) return rc; if ($group_dset_dim$ == 0L) return TREXIO_INVALID_NUM; @@ -1197,6 +1215,7 @@ trexio_read_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* co } #+end_src + #+begin_src c :tangle write_dset_data_64_front.c trexio_exit_code trexio_write_$group_dset$_64 (trexio_t* const file, const $group_dset_dtype_double$* $group_dset$) @@ -1211,7 +1230,6 @@ trexio_write_$group_dset$_64 (trexio_t* const file, const $group_dset_dtype_doub /* Error handling for this call is added by the generator */ rc = trexio_read_$group_dset_dim$_64(file, &($group_dset_dim$)); - if (rc != TREXIO_SUCCESS) return rc; if ($group_dset_dim$ == 0L) return TREXIO_INVALID_NUM; @@ -1264,6 +1282,9 @@ trexio_write_$group_dset$_64 (trexio_t* const file, const $group_dset_dtype_doub } #+end_src +**** Source code for single precision functions + + #+begin_src c :tangle read_dset_data_32_front.c trexio_exit_code trexio_read_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_single$* const $group_dset$) @@ -1331,6 +1352,7 @@ trexio_read_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_single$* co } #+end_src + #+begin_src c :tangle write_dset_data_32_front.c trexio_exit_code trexio_write_$group_dset$_32 (trexio_t* const file, const $group_dset_dtype_single$* $group_dset$) @@ -1397,6 +1419,77 @@ trexio_write_$group_dset$_32 (trexio_t* const file, const $group_dset_dtype_sing } #+end_src +**** Source code for memory-safe functions + + #+begin_src c :tangle read_dset_data_safe_front.c +trexio_exit_code +trexio_read_safe_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const dset_out, const uint64_t dim_out) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + if (dset_out == NULL) return TREXIO_INVALID_ARG_2; + + trexio_exit_code rc; + int64_t $group_dset_dim$ = 0; + + /* Error handling for this call is added by the generator */ + rc = trexio_read_$group_dset_dim$_64(file, &($group_dset_dim$)); + + if ($group_dset_dim$ == 0L) return TREXIO_INVALID_NUM; + + uint32_t rank = $group_dset_rank$; + uint64_t dims[$group_dset_rank$] = {$group_dset_dim_list$}; + + /* The block below is specific to safe API as it checks the boundaries */ + uint64_t dim_size = 1; + for (uint32_t i=0; i dim_size) return TREXIO_UNSAFE_ARRAY_DIM; + /* */ + + return trexio_read_$group_dset$_$default_prec$(file, dset_out); +} + #+end_src + + + #+begin_src c :tangle write_dset_data_safe_front.c +trexio_exit_code +trexio_write_safe_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default$* dset_in, const uint64_t dim_in) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + if (dset_in == NULL) return TREXIO_INVALID_ARG_2; + if (trexio_has_$group_dset$(file) == TREXIO_SUCCESS) return TREXIO_DSET_ALREADY_EXISTS; + + trexio_exit_code rc; + int64_t $group_dset_dim$ = 0; + + /* Error handling for this call is added by the generator */ + rc = trexio_read_$group_dset_dim$_64(file, &($group_dset_dim$)); + + if ($group_dset_dim$ == 0L) return TREXIO_INVALID_NUM; + + uint32_t rank = $group_dset_rank$; + uint64_t dims[$group_dset_rank$] = {$group_dset_dim_list$}; + + /* The block below is specific to safe API as it checks the boundaries */ + uint64_t dim_size = 1; + for (uint32_t i=0; i dim_size) return TREXIO_UNSAFE_ARRAY_DIM; + /* */ + + return trexio_write_$group_dset$_$default_prec$(file, dset_in); + +} + #+end_src + +**** Source code for default functions + #+begin_src c :tangle read_dset_data_def_front.c trexio_exit_code trexio_read_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const $group_dset$) @@ -1405,6 +1498,7 @@ trexio_read_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* cons } #+end_src + #+begin_src c :tangle write_dset_data_def_front.c trexio_exit_code trexio_write_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default$* $group_dset$) @@ -1413,6 +1507,7 @@ trexio_write_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default } #+end_src + #+begin_src c :tangle has_dset_data_front.c trexio_exit_code trexio_has_$group_dset$ (trexio_t* const file) diff --git a/tests/io_safe_dset_float_hdf5.c b/tests/io_safe_dset_float_hdf5.c new file mode 100644 index 0000000..f70fad7 --- /dev/null +++ b/tests/io_safe_dset_float_hdf5.c @@ -0,0 +1,165 @@ +#include "trexio.h" +#include +#include +#include +#include + +#define TEST_BACKEND TREXIO_HDF5 +#define TREXIO_FILE "test_safe_dset_f.h5" +#define RM_COMMAND "rm -rf " TREXIO_FILE + +static int test_write_dset (const char* file_name, const back_end_t backend) { + +/* Try to write a dataset with floating point values into the TREXIO file using safe API */ + + trexio_t* file = NULL; + trexio_exit_code rc; + + // parameters to be written + int num = 12; + double coord[36] = { + 0.00000000 , 1.39250319 , 0.00000000 , + -1.20594314 , 0.69625160 , 0.00000000 , + -1.20594314 , -0.69625160 , 0.00000000 , + 0.00000000 , -1.39250319 , 0.00000000 , + 1.20594314 , -0.69625160 , 0.00000000 , + 1.20594314 , 0.69625160 , 0.00000000 , + -2.14171677 , 1.23652075 , 0.00000000 , + -2.14171677 , -1.23652075 , 0.00000000 , + 0.00000000 , -2.47304151 , 0.00000000 , + 2.14171677 , -1.23652075 , 0.00000000 , + 2.14171677 , 1.23652075 , 0.00000000 , + 0.00000000 , 2.47304151 , 0.00000000 , + }; + +/*================= START OF TEST ==================*/ + + // open file in 'write' mode + file = trexio_open(file_name, 'w', backend); + assert (file != NULL); + + // write numerical attribute in an empty file + rc = trexio_write_nucleus_num(file, num); + assert (rc == TREXIO_SUCCESS); + + /* write numerical dataset with an unsafe dimension + * this should return TREXIO_UNSAFE_ARRAY_DIM indicating + * that access beyong allocated memory is likely to occur */ + uint64_t dim_unsafe = num * 12; + rc = trexio_write_safe_nucleus_coord(file, coord, dim_unsafe); + assert (rc == TREXIO_UNSAFE_ARRAY_DIM); + + /* write numerical dataset with a safe dimension + * this should return TREXIO_SUCCESS */ + uint64_t dim_safe = num * 3; + rc = trexio_write_safe_nucleus_coord(file, coord, dim_safe); + assert (rc == TREXIO_SUCCESS); + + // close current session + rc = trexio_close(file); + assert (rc == TREXIO_SUCCESS); + +/*================= END OF TEST ==================*/ + + return 0; +} + + +static int test_has_dset (const char* file_name, const back_end_t backend) { + +/* Try to check the existence of a dataset in the TREXIO file */ + + trexio_t* file = NULL; + trexio_exit_code rc; + +/*================= START OF TEST ==================*/ + + // open file in 'read' mode + file = trexio_open(file_name, 'r', backend); + assert (file != NULL); + + // check that the previously written dataset exists + rc = trexio_has_nucleus_coord(file); + assert (rc == TREXIO_SUCCESS); + + // close current session + rc = trexio_close(file); + assert (rc == TREXIO_SUCCESS); + +/*================= END OF TEST ==================*/ + + return 0; +} + + +static int test_read_dset (const char* file_name, const back_end_t backend) { + +/* Try to read a dataset with floating point values from the TREXIO file using safe API */ + + trexio_t* file = NULL; + trexio_exit_code rc; + + // parameters to be read + int num; + double* coord; + +/*================= START OF TEST ==================*/ + + // open file in 'read' mode + file = trexio_open(file_name, 'r', backend); + assert (file != NULL); + + // read numerical attribute from the file + rc = trexio_read_nucleus_num(file, &num); + assert (rc == TREXIO_SUCCESS); + assert (num == 12); + + // read numerical (floating point) dataset from the file + coord = (double*) calloc(3*num, sizeof(double)); + + /* write numerical dataset with an unsafe dimension + * this should return TREXIO_UNSAFE_ARRAY_DIM indicating + * that access beyong allocated memory is likely to occur */ + uint64_t dim_unsafe = num * 12; + rc = trexio_read_safe_nucleus_coord(file, coord, dim_unsafe); + assert (rc == TREXIO_UNSAFE_ARRAY_DIM); + + /* write numerical dataset with a safe dimension + * this should return TREXIO_SUCCESS */ + uint64_t dim_safe = num * 3; + rc = trexio_read_safe_nucleus_coord(file, coord, dim_safe); + assert (rc == TREXIO_SUCCESS); + + double x = coord[30] - 2.14171677; + assert( x*x < 1.e-14 ); + free(coord); + + // close current session + rc = trexio_close(file); + assert (rc == TREXIO_SUCCESS); + +/*================= END OF TEST ==================*/ + + return 0; +} + + +int main(void) { + +/*============== Test launcher ================*/ + + int rc; + rc = system(RM_COMMAND); + assert (rc == 0); + + test_write_dset (TREXIO_FILE, TEST_BACKEND); + test_has_dset (TREXIO_FILE, TEST_BACKEND); + test_read_dset (TREXIO_FILE, TEST_BACKEND); + + rc = system(RM_COMMAND); + assert (rc == 0); + + return 0; +} + + diff --git a/tests/io_safe_dset_float_text.c b/tests/io_safe_dset_float_text.c new file mode 100644 index 0000000..709abd0 --- /dev/null +++ b/tests/io_safe_dset_float_text.c @@ -0,0 +1,165 @@ +#include "trexio.h" +#include +#include +#include +#include + +#define TEST_BACKEND TREXIO_TEXT +#define TREXIO_FILE "test_safe_dset_f.dir" +#define RM_COMMAND "rm -rf " TREXIO_FILE + +static int test_write_dset (const char* file_name, const back_end_t backend) { + +/* Try to write a dataset with floating point values into the TREXIO file using safe API */ + + trexio_t* file = NULL; + trexio_exit_code rc; + + // parameters to be written + int num = 12; + double coord[36] = { + 0.00000000 , 1.39250319 , 0.00000000 , + -1.20594314 , 0.69625160 , 0.00000000 , + -1.20594314 , -0.69625160 , 0.00000000 , + 0.00000000 , -1.39250319 , 0.00000000 , + 1.20594314 , -0.69625160 , 0.00000000 , + 1.20594314 , 0.69625160 , 0.00000000 , + -2.14171677 , 1.23652075 , 0.00000000 , + -2.14171677 , -1.23652075 , 0.00000000 , + 0.00000000 , -2.47304151 , 0.00000000 , + 2.14171677 , -1.23652075 , 0.00000000 , + 2.14171677 , 1.23652075 , 0.00000000 , + 0.00000000 , 2.47304151 , 0.00000000 , + }; + +/*================= START OF TEST ==================*/ + + // open file in 'write' mode + file = trexio_open(file_name, 'w', backend); + assert (file != NULL); + + // write numerical attribute in an empty file + rc = trexio_write_nucleus_num(file, num); + assert (rc == TREXIO_SUCCESS); + + /* write numerical dataset with an unsafe dimension + * this should return TREXIO_UNSAFE_ARRAY_DIM indicating + * that access beyong allocated memory is likely to occur */ + uint64_t dim_unsafe = num * 12; + rc = trexio_write_safe_nucleus_coord(file, coord, dim_unsafe); + assert (rc == TREXIO_UNSAFE_ARRAY_DIM); + + /* write numerical dataset with a safe dimension + * this should return TREXIO_SUCCESS */ + uint64_t dim_safe = num * 3; + rc = trexio_write_safe_nucleus_coord(file, coord, dim_safe); + assert (rc == TREXIO_SUCCESS); + + // close current session + rc = trexio_close(file); + assert (rc == TREXIO_SUCCESS); + +/*================= END OF TEST ==================*/ + + return 0; +} + + +static int test_has_dset (const char* file_name, const back_end_t backend) { + +/* Try to check the existence of a dataset in the TREXIO file */ + + trexio_t* file = NULL; + trexio_exit_code rc; + +/*================= START OF TEST ==================*/ + + // open file in 'read' mode + file = trexio_open(file_name, 'r', backend); + assert (file != NULL); + + // check that the previously written dataset exists + rc = trexio_has_nucleus_coord(file); + assert (rc == TREXIO_SUCCESS); + + // close current session + rc = trexio_close(file); + assert (rc == TREXIO_SUCCESS); + +/*================= END OF TEST ==================*/ + + return 0; +} + + +static int test_read_dset (const char* file_name, const back_end_t backend) { + +/* Try to read a dataset with floating point values from the TREXIO file using safe API */ + + trexio_t* file = NULL; + trexio_exit_code rc; + + // parameters to be read + int num; + double* coord; + +/*================= START OF TEST ==================*/ + + // open file in 'read' mode + file = trexio_open(file_name, 'r', backend); + assert (file != NULL); + + // read numerical attribute from the file + rc = trexio_read_nucleus_num(file, &num); + assert (rc == TREXIO_SUCCESS); + assert (num == 12); + + // read numerical (floating point) dataset from the file + coord = (double*) calloc(3*num, sizeof(double)); + + /* write numerical dataset with an unsafe dimension + * this should return TREXIO_UNSAFE_ARRAY_DIM indicating + * that access beyong allocated memory is likely to occur */ + uint64_t dim_unsafe = num * 12; + rc = trexio_read_safe_nucleus_coord(file, coord, dim_unsafe); + assert (rc == TREXIO_UNSAFE_ARRAY_DIM); + + /* write numerical dataset with a safe dimension + * this should return TREXIO_SUCCESS */ + uint64_t dim_safe = num * 3; + rc = trexio_read_safe_nucleus_coord(file, coord, dim_safe); + assert (rc == TREXIO_SUCCESS); + + double x = coord[30] - 2.14171677; + assert( x*x < 1.e-14 ); + free(coord); + + // close current session + rc = trexio_close(file); + assert (rc == TREXIO_SUCCESS); + +/*================= END OF TEST ==================*/ + + return 0; +} + + +int main(void) { + +/*============== Test launcher ================*/ + + int rc; + rc = system(RM_COMMAND); + assert (rc == 0); + + test_write_dset (TREXIO_FILE, TEST_BACKEND); + test_has_dset (TREXIO_FILE, TEST_BACKEND); + test_read_dset (TREXIO_FILE, TEST_BACKEND); + + rc = system(RM_COMMAND); + assert (rc == 0); + + return 0; +} + + From 5a3fbc2317d2dcfaedd552057af0abbe252cdc57 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 28 Jul 2021 10:19:45 +0200 Subject: [PATCH 019/107] compile tests for memory-safe API --- Makefile.am | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile.am b/Makefile.am index bc3e06e..a96cfda 100644 --- a/Makefile.am +++ b/Makefile.am @@ -87,6 +87,8 @@ TESTS_C = \ tests/io_dset_float_text \ tests/io_dset_int_hdf5 \ tests/io_dset_int_text \ + tests/io_safe_dset_float_hdf5 \ + tests/io_safe_dset_float_text \ tests/io_str_hdf5 \ tests/io_str_text \ tests/io_dset_str_hdf5 \ @@ -114,6 +116,8 @@ tests_io_dset_float_hdf5_LDFLAGS = -no-install tests_io_dset_float_text_LDFLAGS = -no-install tests_io_dset_int_hdf5_LDFLAGS = -no-install tests_io_dset_int_text_LDFLAGS = -no-install +tests_io_safe_dset_float_hdf5_LDFLAGS = -no-install +tests_io_safe_dset_float_text_LDFLAGS = -no-install tests_io_str_hdf5_LDFLAGS = -no-install tests_io_str_text_LDFLAGS = -no-install tests_io_dset_str_hdf5_LDFLAGS = -no-install From 5b7030fca1a5c719d82c90bf6a2801345da1a382 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 28 Jul 2021 10:30:23 +0200 Subject: [PATCH 020/107] [WIP] make a test release on TestPyPi currently does not link to existing setup.py --- Makefile.am | 8 +++++++- pyproject.toml | 7 +++++++ setup.cfg | 23 +++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 pyproject.toml create mode 100644 setup.cfg diff --git a/Makefile.am b/Makefile.am index a96cfda..de651ee 100644 --- a/Makefile.am +++ b/Makefile.am @@ -190,6 +190,12 @@ python-test: src/test.py src/_pytrexio*.so cd src/ && python3 test.py $(RM) -r -- src/__pycache__ +python-build: pyproject.toml setup.cfg + python3 -m build + +python-release: pyproject.toml setup.cfg + python3 -m twine upload --repository testpypi dist/* + # Advanced compilation using Python-native distutils # # swig -python -threads pytrexio.i ----> Add thread support for all the interface @@ -216,7 +222,7 @@ src/_pytrexio*.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i CLEANFILES += src/pytrexio_wrap.c src/pytrexio.py src/_pytrexio*.so -.PHONY: cppcheck python python-test +.PHONY: cppcheck python python-build python-test endif diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d484910 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = [ + "setuptools>=42", + "h5py", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b9e161b --- /dev/null +++ b/setup.cfg @@ -0,0 +1,23 @@ +[metadata] +name = pytrexio666 +version = 0.1 +author = TREX-CoE +description = Python API of the TREXIO library +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/TREX-CoE/trexio +project_urls = + Bug Tracker = https://github.com/TREX-CoE/trexio/issues +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: BSD License + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.6 + +[options.packages.find] +where = src + From f3dbfc7be226976d092b89eb247ad7c1e35c80b7 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 28 Jul 2021 10:33:33 +0200 Subject: [PATCH 021/107] added License and install_requires in setup.py + cleaning --- src/setup.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/setup.py b/src/setup.py index 901902a..81bd567 100644 --- a/src/setup.py +++ b/src/setup.py @@ -8,20 +8,22 @@ from distutils.core import setup, Extension pytrexio_module = Extension('_pytrexio', - sources=['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'], - include_dirs=['/usr/include/hdf5/serial'], - #runtime_library_dirs=['/usr/lib/x86_64-linux-gnu/hdf5/serial'], - libraries=['hdf5', 'hdf5_hl'], - extra_compile_args=['-Wno-discarded-qualifiers'], - extra_link_args=['-L/usr/lib/x86_64-linux-gnu/hdf5/serial'] - ) + sources = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'], + include_dirs = ['/usr/include/hdf5/serial'], + #runtime_library_dirs=['/usr/lib/x86_64-linux-gnu/hdf5/serial'], + libraries = ['hdf5', 'hdf5_hl'], + extra_compile_args = ['-Wno-discarded-qualifiers'], + extra_link_args = ['-L/usr/lib/x86_64-linux-gnu/hdf5/serial'] + ) -setup (name = 'pytrexio', - version = '0.1', - author = "TREX-CoE", - description = """Python API of the TREXIO library""", - ext_modules = [pytrexio_module], - py_modules = ["pytrexio"], - url='https://github.com/TREX-CoE/trexio', - packages=['distutils', 'distutils.command'], - ) +setup(name = 'pytrexio', + version = '0.1', + author = "TREX-CoE", + description = """Python API of the TREXIO library""", + ext_modules = [pytrexio_module], + py_modules = ["pytrexio"], + url = 'https://github.com/TREX-CoE/trexio', + license = 'BSD', + packages = ['distutils', 'distutils.command'], + install_requires = ['h5py'] + ) From ea0ea0ac38d3d68d329f0c213d72308667dd4f80 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 28 Jul 2021 15:57:42 +0200 Subject: [PATCH 022/107] move from distutils to setuptools sdist and install arguments of setup.py work! --- MANIFEST.in | 1 + Makefile.am | 23 +++++++++++++---------- setup.cfg | 18 ++---------------- setup.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ src/setup.py | 29 ----------------------------- 5 files changed, 65 insertions(+), 55 deletions(-) create mode 100644 MANIFEST.in create mode 100644 setup.py delete mode 100644 src/setup.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7bbd55e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include README.md src/pytrexio.py src/pytrexio_wrap.c src/trexio.c src/trexio_hdf5.c src/trexio_text.c diff --git a/Makefile.am b/Makefile.am index de651ee..de44d0e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -184,26 +184,29 @@ cppcheck.out: $(trexio_h) -I../include *.c *.h 2>../$@ -python: src/_pytrexio*.so +python: _pytrexio*.so -python-test: src/test.py src/_pytrexio*.so - cd src/ && python3 test.py - $(RM) -r -- src/__pycache__ +python-test: test.py _pytrexio*.so + python3 test.py + $(RM) -r -- __pycache__ -python-build: pyproject.toml setup.cfg +python-sdist: setup.py + python3 setup.py sdist + + +python-build: pyproject.toml setup.py setup.cfg MANIFEST.in python3 -m build -python-release: pyproject.toml setup.cfg +python-release: pyproject.toml setup.py setup.cfg MANIFEST.in python3 -m twine upload --repository testpypi dist/* # Advanced compilation using Python-native distutils # # swig -python -threads pytrexio.i ----> Add thread support for all the interface # -src/_pytrexio*.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i +_pytrexio*.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i cp $(trexio_h) src/ - cd src/ && \ - swig -python -py3 -o pytrexio_wrap.c pytrexio.i && \ + cd src/ && swig -python -py3 -o pytrexio_wrap.c pytrexio.i python3 setup.py build_ext --inplace --swig-opts="-modern" $(RM) -- src/trexio.h @@ -220,7 +223,7 @@ src/_pytrexio*.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i # $(RM) -- src/trexio.h # -CLEANFILES += src/pytrexio_wrap.c src/pytrexio.py src/_pytrexio*.so +CLEANFILES += src/pytrexio_wrap.c src/pytrexio.py _pytrexio*.so .PHONY: cppcheck python python-build python-test diff --git a/setup.cfg b/setup.cfg index b9e161b..a6688ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,23 +1,9 @@ [metadata] -name = pytrexio666 -version = 0.1 -author = TREX-CoE -description = Python API of the TREXIO library -long_description = file: README.md -long_description_content_type = text/markdown -url = https://github.com/TREX-CoE/trexio +description-file = README.md project_urls = Bug Tracker = https://github.com/TREX-CoE/trexio/issues -classifiers = - Programming Language :: Python :: 3 - License :: OSI Approved :: BSD License + [options] -package_dir = - = src -packages = find: python_requires = >=3.6 -[options.packages.find] -where = src - diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e83c70e --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +""" +setup.py file for pytrexio +""" + +#from distutils.core import setup, Extension +from setuptools import setup, Extension, find_packages +import os + +rootpath = os.path.dirname(os.path.abspath(__file__)) +srcpath = os.path.join(rootpath, 'src') +c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] + + +with open("README.md", "r") as fh: + long_description = fh.read() + + +pytrexio_module = Extension('_pytrexio', + sources = [os.path.join(srcpath, code) for code in c_files], + include_dirs = ['/usr/include/hdf5/serial', 'srcpath'], + libraries = ['hdf5', 'hdf5_hl'], + extra_compile_args = ['-Wno-discarded-qualifiers'], + extra_link_args = ['-L/usr/lib/x86_64-linux-gnu/hdf5/serial'] + ) + + +setup(name = 'pytrexio', + version = '0.1', + author = "Evgeny Posenitskiy", + author_email = "posenitskiy@irsamc.ups-tlse.fr", + description = """Python API of the TREXIO library""", + long_description = long_description, + long_description_content_type="text/markdown", + ext_modules = [pytrexio_module], + py_modules = ["pytrexio"], + url = 'https://github.com/TREX-CoE/trexio', + license = 'BSD', + packages = find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "Programming Language :: C" + "License :: OSI Approved :: BSD", + "Operating System :: POSIX :: Linux", + ], + install_requires = ['h5py'] + ) + diff --git a/src/setup.py b/src/setup.py deleted file mode 100644 index 81bd567..0000000 --- a/src/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -""" -setup.py file for pytrexio -""" - -from distutils.core import setup, Extension - - -pytrexio_module = Extension('_pytrexio', - sources = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'], - include_dirs = ['/usr/include/hdf5/serial'], - #runtime_library_dirs=['/usr/lib/x86_64-linux-gnu/hdf5/serial'], - libraries = ['hdf5', 'hdf5_hl'], - extra_compile_args = ['-Wno-discarded-qualifiers'], - extra_link_args = ['-L/usr/lib/x86_64-linux-gnu/hdf5/serial'] - ) - -setup(name = 'pytrexio', - version = '0.1', - author = "TREX-CoE", - description = """Python API of the TREXIO library""", - ext_modules = [pytrexio_module], - py_modules = ["pytrexio"], - url = 'https://github.com/TREX-CoE/trexio', - license = 'BSD', - packages = ['distutils', 'distutils.command'], - install_requires = ['h5py'] - ) From e0162d457034a101000079e6e9d6ac0028baa73e Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 28 Jul 2021 15:59:29 +0200 Subject: [PATCH 023/107] return double datasets as numpy arrays using numpy.i and safe API --- src/pytrexio.i | 18 +++++++++++++++++- src/test.py | 26 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index c8eb1c8..47f60a4 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -46,6 +46,22 @@ #define TREXIO_HDF5 0 #define TREXIO_TEXT 0 */ + +/* This is an attempt to make SWIG treat double * dset_out, const uint64_t dim_out pattern + as a special case in order to return the NumPy array to Python from C pointer to array + provided by trexio_read_safe_[dset_num] function. + NOTE: numpy.i is currently not part of SWIG but included in the numpy distribution (under numpy/tools/swig/numpy.i) + This means that the interface file have to be provided to SWIG upon compilation either by + copying it to the local working directory or by providing -l/path/to/numpy.i flag upon SWIG compilation +*/ +%include "numpy.i" + +%init %{ +import_array(); +%} + +%apply (double* ARGOUT_ARRAY1, int DIM1) {(double * const dset_out, const uint64_t dim_out)}; + /* This tells SWIG to treat char ** dset_in pattern as a special case Enables access to trexio_[...]_write_dset_str set of functions directly, i.e. by converting input list of strings from Python into char ** of C @@ -79,7 +95,7 @@ /* [WIP] This is an attempt to make SWIG treat char ** dset_out as a special case In order to return list of string to Python from C-native char ** dset_out, - which is modified (but not allocated) within the trexio_[...}read_dset_str function + which is modified (but not allocated) within the trexio_[...]_read_dset_str function */ %typemap(in, numinputs=0) char ** dset_out (char * temp) { /*temp = (char *) malloc(1028*sizeof(char));*/ diff --git a/src/test.py b/src/test.py index 83ea513..ed3632f 100644 --- a/src/test.py +++ b/src/test.py @@ -1,5 +1,6 @@ import os import shutil +#import numpy as np from pytrexio import * @@ -95,9 +96,26 @@ assert rc==TREXIO_SUCCESS for i in range(nucleus_num): assert charges2[i]==charges[i] +#charge_numpy = np.zeros(nucleus_num, dtype=np.float64) +#print(charge_numpy) + +rc, charge_numpy = trexio_read_safe_nucleus_charge(test_file2, 12) + +print(charge_numpy) +print(charge_numpy[11]) +assert rc==TREXIO_SUCCESS + +# unsafe call to read_safe should not only have return code = TREXIO_UNSAFE_ARRAY_DIM +# but also should not return numpy array filled with garbage +rc, charge_numpy = trexio_read_safe_nucleus_charge(test_file2, 12*5) + +#print(charge_numpy) +assert rc==TREXIO_UNSAFE_ARRAY_DIM + # [WIP]: ideally, the list of strings should be returned as below #rc, label_2d = trexio_read_nucleus_label(test_file2, 10) # [WIP]: currently only low-level routines (return one long string instead of an array of strings) work + rc, labels_1d = trexio_read_nucleus_label_low(test_file2, 10) assert rc==TREXIO_SUCCESS @@ -109,5 +127,13 @@ for i in range(nucleus_num): rc = trexio_close(test_file2) assert rc==TREXIO_SUCCESS +try: + if TEST_TREXIO_BACKEND == TREXIO_HDF5: + os.remove(output_filename) + elif TEST_TREXIO_BACKEND == TREXIO_TEXT: + shutil.rmtree(output_filename) +except: + print (f'No output file {output_filename} has been produced') + #==========================================================# From 621f4bc8b14784531e583bfd65b6538c525afc32 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 28 Jul 2021 16:25:54 +0200 Subject: [PATCH 024/107] release script for TestPyPI --- setup.py | 6 +++--- test-pypi-release-pytrexio.sh | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) create mode 100755 test-pypi-release-pytrexio.sh diff --git a/setup.py b/setup.py index e83c70e..6161edd 100644 --- a/setup.py +++ b/setup.py @@ -40,9 +40,9 @@ setup(name = 'pytrexio', packages = find_packages(), classifiers=[ "Programming Language :: Python :: 3", - "Programming Language :: C" - "License :: OSI Approved :: BSD", - "Operating System :: POSIX :: Linux", + "Programming Language :: C", + "License :: OSI Approved :: BSD License", + "Operating System :: POSIX :: Linux" ], install_requires = ['h5py'] ) diff --git a/test-pypi-release-pytrexio.sh b/test-pypi-release-pytrexio.sh new file mode 100755 index 0000000..e208d33 --- /dev/null +++ b/test-pypi-release-pytrexio.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -x +set -e + +#source version.py + +python3 -m pip install --upgrade setuptools wheel twine +python3 -s setup.py --no-user-cfg build +python3 setup.py sdist bdist_wheel +python3 -m pip install dist/pytrexio-0.1-cp38-cp38-linux_x86_64.whl +#python3 -m twine upload dist/pytrexio-0.1.tar.gz + +# NOTE:I get an error: +# Binary wheel 'pytrexio-0.1-cp38-cp38-linux_x86_64.whl' has an unsupported platform tag 'linux_x86_64'. +# when uploading the wheel instead of the tar.gz file to testpypi +# This is a well-known issue, see https://stackoverflow.com/questions/59451069/binary-wheel-cant-be-uploaded-on-pypi-using-twine + +python3 -m twine upload --repository testpypi dist/pytrexio-0.1.tar.gz + +rm -rf build dist pytrexio.egg-info From 6ce91a666969b9ea76ff3c8ebc3b0d17e2f10eaf Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 4 Aug 2021 10:18:36 +0300 Subject: [PATCH 025/107] create python directory and add TODO objectives --- MANIFEST.in => python/MANIFEST.in | 0 pyproject.toml => python/pyproject.toml | 0 setup.cfg => python/setup.cfg | 0 setup.py => python/setup.py | 0 .../test-pypi-release-pytrexio.sh | 6 ++++-- {src => python}/test.py | 19 ++++++++++++++++--- 6 files changed, 20 insertions(+), 5 deletions(-) rename MANIFEST.in => python/MANIFEST.in (100%) rename pyproject.toml => python/pyproject.toml (100%) rename setup.cfg => python/setup.cfg (100%) rename setup.py => python/setup.py (100%) rename test-pypi-release-pytrexio.sh => python/test-pypi-release-pytrexio.sh (72%) rename {src => python}/test.py (84%) diff --git a/MANIFEST.in b/python/MANIFEST.in similarity index 100% rename from MANIFEST.in rename to python/MANIFEST.in diff --git a/pyproject.toml b/python/pyproject.toml similarity index 100% rename from pyproject.toml rename to python/pyproject.toml diff --git a/setup.cfg b/python/setup.cfg similarity index 100% rename from setup.cfg rename to python/setup.cfg diff --git a/setup.py b/python/setup.py similarity index 100% rename from setup.py rename to python/setup.py diff --git a/test-pypi-release-pytrexio.sh b/python/test-pypi-release-pytrexio.sh similarity index 72% rename from test-pypi-release-pytrexio.sh rename to python/test-pypi-release-pytrexio.sh index e208d33..a4fa0fd 100755 --- a/test-pypi-release-pytrexio.sh +++ b/python/test-pypi-release-pytrexio.sh @@ -15,7 +15,9 @@ python3 -m pip install dist/pytrexio-0.1-cp38-cp38-linux_x86_64.whl # Binary wheel 'pytrexio-0.1-cp38-cp38-linux_x86_64.whl' has an unsupported platform tag 'linux_x86_64'. # when uploading the wheel instead of the tar.gz file to testpypi # This is a well-known issue, see https://stackoverflow.com/questions/59451069/binary-wheel-cant-be-uploaded-on-pypi-using-twine - -python3 -m twine upload --repository testpypi dist/pytrexio-0.1.tar.gz + +# TODO: see how SQLite does things since it-s similar to out problem in a way that the have src/ directory with C source and header files and they have master setup.py script to compile and ship it + +#python3 -m twine upload --repository testpypi dist/pytrexio-0.1.tar.gz rm -rf build dist pytrexio.egg-info diff --git a/src/test.py b/python/test.py similarity index 84% rename from src/test.py rename to python/test.py index ed3632f..d0649b5 100644 --- a/src/test.py +++ b/python/test.py @@ -4,6 +4,15 @@ import shutil from pytrexio import * +# TODO: make a user-friendly more pythonic API that will have to be autogenerated +# add Exception handling +# check of dimensions and call to safe API +# conversion to and from numpy arrays + +# automatically download (hopefully the latest version) numpy.i using +# wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i +# but also keep the original copy in case if wget fails + #=========================================================# #======== SETUP THE BACK END AND OUTPUT FILE NAME ========# #=========================================================# @@ -82,14 +91,18 @@ assert rc==TREXIO_SUCCESS test_file2 = trexio_open(output_filename, 'r', TEST_TREXIO_BACKEND) +# TODO: think about adding exception handling if rc != TREXIO_SUCCESS throw an exception but do not return the code itself and then there is less arguments +# TODO: maybe also for the write + result = trexio_read_nucleus_num(test_file2) -assert result[0]==0 +assert result[0]==TREXIO_SUCCESS assert result[1]==nucleus_num #print(result) charges2 = doubleArray(nucleus_num) -for i in range(nucleus_num): - charges2[i] = -1. +print(charges2[3]) +#for i in range(nucleus_num): +# charges2[i] = -1. rc = trexio_read_nucleus_charge(test_file2, charges2) assert rc==TREXIO_SUCCESS From 9a28b8d4bcc0f66cea8db8cb590363d08ab4ed5d Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 4 Aug 2021 12:07:27 +0300 Subject: [PATCH 026/107] Revert "create python directory and add TODO objectives" This reverts commit 6ce91a666969b9ea76ff3c8ebc3b0d17e2f10eaf. --- python/MANIFEST.in => MANIFEST.in | 0 python/pyproject.toml => pyproject.toml | 0 python/setup.cfg => setup.cfg | 0 python/setup.py => setup.py | 0 {python => src}/test.py | 19 +++---------------- ...trexio.sh => test-pypi-release-pytrexio.sh | 6 ++---- 6 files changed, 5 insertions(+), 20 deletions(-) rename python/MANIFEST.in => MANIFEST.in (100%) rename python/pyproject.toml => pyproject.toml (100%) rename python/setup.cfg => setup.cfg (100%) rename python/setup.py => setup.py (100%) rename {python => src}/test.py (84%) rename python/test-pypi-release-pytrexio.sh => test-pypi-release-pytrexio.sh (72%) diff --git a/python/MANIFEST.in b/MANIFEST.in similarity index 100% rename from python/MANIFEST.in rename to MANIFEST.in diff --git a/python/pyproject.toml b/pyproject.toml similarity index 100% rename from python/pyproject.toml rename to pyproject.toml diff --git a/python/setup.cfg b/setup.cfg similarity index 100% rename from python/setup.cfg rename to setup.cfg diff --git a/python/setup.py b/setup.py similarity index 100% rename from python/setup.py rename to setup.py diff --git a/python/test.py b/src/test.py similarity index 84% rename from python/test.py rename to src/test.py index d0649b5..ed3632f 100644 --- a/python/test.py +++ b/src/test.py @@ -4,15 +4,6 @@ import shutil from pytrexio import * -# TODO: make a user-friendly more pythonic API that will have to be autogenerated -# add Exception handling -# check of dimensions and call to safe API -# conversion to and from numpy arrays - -# automatically download (hopefully the latest version) numpy.i using -# wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i -# but also keep the original copy in case if wget fails - #=========================================================# #======== SETUP THE BACK END AND OUTPUT FILE NAME ========# #=========================================================# @@ -91,18 +82,14 @@ assert rc==TREXIO_SUCCESS test_file2 = trexio_open(output_filename, 'r', TEST_TREXIO_BACKEND) -# TODO: think about adding exception handling if rc != TREXIO_SUCCESS throw an exception but do not return the code itself and then there is less arguments -# TODO: maybe also for the write - result = trexio_read_nucleus_num(test_file2) -assert result[0]==TREXIO_SUCCESS +assert result[0]==0 assert result[1]==nucleus_num #print(result) charges2 = doubleArray(nucleus_num) -print(charges2[3]) -#for i in range(nucleus_num): -# charges2[i] = -1. +for i in range(nucleus_num): + charges2[i] = -1. rc = trexio_read_nucleus_charge(test_file2, charges2) assert rc==TREXIO_SUCCESS diff --git a/python/test-pypi-release-pytrexio.sh b/test-pypi-release-pytrexio.sh similarity index 72% rename from python/test-pypi-release-pytrexio.sh rename to test-pypi-release-pytrexio.sh index a4fa0fd..e208d33 100755 --- a/python/test-pypi-release-pytrexio.sh +++ b/test-pypi-release-pytrexio.sh @@ -15,9 +15,7 @@ python3 -m pip install dist/pytrexio-0.1-cp38-cp38-linux_x86_64.whl # Binary wheel 'pytrexio-0.1-cp38-cp38-linux_x86_64.whl' has an unsupported platform tag 'linux_x86_64'. # when uploading the wheel instead of the tar.gz file to testpypi # This is a well-known issue, see https://stackoverflow.com/questions/59451069/binary-wheel-cant-be-uploaded-on-pypi-using-twine - -# TODO: see how SQLite does things since it-s similar to out problem in a way that the have src/ directory with C source and header files and they have master setup.py script to compile and ship it - -#python3 -m twine upload --repository testpypi dist/pytrexio-0.1.tar.gz + +python3 -m twine upload --repository testpypi dist/pytrexio-0.1.tar.gz rm -rf build dist pytrexio.egg-info From 60c28c1f3bbd856c89acc2a014f4f1190392e715 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 4 Aug 2021 14:13:23 +0300 Subject: [PATCH 027/107] rename python test and add temporary TODO --- src/test.py => tests/test_py.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) rename src/test.py => tests/test_py.py (84%) diff --git a/src/test.py b/tests/test_py.py similarity index 84% rename from src/test.py rename to tests/test_py.py index ed3632f..d0649b5 100644 --- a/src/test.py +++ b/tests/test_py.py @@ -4,6 +4,15 @@ import shutil from pytrexio import * +# TODO: make a user-friendly more pythonic API that will have to be autogenerated +# add Exception handling +# check of dimensions and call to safe API +# conversion to and from numpy arrays + +# automatically download (hopefully the latest version) numpy.i using +# wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i +# but also keep the original copy in case if wget fails + #=========================================================# #======== SETUP THE BACK END AND OUTPUT FILE NAME ========# #=========================================================# @@ -82,14 +91,18 @@ assert rc==TREXIO_SUCCESS test_file2 = trexio_open(output_filename, 'r', TEST_TREXIO_BACKEND) +# TODO: think about adding exception handling if rc != TREXIO_SUCCESS throw an exception but do not return the code itself and then there is less arguments +# TODO: maybe also for the write + result = trexio_read_nucleus_num(test_file2) -assert result[0]==0 +assert result[0]==TREXIO_SUCCESS assert result[1]==nucleus_num #print(result) charges2 = doubleArray(nucleus_num) -for i in range(nucleus_num): - charges2[i] = -1. +print(charges2[3]) +#for i in range(nucleus_num): +# charges2[i] = -1. rc = trexio_read_nucleus_charge(test_file2, charges2) assert rc==TREXIO_SUCCESS From b8c188343ec06507b028de28b1d50a75231bd574 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 5 Aug 2021 10:34:16 +0300 Subject: [PATCH 028/107] better python build --- Makefile.am | 30 +++++++++++++++++++++--------- setup.py | 8 +++++--- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Makefile.am b/Makefile.am index de44d0e..d99d9b8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -184,29 +184,41 @@ cppcheck.out: $(trexio_h) -I../include *.c *.h 2>../$@ +setup_py = $(srcdir)/setup.py +setup_cfg = $(srcdir)/setup.cfg +pytrexio_py = $(srcdir)/src/pytrexio.py +pytrexio_c = $(srcdir)/src/pytrexio_wrap.c +pytrexio_i = $(srcdir)/src/pytrexio.i +TESTS_PY = $(srcdir)/tests/test_py.py + +#python-prepare: +# cp include/trexio.h src/*.c src/*.h python/src/ + python: _pytrexio*.so -python-test: test.py _pytrexio*.so - python3 test.py - $(RM) -r -- __pycache__ +python-test: $(TESTS_PY) $(pytrexio_py) + cp $(pytrexio_py) . + python3 $(TESTS_PY) + $(RM) -r -- __pycache__ pytrexio.py -python-sdist: setup.py +python-sdist: $(setup_py) $(pytrexio_py) python3 setup.py sdist - -python-build: pyproject.toml setup.py setup.cfg MANIFEST.in +python-build: $(setup_py) $(pytrexio_py) pyproject.toml python3 -m build -python-release: pyproject.toml setup.py setup.cfg MANIFEST.in +python-release: $(setup_py) $(pytrexio_py) pyproject.toml python3 -m twine upload --repository testpypi dist/* # Advanced compilation using Python-native distutils # # swig -python -threads pytrexio.i ----> Add thread support for all the interface # -_pytrexio*.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i +$(pytrexio_c): $(ORG_FILES) $(trexio_h) $(pytrexio_i) cp $(trexio_h) src/ cd src/ && swig -python -py3 -o pytrexio_wrap.c pytrexio.i + +_pytrexio*.so: $(pytrexio_c) $(setup_py) python3 setup.py build_ext --inplace --swig-opts="-modern" $(RM) -- src/trexio.h @@ -225,7 +237,7 @@ _pytrexio*.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i CLEANFILES += src/pytrexio_wrap.c src/pytrexio.py _pytrexio*.so -.PHONY: cppcheck python python-build python-test +.PHONY: cppcheck python python-build python-test python-release endif diff --git a/setup.py b/setup.py index 6161edd..b574a40 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ import os rootpath = os.path.dirname(os.path.abspath(__file__)) srcpath = os.path.join(rootpath, 'src') c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] +h_files = ['trexio.h', 'trexio_hdf5.h', 'trexio_text.h', 'trexio_s.h', 'trexio_private.h'] with open("README.md", "r") as fh: @@ -19,7 +20,7 @@ with open("README.md", "r") as fh: pytrexio_module = Extension('_pytrexio', sources = [os.path.join(srcpath, code) for code in c_files], - include_dirs = ['/usr/include/hdf5/serial', 'srcpath'], + include_dirs = ['/usr/include/hdf5/serial', srcpath], libraries = ['hdf5', 'hdf5_hl'], extra_compile_args = ['-Wno-discarded-qualifiers'], extra_link_args = ['-L/usr/lib/x86_64-linux-gnu/hdf5/serial'] @@ -28,13 +29,14 @@ pytrexio_module = Extension('_pytrexio', setup(name = 'pytrexio', version = '0.1', - author = "Evgeny Posenitskiy", + author = "TREX-CoE", author_email = "posenitskiy@irsamc.ups-tlse.fr", description = """Python API of the TREXIO library""", long_description = long_description, - long_description_content_type="text/markdown", + long_description_content_type = "text/markdown", ext_modules = [pytrexio_module], py_modules = ["pytrexio"], + scripts = ["tests/test_py.py"], url = 'https://github.com/TREX-CoE/trexio', license = 'BSD', packages = find_packages(), From 2854d617d4cde2bd0b4edf1c4cce40da12b58c9e Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 5 Aug 2021 12:55:18 +0300 Subject: [PATCH 029/107] make python directory and adapt build process --- MANIFEST.in | 1 - python/MANIFEST.in | 1 + pyproject.toml => python/pyproject.toml | 0 python/setup.cfg | 12 ++++++++++++ setup.py => python/setup.py | 2 +- .../test-pypi-release-pytrexio.sh | 14 ++++++++++---- {tests => python/test}/test_py.py | 0 setup.cfg | 9 --------- 8 files changed, 24 insertions(+), 15 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 python/MANIFEST.in rename pyproject.toml => python/pyproject.toml (100%) create mode 100644 python/setup.cfg rename setup.py => python/setup.py (97%) rename test-pypi-release-pytrexio.sh => python/test-pypi-release-pytrexio.sh (66%) rename {tests => python/test}/test_py.py (100%) delete mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7bbd55e..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md src/pytrexio.py src/pytrexio_wrap.c src/trexio.c src/trexio_hdf5.c src/trexio_text.c diff --git a/python/MANIFEST.in b/python/MANIFEST.in new file mode 100644 index 0000000..93f9f54 --- /dev/null +++ b/python/MANIFEST.in @@ -0,0 +1 @@ +include src/pytrexio.py src/pytrexio_wrap.c src/trexio.c src/trexio_hdf5.c src/trexio_text.c diff --git a/pyproject.toml b/python/pyproject.toml similarity index 100% rename from pyproject.toml rename to python/pyproject.toml diff --git a/python/setup.cfg b/python/setup.cfg new file mode 100644 index 0000000..9549416 --- /dev/null +++ b/python/setup.cfg @@ -0,0 +1,12 @@ +[metadata] +long_description = file: README.md +licence_file = LICENSE +project_urls = + Bug Tracker = https://github.com/TREX-CoE/trexio/issues +license = BSD 3-Clause License + +[options] +zip_safe = False +python_requires = >=3.6 +install_requires = h5py + diff --git a/setup.py b/python/setup.py similarity index 97% rename from setup.py rename to python/setup.py index b574a40..dc6512c 100644 --- a/setup.py +++ b/python/setup.py @@ -13,6 +13,7 @@ srcpath = os.path.join(rootpath, 'src') c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] h_files = ['trexio.h', 'trexio_hdf5.h', 'trexio_text.h', 'trexio_s.h', 'trexio_private.h'] +# setup.py does not copy all with open("README.md", "r") as fh: long_description = fh.read() @@ -36,7 +37,6 @@ setup(name = 'pytrexio', long_description_content_type = "text/markdown", ext_modules = [pytrexio_module], py_modules = ["pytrexio"], - scripts = ["tests/test_py.py"], url = 'https://github.com/TREX-CoE/trexio', license = 'BSD', packages = find_packages(), diff --git a/test-pypi-release-pytrexio.sh b/python/test-pypi-release-pytrexio.sh similarity index 66% rename from test-pypi-release-pytrexio.sh rename to python/test-pypi-release-pytrexio.sh index e208d33..3262540 100755 --- a/test-pypi-release-pytrexio.sh +++ b/python/test-pypi-release-pytrexio.sh @@ -6,9 +6,14 @@ set -e #source version.py python3 -m pip install --upgrade setuptools wheel twine -python3 -s setup.py --no-user-cfg build +#python3 -s setup.py --no-user-cfg build +python3 -s setup.py build python3 setup.py sdist bdist_wheel -python3 -m pip install dist/pytrexio-0.1-cp38-cp38-linux_x86_64.whl +python3 -m pip install dist/pytrexio-0.1-*.whl --force-reinstall + +# remove +#python3 -m pip uninstall pytrexio + #python3 -m twine upload dist/pytrexio-0.1.tar.gz # NOTE:I get an error: @@ -16,6 +21,7 @@ python3 -m pip install dist/pytrexio-0.1-cp38-cp38-linux_x86_64.whl # when uploading the wheel instead of the tar.gz file to testpypi # This is a well-known issue, see https://stackoverflow.com/questions/59451069/binary-wheel-cant-be-uploaded-on-pypi-using-twine -python3 -m twine upload --repository testpypi dist/pytrexio-0.1.tar.gz +#python3 -m twine upload --repository testpypi dist/pytrexio-0.1.tar.gz -rm -rf build dist pytrexio.egg-info +rm -rf build pytrexio.egg-info +rm -rf dist diff --git a/tests/test_py.py b/python/test/test_py.py similarity index 100% rename from tests/test_py.py rename to python/test/test_py.py diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index a6688ac..0000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[metadata] -description-file = README.md -project_urls = - Bug Tracker = https://github.com/TREX-CoE/trexio/issues - - -[options] -python_requires = >=3.6 - From 12280e42199e5489d618c1cc5c1faf1a7d3c088a Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 5 Aug 2021 17:48:23 +0300 Subject: [PATCH 030/107] more general MANIFEST.in --- python/MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 93f9f54..10daee9 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1 +1 @@ -include src/pytrexio.py src/pytrexio_wrap.c src/trexio.c src/trexio_hdf5.c src/trexio_text.c +include src/*.c src/trexio*.h src/*.py From bb18a9c82641faf8306051f50ccf800be6874e36 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 5 Aug 2021 17:48:53 +0300 Subject: [PATCH 031/107] minor cleaning --- python/setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index dc6512c..29ae023 100644 --- a/python/setup.py +++ b/python/setup.py @@ -4,16 +4,12 @@ setup.py file for pytrexio """ -#from distutils.core import setup, Extension from setuptools import setup, Extension, find_packages import os rootpath = os.path.dirname(os.path.abspath(__file__)) srcpath = os.path.join(rootpath, 'src') c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] -h_files = ['trexio.h', 'trexio_hdf5.h', 'trexio_text.h', 'trexio_s.h', 'trexio_private.h'] - -# setup.py does not copy all with open("README.md", "r") as fh: long_description = fh.read() From d0c19e0ce0ca69237961da657ba7e3a63aee75b7 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 5 Aug 2021 17:55:34 +0300 Subject: [PATCH 032/107] pytrexio build and pip install working --- Makefile.am | 69 +- python/.gitignore | 10 + python/install_pytrexio.sh | 47 + python/test-pypi-release-pytrexio.sh | 27 - src/numpy.i | 3028 ++++++++++++++++++++++++++ tools/prepare_python.sh | 31 + 6 files changed, 3142 insertions(+), 70 deletions(-) create mode 100644 python/.gitignore create mode 100755 python/install_pytrexio.sh delete mode 100755 python/test-pypi-release-pytrexio.sh create mode 100644 src/numpy.i create mode 100755 tools/prepare_python.sh diff --git a/Makefile.am b/Makefile.am index d99d9b8..f5798b7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -137,7 +137,7 @@ tests_test_f_SOURCES = $(test_trexio_f) tests/test_f.f90 tests_test_f_LDFLAGS = -no-install clean-local: - -rm -rf -- *.dir/ *.h5 + -rm -rf -- *.dir/ *.h5 __pycache__/ # =============== DOCUMENTATION =============== # @@ -184,60 +184,43 @@ cppcheck.out: $(trexio_h) -I../include *.c *.h 2>../$@ -setup_py = $(srcdir)/setup.py -setup_cfg = $(srcdir)/setup.cfg -pytrexio_py = $(srcdir)/src/pytrexio.py -pytrexio_c = $(srcdir)/src/pytrexio_wrap.c -pytrexio_i = $(srcdir)/src/pytrexio.i -TESTS_PY = $(srcdir)/tests/test_py.py +setup_py = $(srcdir)/python/setup.py +setup_cfg = $(srcdir)/python/setup.cfg +pytrexio_py = $(srcdir)/python/pytrexio.py +TESTS_PY = $(srcdir)/python/test/test_py.py +pytrexio_c = $(srcdir)/src/pytrexio_wrap.c +pytrexio_i = $(srcdir)/src/pytrexio.i +numpy_i = $(srcdir)/src/numpy.i -#python-prepare: -# cp include/trexio.h src/*.c src/*.h python/src/ -python: _pytrexio*.so - -python-test: $(TESTS_PY) $(pytrexio_py) - cp $(pytrexio_py) . +python-test: $(pytrexio_py) $(TESTS_PY) python3 $(TESTS_PY) - $(RM) -r -- __pycache__ pytrexio.py + $(RM) -r -- __pycache__ -python-sdist: $(setup_py) $(pytrexio_py) - python3 setup.py sdist +python-install: $(pytrexio_py) $(setup_py) $(setup_cfg) + cd python && ./install_pytrexio.sh -python-build: $(setup_py) $(pytrexio_py) pyproject.toml - python3 -m build +$(pytrexio_py): $(pytrexio_c) + cd tools && ./prepare_python.sh -python-release: $(setup_py) $(pytrexio_py) pyproject.toml - python3 -m twine upload --repository testpypi dist/* - -# Advanced compilation using Python-native distutils -# -# swig -python -threads pytrexio.i ----> Add thread support for all the interface -# -$(pytrexio_c): $(ORG_FILES) $(trexio_h) $(pytrexio_i) +# Build Python module and C wrapper code for TREXIO using SWIG +# [?] swig -python -threads pytrexio.i ----> Add thread support for all the interface +$(pytrexio_c): $(ORG_FILES) $(trexio_h) $(pytrexio_i) $(numpy_i) cp $(trexio_h) src/ cd src/ && swig -python -py3 -o pytrexio_wrap.c pytrexio.i - -_pytrexio*.so: $(pytrexio_c) $(setup_py) - python3 setup.py build_ext --inplace --swig-opts="-modern" $(RM) -- src/trexio.h -# Manual compilation of Python module -# -#src/_pytrexio.so: $(ORG_FILES) $(trexio_h) src/pytrexio.i -# cp $(trexio_h) src/ -# cd src/ && \ -# swig -python -py3 -o pytrexio_wrap.c pytrexio.i && \ -# $(CC) $(CPPFLAGS) -I/usr/include/python3.8 -fPIC -Wno-discarded-qualifiers \ -# -c trexio.c trexio_hdf5.c trexio_text.c pytrexio_wrap.c && \ -# $(CC) -shared trexio.o trexio_hdf5.o trexio_text.o pytrexio_wrap.o \ -# $(LDFLAGS) $(LIBS) -o _pytrexio.so -# $(RM) -- src/trexio.h -# +$(numpy_i): + wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i -O $(numpy_i) -CLEANFILES += src/pytrexio_wrap.c src/pytrexio.py _pytrexio*.so -.PHONY: cppcheck python python-build python-test python-release +CLEANFILES += src/pytrexio_wrap.c \ + src/pytrexio.py \ + python/pytrexio.py \ + python/src/*.c \ + python/src/*.h + +.PHONY: cppcheck python-test python-install endif diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 0000000..b968958 --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,10 @@ +AUTHORS +LICENSE +README.md +pytrexio.py +src/ + +build/ +dist/ +*.egg-info/ +__pycache__/ diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh new file mode 100755 index 0000000..65fb3a0 --- /dev/null +++ b/python/install_pytrexio.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -x +set -e + +# This script should update the version of the Python package +#source version.py +#exit 0 + +# Install/upgrade packages required for the installation +python3 -m pip install --upgrade setuptools wheel twine + +# Create build directory and compile extension files (*.c) +# --no-user-cfg disables custom .cfg files of the user machine, so that only setup.cfg is used +python3 -s setup.py --no-user-cfg build + +# Local inplace build of the .so module with SWIG-produced pytrexio_wrap.c (from the SWIG documentation) +#python3 setup.py build_ext --inplace --swig-opts="-modern" + +# Create distributions: +# 1) sdist produces .tar.gz with all files necessary for manual compilation; +# 2) bdist_whell produces .whl wheel distribution (see https://www.python.org/dev/peps/pep-0425/). +python3 setup.py sdist bdist_wheel + +# Install pytrexio in the current environment from the aforementioned wheel +# --force-reinstall is needed here because build-system pre-installs pytrexio in the environment +# but does not install things in the corresponding site-packages directory +python3 -m pip install dist/pytrexio-0.1-*.whl --force-reinstall + +# Uninstall pytrexio: this only works from the pytrexio root directory (more likely python/ folder) and only after --force-reinstall +#python3 -m pip uninstall pytrexio + +# Test the current release by uploading to TestPyPI sandbox +#python3 -m twine upload --repository testpypi dist/pytrexio-*.tar.gz + +# NOTE:I get an error: +# Binary wheel 'pytrexio-0.1-cp38-cp38-linux_x86_64.whl' has an unsupported platform tag 'linux_x86_64'. +# when uploading the wheel instead of the tar.gz file to testpypi +# This is a well-known issue, see https://stackoverflow.com/questions/59451069/binary-wheel-cant-be-uploaded-on-pypi-using-twine +# Once fixed both .tar.gz and .whl distribution can be uploaded, meaning that dist/pytrexio-*.tar.gz can be replaced by dist/* + +# Upload updated version of PyTREXIO to PyPI +#python3 -m twine upload dist/pytrexio-*.tar.gz + +# Cleaning +rm -rf build dist pytrexio.egg-info + diff --git a/python/test-pypi-release-pytrexio.sh b/python/test-pypi-release-pytrexio.sh deleted file mode 100755 index 3262540..0000000 --- a/python/test-pypi-release-pytrexio.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -x -set -e - -#source version.py - -python3 -m pip install --upgrade setuptools wheel twine -#python3 -s setup.py --no-user-cfg build -python3 -s setup.py build -python3 setup.py sdist bdist_wheel -python3 -m pip install dist/pytrexio-0.1-*.whl --force-reinstall - -# remove -#python3 -m pip uninstall pytrexio - -#python3 -m twine upload dist/pytrexio-0.1.tar.gz - -# NOTE:I get an error: -# Binary wheel 'pytrexio-0.1-cp38-cp38-linux_x86_64.whl' has an unsupported platform tag 'linux_x86_64'. -# when uploading the wheel instead of the tar.gz file to testpypi -# This is a well-known issue, see https://stackoverflow.com/questions/59451069/binary-wheel-cant-be-uploaded-on-pypi-using-twine - -#python3 -m twine upload --repository testpypi dist/pytrexio-0.1.tar.gz - -rm -rf build pytrexio.egg-info -rm -rf dist diff --git a/src/numpy.i b/src/numpy.i new file mode 100644 index 0000000..99ed073 --- /dev/null +++ b/src/numpy.i @@ -0,0 +1,3028 @@ +/* -*- C -*- (not really, but good for syntax highlighting) */ + +/* + * Copyright (c) 2005-2015, NumPy Developers. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * * Neither the name of the NumPy Developers nor the names of any + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef SWIGPYTHON + +%{ +#ifndef SWIG_FILE_WITH_INIT +#define NO_IMPORT_ARRAY +#endif +#include "stdio.h" +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include +%} + +/**********************************************************************/ + +%fragment("NumPy_Backward_Compatibility", "header") +{ +%#if NPY_API_VERSION < 0x00000007 +%#define NPY_ARRAY_DEFAULT NPY_DEFAULT +%#define NPY_ARRAY_FARRAY NPY_FARRAY +%#define NPY_FORTRANORDER NPY_FORTRAN +%#endif +} + +/**********************************************************************/ + +/* The following code originally appeared in + * enthought/kiva/agg/src/numeric.i written by Eric Jones. It was + * translated from C++ to C by John Hunter. Bill Spotz has modified + * it to fix some minor bugs, upgrade from Numeric to numpy (all + * versions), add some comments and functionality, and convert from + * direct code insertion to SWIG fragments. + */ + +%fragment("NumPy_Macros", "header") +{ +/* Macros to extract array attributes. + */ +%#if NPY_API_VERSION < 0x00000007 +%#define is_array(a) ((a) && PyArray_Check((PyArrayObject*)a)) +%#define array_type(a) (int)(PyArray_TYPE((PyArrayObject*)a)) +%#define array_numdims(a) (((PyArrayObject*)a)->nd) +%#define array_dimensions(a) (((PyArrayObject*)a)->dimensions) +%#define array_size(a,i) (((PyArrayObject*)a)->dimensions[i]) +%#define array_strides(a) (((PyArrayObject*)a)->strides) +%#define array_stride(a,i) (((PyArrayObject*)a)->strides[i]) +%#define array_data(a) (((PyArrayObject*)a)->data) +%#define array_descr(a) (((PyArrayObject*)a)->descr) +%#define array_flags(a) (((PyArrayObject*)a)->flags) +%#define array_clearflags(a,f) (((PyArrayObject*)a)->flags) &= ~f +%#define array_enableflags(a,f) (((PyArrayObject*)a)->flags) = f +%#define array_is_fortran(a) (PyArray_ISFORTRAN((PyArrayObject*)a)) +%#else +%#define is_array(a) ((a) && PyArray_Check(a)) +%#define array_type(a) PyArray_TYPE((PyArrayObject*)a) +%#define array_numdims(a) PyArray_NDIM((PyArrayObject*)a) +%#define array_dimensions(a) PyArray_DIMS((PyArrayObject*)a) +%#define array_strides(a) PyArray_STRIDES((PyArrayObject*)a) +%#define array_stride(a,i) PyArray_STRIDE((PyArrayObject*)a,i) +%#define array_size(a,i) PyArray_DIM((PyArrayObject*)a,i) +%#define array_data(a) PyArray_DATA((PyArrayObject*)a) +%#define array_descr(a) PyArray_DESCR((PyArrayObject*)a) +%#define array_flags(a) PyArray_FLAGS((PyArrayObject*)a) +%#define array_enableflags(a,f) PyArray_ENABLEFLAGS((PyArrayObject*)a,f) +%#define array_clearflags(a,f) PyArray_CLEARFLAGS((PyArrayObject*)a,f) +%#define array_is_fortran(a) (PyArray_IS_F_CONTIGUOUS((PyArrayObject*)a)) +%#endif +%#define array_is_contiguous(a) (PyArray_ISCONTIGUOUS((PyArrayObject*)a)) +%#define array_is_native(a) (PyArray_ISNOTSWAPPED((PyArrayObject*)a)) +} + +/**********************************************************************/ + +%fragment("NumPy_Utilities", + "header") +{ + /* Given a PyObject, return a string describing its type. + */ + const char* pytype_string(PyObject* py_obj) + { + if (py_obj == NULL ) return "C NULL value"; + if (py_obj == Py_None ) return "Python None" ; + if (PyCallable_Check(py_obj)) return "callable" ; + if (PyBytes_Check( py_obj)) return "string" ; + if (PyLong_Check( py_obj)) return "int" ; + if (PyFloat_Check( py_obj)) return "float" ; + if (PyDict_Check( py_obj)) return "dict" ; + if (PyList_Check( py_obj)) return "list" ; + if (PyTuple_Check( py_obj)) return "tuple" ; + + return "unknown type"; + } + + /* Given a NumPy typecode, return a string describing the type. + */ + const char* typecode_string(int typecode) + { + static const char* type_names[25] = {"bool", + "byte", + "unsigned byte", + "short", + "unsigned short", + "int", + "unsigned int", + "long", + "unsigned long", + "long long", + "unsigned long long", + "float", + "double", + "long double", + "complex float", + "complex double", + "complex long double", + "object", + "string", + "unicode", + "void", + "ntypes", + "notype", + "char", + "unknown"}; + return typecode < 24 ? type_names[typecode] : type_names[24]; + } + + /* Make sure input has correct numpy type. This now just calls + PyArray_EquivTypenums(). + */ + int type_match(int actual_type, + int desired_type) + { + return PyArray_EquivTypenums(actual_type, desired_type); + } + +%#ifdef SWIGPY_USE_CAPSULE + void free_cap(PyObject * cap) + { + void* array = (void*) PyCapsule_GetPointer(cap,SWIGPY_CAPSULE_NAME); + if (array != NULL) free(array); + } +%#endif + + +} + +/**********************************************************************/ + +%fragment("NumPy_Object_to_Array", + "header", + fragment="NumPy_Backward_Compatibility", + fragment="NumPy_Macros", + fragment="NumPy_Utilities") +{ + /* Given a PyObject pointer, cast it to a PyArrayObject pointer if + * legal. If not, set the python error string appropriately and + * return NULL. + */ + PyArrayObject* obj_to_array_no_conversion(PyObject* input, + int typecode) + { + PyArrayObject* ary = NULL; + if (is_array(input) && (typecode == NPY_NOTYPE || + PyArray_EquivTypenums(array_type(input), typecode))) + { + ary = (PyArrayObject*) input; + } + else if is_array(input) + { + const char* desired_type = typecode_string(typecode); + const char* actual_type = typecode_string(array_type(input)); + PyErr_Format(PyExc_TypeError, + "Array of type '%s' required. Array of type '%s' given", + desired_type, actual_type); + ary = NULL; + } + else + { + const char* desired_type = typecode_string(typecode); + const char* actual_type = pytype_string(input); + PyErr_Format(PyExc_TypeError, + "Array of type '%s' required. A '%s' was given", + desired_type, + actual_type); + ary = NULL; + } + return ary; + } + + /* Convert the given PyObject to a NumPy array with the given + * typecode. On success, return a valid PyArrayObject* with the + * correct type. On failure, the python error string will be set and + * the routine returns NULL. + */ + PyArrayObject* obj_to_array_allow_conversion(PyObject* input, + int typecode, + int* is_new_object) + { + PyArrayObject* ary = NULL; + PyObject* py_obj; + if (is_array(input) && (typecode == NPY_NOTYPE || + PyArray_EquivTypenums(array_type(input),typecode))) + { + ary = (PyArrayObject*) input; + *is_new_object = 0; + } + else + { + py_obj = PyArray_FROMANY(input, typecode, 0, 0, NPY_ARRAY_DEFAULT); + /* If NULL, PyArray_FromObject will have set python error value.*/ + ary = (PyArrayObject*) py_obj; + *is_new_object = 1; + } + return ary; + } + + /* Given a PyArrayObject, check to see if it is contiguous. If so, + * return the input pointer and flag it as not a new object. If it is + * not contiguous, create a new PyArrayObject using the original data, + * flag it as a new object and return the pointer. + */ + PyArrayObject* make_contiguous(PyArrayObject* ary, + int* is_new_object, + int min_dims, + int max_dims) + { + PyArrayObject* result; + if (array_is_contiguous(ary)) + { + result = ary; + *is_new_object = 0; + } + else + { + result = (PyArrayObject*) PyArray_ContiguousFromObject((PyObject*)ary, + array_type(ary), + min_dims, + max_dims); + *is_new_object = 1; + } + return result; + } + + /* Given a PyArrayObject, check to see if it is Fortran-contiguous. + * If so, return the input pointer, but do not flag it as not a new + * object. If it is not Fortran-contiguous, create a new + * PyArrayObject using the original data, flag it as a new object + * and return the pointer. + */ + PyArrayObject* make_fortran(PyArrayObject* ary, + int* is_new_object) + { + PyArrayObject* result; + if (array_is_fortran(ary)) + { + result = ary; + *is_new_object = 0; + } + else + { + Py_INCREF(array_descr(ary)); + result = (PyArrayObject*) PyArray_FromArray(ary, + array_descr(ary), +%#if NPY_API_VERSION < 0x00000007 + NPY_FORTRANORDER); +%#else + NPY_ARRAY_F_CONTIGUOUS); +%#endif + *is_new_object = 1; + } + return result; + } + + /* Convert a given PyObject to a contiguous PyArrayObject of the + * specified type. If the input object is not a contiguous + * PyArrayObject, a new one will be created and the new object flag + * will be set. + */ + PyArrayObject* obj_to_array_contiguous_allow_conversion(PyObject* input, + int typecode, + int* is_new_object) + { + int is_new1 = 0; + int is_new2 = 0; + PyArrayObject* ary2; + PyArrayObject* ary1 = obj_to_array_allow_conversion(input, + typecode, + &is_new1); + if (ary1) + { + ary2 = make_contiguous(ary1, &is_new2, 0, 0); + if ( is_new1 && is_new2) + { + Py_DECREF(ary1); + } + ary1 = ary2; + } + *is_new_object = is_new1 || is_new2; + return ary1; + } + + /* Convert a given PyObject to a Fortran-ordered PyArrayObject of the + * specified type. If the input object is not a Fortran-ordered + * PyArrayObject, a new one will be created and the new object flag + * will be set. + */ + PyArrayObject* obj_to_array_fortran_allow_conversion(PyObject* input, + int typecode, + int* is_new_object) + { + int is_new1 = 0; + int is_new2 = 0; + PyArrayObject* ary2; + PyArrayObject* ary1 = obj_to_array_allow_conversion(input, + typecode, + &is_new1); + if (ary1) + { + ary2 = make_fortran(ary1, &is_new2); + if (is_new1 && is_new2) + { + Py_DECREF(ary1); + } + ary1 = ary2; + } + *is_new_object = is_new1 || is_new2; + return ary1; + } +} /* end fragment */ + +/**********************************************************************/ + +%fragment("NumPy_Array_Requirements", + "header", + fragment="NumPy_Backward_Compatibility", + fragment="NumPy_Macros") +{ + /* Test whether a python object is contiguous. If array is + * contiguous, return 1. Otherwise, set the python error string and + * return 0. + */ + int require_contiguous(PyArrayObject* ary) + { + int contiguous = 1; + if (!array_is_contiguous(ary)) + { + PyErr_SetString(PyExc_TypeError, + "Array must be contiguous. A non-contiguous array was given"); + contiguous = 0; + } + return contiguous; + } + + /* Test whether a python object is (C_ or F_) contiguous. If array is + * contiguous, return 1. Otherwise, set the python error string and + * return 0. + */ + int require_c_or_f_contiguous(PyArrayObject* ary) + { + int contiguous = 1; + if (!(array_is_contiguous(ary) || array_is_fortran(ary))) + { + PyErr_SetString(PyExc_TypeError, + "Array must be contiguous (C_ or F_). A non-contiguous array was given"); + contiguous = 0; + } + return contiguous; + } + + /* Require that a numpy array is not byte-swapped. If the array is + * not byte-swapped, return 1. Otherwise, set the python error string + * and return 0. + */ + int require_native(PyArrayObject* ary) + { + int native = 1; + if (!array_is_native(ary)) + { + PyErr_SetString(PyExc_TypeError, + "Array must have native byteorder. " + "A byte-swapped array was given"); + native = 0; + } + return native; + } + + /* Require the given PyArrayObject to have a specified number of + * dimensions. If the array has the specified number of dimensions, + * return 1. Otherwise, set the python error string and return 0. + */ + int require_dimensions(PyArrayObject* ary, + int exact_dimensions) + { + int success = 1; + if (array_numdims(ary) != exact_dimensions) + { + PyErr_Format(PyExc_TypeError, + "Array must have %d dimensions. Given array has %d dimensions", + exact_dimensions, + array_numdims(ary)); + success = 0; + } + return success; + } + + /* Require the given PyArrayObject to have one of a list of specified + * number of dimensions. If the array has one of the specified number + * of dimensions, return 1. Otherwise, set the python error string + * and return 0. + */ + int require_dimensions_n(PyArrayObject* ary, + int* exact_dimensions, + int n) + { + int success = 0; + int i; + char dims_str[255] = ""; + char s[255]; + for (i = 0; i < n && !success; i++) + { + if (array_numdims(ary) == exact_dimensions[i]) + { + success = 1; + } + } + if (!success) + { + for (i = 0; i < n-1; i++) + { + sprintf(s, "%d, ", exact_dimensions[i]); + strcat(dims_str,s); + } + sprintf(s, " or %d", exact_dimensions[n-1]); + strcat(dims_str,s); + PyErr_Format(PyExc_TypeError, + "Array must have %s dimensions. Given array has %d dimensions", + dims_str, + array_numdims(ary)); + } + return success; + } + + /* Require the given PyArrayObject to have a specified shape. If the + * array has the specified shape, return 1. Otherwise, set the python + * error string and return 0. + */ + int require_size(PyArrayObject* ary, + npy_intp* size, + int n) + { + int i; + int success = 1; + size_t len; + char desired_dims[255] = "["; + char s[255]; + char actual_dims[255] = "["; + for(i=0; i < n;i++) + { + if (size[i] != -1 && size[i] != array_size(ary,i)) + { + success = 0; + } + } + if (!success) + { + for (i = 0; i < n; i++) + { + if (size[i] == -1) + { + sprintf(s, "*,"); + } + else + { + sprintf(s, "%ld,", (long int)size[i]); + } + strcat(desired_dims,s); + } + len = strlen(desired_dims); + desired_dims[len-1] = ']'; + for (i = 0; i < n; i++) + { + sprintf(s, "%ld,", (long int)array_size(ary,i)); + strcat(actual_dims,s); + } + len = strlen(actual_dims); + actual_dims[len-1] = ']'; + PyErr_Format(PyExc_TypeError, + "Array must have shape of %s. Given array has shape of %s", + desired_dims, + actual_dims); + } + return success; + } + + /* Require the given PyArrayObject to to be Fortran ordered. If the + * the PyArrayObject is already Fortran ordered, do nothing. Else, + * set the Fortran ordering flag and recompute the strides. + */ + int require_fortran(PyArrayObject* ary) + { + int success = 1; + int nd = array_numdims(ary); + int i; + npy_intp * strides = array_strides(ary); + if (array_is_fortran(ary)) return success; + int n_non_one = 0; + /* Set the Fortran ordered flag */ + const npy_intp *dims = array_dimensions(ary); + for (i=0; i < nd; ++i) + n_non_one += (dims[i] != 1) ? 1 : 0; + if (n_non_one > 1) + array_clearflags(ary,NPY_ARRAY_CARRAY); + array_enableflags(ary,NPY_ARRAY_FARRAY); + /* Recompute the strides */ + strides[0] = strides[nd-1]; + for (i=1; i < nd; ++i) + strides[i] = strides[i-1] * array_size(ary,i-1); + return success; + } +} + +/* Combine all NumPy fragments into one for convenience */ +%fragment("NumPy_Fragments", + "header", + fragment="NumPy_Backward_Compatibility", + fragment="NumPy_Macros", + fragment="NumPy_Utilities", + fragment="NumPy_Object_to_Array", + fragment="NumPy_Array_Requirements") +{ +} + +/* End John Hunter translation (with modifications by Bill Spotz) + */ + +/* %numpy_typemaps() macro + * + * This macro defines a family of 75 typemaps that allow C arguments + * of the form + * + * 1. (DATA_TYPE IN_ARRAY1[ANY]) + * 2. (DATA_TYPE* IN_ARRAY1, DIM_TYPE DIM1) + * 3. (DIM_TYPE DIM1, DATA_TYPE* IN_ARRAY1) + * + * 4. (DATA_TYPE IN_ARRAY2[ANY][ANY]) + * 5. (DATA_TYPE* IN_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + * 6. (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_ARRAY2) + * 7. (DATA_TYPE* IN_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + * 8. (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_FARRAY2) + * + * 9. (DATA_TYPE IN_ARRAY3[ANY][ANY][ANY]) + * 10. (DATA_TYPE* IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + * 11. (DATA_TYPE** IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + * 12. (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* IN_ARRAY3) + * 13. (DATA_TYPE* IN_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + * 14. (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* IN_FARRAY3) + * + * 15. (DATA_TYPE IN_ARRAY4[ANY][ANY][ANY][ANY]) + * 16. (DATA_TYPE* IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + * 17. (DATA_TYPE** IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + * 18. (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, , DIM_TYPE DIM4, DATA_TYPE* IN_ARRAY4) + * 19. (DATA_TYPE* IN_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + * 20. (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* IN_FARRAY4) + * + * 21. (DATA_TYPE INPLACE_ARRAY1[ANY]) + * 22. (DATA_TYPE* INPLACE_ARRAY1, DIM_TYPE DIM1) + * 23. (DIM_TYPE DIM1, DATA_TYPE* INPLACE_ARRAY1) + * + * 24. (DATA_TYPE INPLACE_ARRAY2[ANY][ANY]) + * 25. (DATA_TYPE* INPLACE_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + * 26. (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* INPLACE_ARRAY2) + * 27. (DATA_TYPE* INPLACE_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + * 28. (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* INPLACE_FARRAY2) + * + * 29. (DATA_TYPE INPLACE_ARRAY3[ANY][ANY][ANY]) + * 30. (DATA_TYPE* INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + * 31. (DATA_TYPE** INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + * 32. (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* INPLACE_ARRAY3) + * 33. (DATA_TYPE* INPLACE_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + * 34. (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* INPLACE_FARRAY3) + * + * 35. (DATA_TYPE INPLACE_ARRAY4[ANY][ANY][ANY][ANY]) + * 36. (DATA_TYPE* INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + * 37. (DATA_TYPE** INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + * 38. (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* INPLACE_ARRAY4) + * 39. (DATA_TYPE* INPLACE_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + * 40. (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* INPLACE_FARRAY4) + * + * 41. (DATA_TYPE ARGOUT_ARRAY1[ANY]) + * 42. (DATA_TYPE* ARGOUT_ARRAY1, DIM_TYPE DIM1) + * 43. (DIM_TYPE DIM1, DATA_TYPE* ARGOUT_ARRAY1) + * + * 44. (DATA_TYPE ARGOUT_ARRAY2[ANY][ANY]) + * + * 45. (DATA_TYPE ARGOUT_ARRAY3[ANY][ANY][ANY]) + * + * 46. (DATA_TYPE ARGOUT_ARRAY4[ANY][ANY][ANY][ANY]) + * + * 47. (DATA_TYPE** ARGOUTVIEW_ARRAY1, DIM_TYPE* DIM1) + * 48. (DIM_TYPE* DIM1, DATA_TYPE** ARGOUTVIEW_ARRAY1) + * + * 49. (DATA_TYPE** ARGOUTVIEW_ARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) + * 50. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEW_ARRAY2) + * 51. (DATA_TYPE** ARGOUTVIEW_FARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) + * 52. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEW_FARRAY2) + * + * 53. (DATA_TYPE** ARGOUTVIEW_ARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) + * 54. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEW_ARRAY3) + * 55. (DATA_TYPE** ARGOUTVIEW_FARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) + * 56. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEW_FARRAY3) + * + * 57. (DATA_TYPE** ARGOUTVIEW_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) + * 58. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEW_ARRAY4) + * 59. (DATA_TYPE** ARGOUTVIEW_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) + * 60. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEW_FARRAY4) + * + * 61. (DATA_TYPE** ARGOUTVIEWM_ARRAY1, DIM_TYPE* DIM1) + * 62. (DIM_TYPE* DIM1, DATA_TYPE** ARGOUTVIEWM_ARRAY1) + * + * 63. (DATA_TYPE** ARGOUTVIEWM_ARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) + * 64. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEWM_ARRAY2) + * 65. (DATA_TYPE** ARGOUTVIEWM_FARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) + * 66. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEWM_FARRAY2) + * + * 67. (DATA_TYPE** ARGOUTVIEWM_ARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) + * 68. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEWM_ARRAY3) + * 69. (DATA_TYPE** ARGOUTVIEWM_FARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) + * 70. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEWM_FARRAY3) + * + * 71. (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) + * 72. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_ARRAY4) + * 73. (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) + * 74. (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_FARRAY4) + * + * 75. (DATA_TYPE* INPLACE_ARRAY_FLAT, DIM_TYPE DIM_FLAT) + * + * where "DATA_TYPE" is any type supported by the NumPy module, and + * "DIM_TYPE" is any int-like type suitable for specifying dimensions. + * The difference between "ARRAY" typemaps and "FARRAY" typemaps is + * that the "FARRAY" typemaps expect Fortran ordering of + * multidimensional arrays. In python, the dimensions will not need + * to be specified (except for the "DATA_TYPE* ARGOUT_ARRAY1" + * typemaps). The IN_ARRAYs can be a numpy array or any sequence that + * can be converted to a numpy array of the specified type. The + * INPLACE_ARRAYs must be numpy arrays of the appropriate type. The + * ARGOUT_ARRAYs will be returned as new numpy arrays of the + * appropriate type. + * + * These typemaps can be applied to existing functions using the + * %apply directive. For example: + * + * %apply (double* IN_ARRAY1, int DIM1) {(double* series, int length)}; + * double prod(double* series, int length); + * + * %apply (int DIM1, int DIM2, double* INPLACE_ARRAY2) + * {(int rows, int cols, double* matrix )}; + * void floor(int rows, int cols, double* matrix, double f); + * + * %apply (double IN_ARRAY3[ANY][ANY][ANY]) + * {(double tensor[2][2][2] )}; + * %apply (double ARGOUT_ARRAY3[ANY][ANY][ANY]) + * {(double low[2][2][2] )}; + * %apply (double ARGOUT_ARRAY3[ANY][ANY][ANY]) + * {(double upp[2][2][2] )}; + * void luSplit(double tensor[2][2][2], + * double low[2][2][2], + * double upp[2][2][2] ); + * + * or directly with + * + * double prod(double* IN_ARRAY1, int DIM1); + * + * void floor(int DIM1, int DIM2, double* INPLACE_ARRAY2, double f); + * + * void luSplit(double IN_ARRAY3[ANY][ANY][ANY], + * double ARGOUT_ARRAY3[ANY][ANY][ANY], + * double ARGOUT_ARRAY3[ANY][ANY][ANY]); + */ + +%define %numpy_typemaps(DATA_TYPE, DATA_TYPECODE, DIM_TYPE) + +/************************/ +/* Input Array Typemaps */ +/************************/ + +/* Typemap suite for (DATA_TYPE IN_ARRAY1[ANY]) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE IN_ARRAY1[ANY]) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE IN_ARRAY1[ANY]) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[1] = { $1_dim0 }; + array = obj_to_array_contiguous_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 1) || + !require_size(array, size, 1)) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} +%typemap(freearg) + (DATA_TYPE IN_ARRAY1[ANY]) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE* IN_ARRAY1, DIM_TYPE DIM1) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* IN_ARRAY1, DIM_TYPE DIM1) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* IN_ARRAY1, DIM_TYPE DIM1) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[1] = { -1 }; + array = obj_to_array_contiguous_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 1) || + !require_size(array, size, 1)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); +} +%typemap(freearg) + (DATA_TYPE* IN_ARRAY1, DIM_TYPE DIM1) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DIM_TYPE DIM1, DATA_TYPE* IN_ARRAY1) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DATA_TYPE* IN_ARRAY1) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DATA_TYPE* IN_ARRAY1) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[1] = {-1}; + array = obj_to_array_contiguous_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 1) || + !require_size(array, size, 1)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DATA_TYPE*) array_data(array); +} +%typemap(freearg) + (DIM_TYPE DIM1, DATA_TYPE* IN_ARRAY1) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE IN_ARRAY2[ANY][ANY]) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE IN_ARRAY2[ANY][ANY]) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE IN_ARRAY2[ANY][ANY]) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[2] = { $1_dim0, $1_dim1 }; + array = obj_to_array_contiguous_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 2) || + !require_size(array, size, 2)) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} +%typemap(freearg) + (DATA_TYPE IN_ARRAY2[ANY][ANY]) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE* IN_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* IN_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* IN_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[2] = { -1, -1 }; + array = obj_to_array_contiguous_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 2) || + !require_size(array, size, 2)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); +} +%typemap(freearg) + (DATA_TYPE* IN_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_ARRAY2) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_ARRAY2) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_ARRAY2) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[2] = { -1, -1 }; + array = obj_to_array_contiguous_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 2) || + !require_size(array, size, 2)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DATA_TYPE*) array_data(array); +} +%typemap(freearg) + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_ARRAY2) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE* IN_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* IN_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* IN_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[2] = { -1, -1 }; + array = obj_to_array_fortran_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 2) || + !require_size(array, size, 2) || !require_fortran(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); +} +%typemap(freearg) + (DATA_TYPE* IN_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_FARRAY2) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_FARRAY2) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_FARRAY2) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[2] = { -1, -1 }; + array = obj_to_array_fortran_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 2) || + !require_size(array, size, 2) || !require_fortran(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DATA_TYPE*) array_data(array); +} +%typemap(freearg) + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* IN_FARRAY2) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE IN_ARRAY3[ANY][ANY][ANY]) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE IN_ARRAY3[ANY][ANY][ANY]) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE IN_ARRAY3[ANY][ANY][ANY]) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[3] = { $1_dim0, $1_dim1, $1_dim2 }; + array = obj_to_array_contiguous_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 3) || + !require_size(array, size, 3)) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} +%typemap(freearg) + (DATA_TYPE IN_ARRAY3[ANY][ANY][ANY]) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE* IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[3] = { -1, -1, -1 }; + array = obj_to_array_contiguous_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 3) || + !require_size(array, size, 3)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); + $4 = (DIM_TYPE) array_size(array,2); +} +%typemap(freearg) + (DATA_TYPE* IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE** IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE** IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + /* for now, only concerned with lists */ + $1 = PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE** IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + (DATA_TYPE** array=NULL, PyArrayObject** object_array=NULL, int* is_new_object_array=NULL) +{ + npy_intp size[2] = { -1, -1 }; + PyArrayObject* temp_array; + Py_ssize_t i; + int is_new_object; + + /* length of the list */ + $2 = PyList_Size($input); + + /* the arrays */ + array = (DATA_TYPE **)malloc($2*sizeof(DATA_TYPE *)); + object_array = (PyArrayObject **)calloc($2,sizeof(PyArrayObject *)); + is_new_object_array = (int *)calloc($2,sizeof(int)); + + if (array == NULL || object_array == NULL || is_new_object_array == NULL) + { + SWIG_fail; + } + + for (i=0; i<$2; i++) + { + temp_array = obj_to_array_contiguous_allow_conversion(PySequence_GetItem($input,i), DATA_TYPECODE, &is_new_object); + + /* the new array must be stored so that it can be destroyed in freearg */ + object_array[i] = temp_array; + is_new_object_array[i] = is_new_object; + + if (!temp_array || !require_dimensions(temp_array, 2)) SWIG_fail; + + /* store the size of the first array in the list, then use that for comparison. */ + if (i == 0) + { + size[0] = array_size(temp_array,0); + size[1] = array_size(temp_array,1); + } + + if (!require_size(temp_array, size, 2)) SWIG_fail; + + array[i] = (DATA_TYPE*) array_data(temp_array); + } + + $1 = (DATA_TYPE**) array; + $3 = (DIM_TYPE) size[0]; + $4 = (DIM_TYPE) size[1]; +} +%typemap(freearg) + (DATA_TYPE** IN_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + Py_ssize_t i; + + if (array$argnum!=NULL) free(array$argnum); + + /*freeing the individual arrays if needed */ + if (object_array$argnum!=NULL) + { + if (is_new_object_array$argnum!=NULL) + { + for (i=0; i<$2; i++) + { + if (object_array$argnum[i] != NULL && is_new_object_array$argnum[i]) + { Py_DECREF(object_array$argnum[i]); } + } + free(is_new_object_array$argnum); + } + free(object_array$argnum); + } +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, + * DATA_TYPE* IN_ARRAY3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* IN_ARRAY3) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* IN_ARRAY3) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[3] = { -1, -1, -1 }; + array = obj_to_array_contiguous_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 3) || + !require_size(array, size, 3)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DIM_TYPE) array_size(array,2); + $4 = (DATA_TYPE*) array_data(array); +} +%typemap(freearg) + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* IN_ARRAY3) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE* IN_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* IN_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* IN_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[3] = { -1, -1, -1 }; + array = obj_to_array_fortran_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 3) || + !require_size(array, size, 3) | !require_fortran(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); + $4 = (DIM_TYPE) array_size(array,2); +} +%typemap(freearg) + (DATA_TYPE* IN_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, + * DATA_TYPE* IN_FARRAY3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* IN_FARRAY3) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* IN_FARRAY3) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[3] = { -1, -1, -1 }; + array = obj_to_array_fortran_allow_conversion($input, + DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 3) || + !require_size(array, size, 3) || !require_fortran(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DIM_TYPE) array_size(array,2); + $4 = (DATA_TYPE*) array_data(array); +} +%typemap(freearg) + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* IN_FARRAY3) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE IN_ARRAY4[ANY][ANY][ANY][ANY]) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE IN_ARRAY4[ANY][ANY][ANY][ANY]) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE IN_ARRAY4[ANY][ANY][ANY][ANY]) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[4] = { $1_dim0, $1_dim1, $1_dim2 , $1_dim3}; + array = obj_to_array_contiguous_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 4) || + !require_size(array, size, 4)) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} +%typemap(freearg) + (DATA_TYPE IN_ARRAY4[ANY][ANY][ANY][ANY]) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE* IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3, DIM_TYPE DIM4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[4] = { -1, -1, -1, -1 }; + array = obj_to_array_contiguous_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 4) || + !require_size(array, size, 4)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); + $4 = (DIM_TYPE) array_size(array,2); + $5 = (DIM_TYPE) array_size(array,3); +} +%typemap(freearg) + (DATA_TYPE* IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE** IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3, DIM_TYPE DIM4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE** IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + /* for now, only concerned with lists */ + $1 = PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE** IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + (DATA_TYPE** array=NULL, PyArrayObject** object_array=NULL, int* is_new_object_array=NULL) +{ + npy_intp size[3] = { -1, -1, -1 }; + PyArrayObject* temp_array; + Py_ssize_t i; + int is_new_object; + + /* length of the list */ + $2 = PyList_Size($input); + + /* the arrays */ + array = (DATA_TYPE **)malloc($2*sizeof(DATA_TYPE *)); + object_array = (PyArrayObject **)calloc($2,sizeof(PyArrayObject *)); + is_new_object_array = (int *)calloc($2,sizeof(int)); + + if (array == NULL || object_array == NULL || is_new_object_array == NULL) + { + SWIG_fail; + } + + for (i=0; i<$2; i++) + { + temp_array = obj_to_array_contiguous_allow_conversion(PySequence_GetItem($input,i), DATA_TYPECODE, &is_new_object); + + /* the new array must be stored so that it can be destroyed in freearg */ + object_array[i] = temp_array; + is_new_object_array[i] = is_new_object; + + if (!temp_array || !require_dimensions(temp_array, 3)) SWIG_fail; + + /* store the size of the first array in the list, then use that for comparison. */ + if (i == 0) + { + size[0] = array_size(temp_array,0); + size[1] = array_size(temp_array,1); + size[2] = array_size(temp_array,2); + } + + if (!require_size(temp_array, size, 3)) SWIG_fail; + + array[i] = (DATA_TYPE*) array_data(temp_array); + } + + $1 = (DATA_TYPE**) array; + $3 = (DIM_TYPE) size[0]; + $4 = (DIM_TYPE) size[1]; + $5 = (DIM_TYPE) size[2]; +} +%typemap(freearg) + (DATA_TYPE** IN_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + Py_ssize_t i; + + if (array$argnum!=NULL) free(array$argnum); + + /*freeing the individual arrays if needed */ + if (object_array$argnum!=NULL) + { + if (is_new_object_array$argnum!=NULL) + { + for (i=0; i<$2; i++) + { + if (object_array$argnum[i] != NULL && is_new_object_array$argnum[i]) + { Py_DECREF(object_array$argnum[i]); } + } + free(is_new_object_array$argnum); + } + free(object_array$argnum); + } +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, + * DATA_TYPE* IN_ARRAY4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* IN_ARRAY4) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* IN_ARRAY4) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[4] = { -1, -1, -1 , -1}; + array = obj_to_array_contiguous_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 4) || + !require_size(array, size, 4)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DIM_TYPE) array_size(array,2); + $4 = (DIM_TYPE) array_size(array,3); + $5 = (DATA_TYPE*) array_data(array); +} +%typemap(freearg) + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* IN_ARRAY4) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DATA_TYPE* IN_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3, DIM_TYPE DIM4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* IN_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* IN_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[4] = { -1, -1, -1, -1 }; + array = obj_to_array_fortran_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 4) || + !require_size(array, size, 4) | !require_fortran(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); + $4 = (DIM_TYPE) array_size(array,2); + $5 = (DIM_TYPE) array_size(array,3); +} +%typemap(freearg) + (DATA_TYPE* IN_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, + * DATA_TYPE* IN_FARRAY4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* IN_FARRAY4) +{ + $1 = is_array($input) || PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* IN_FARRAY4) + (PyArrayObject* array=NULL, int is_new_object=0) +{ + npy_intp size[4] = { -1, -1, -1 , -1 }; + array = obj_to_array_fortran_allow_conversion($input, DATA_TYPECODE, + &is_new_object); + if (!array || !require_dimensions(array, 4) || + !require_size(array, size, 4) || !require_fortran(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DIM_TYPE) array_size(array,2); + $4 = (DIM_TYPE) array_size(array,3); + $5 = (DATA_TYPE*) array_data(array); +} +%typemap(freearg) + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* IN_FARRAY4) +{ + if (is_new_object$argnum && array$argnum) + { Py_DECREF(array$argnum); } +} + +/***************************/ +/* In-Place Array Typemaps */ +/***************************/ + +/* Typemap suite for (DATA_TYPE INPLACE_ARRAY1[ANY]) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE INPLACE_ARRAY1[ANY]) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE INPLACE_ARRAY1[ANY]) + (PyArrayObject* array=NULL) +{ + npy_intp size[1] = { $1_dim0 }; + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,1) || !require_size(array, size, 1) || + !require_contiguous(array) || !require_native(array)) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} + +/* Typemap suite for (DATA_TYPE* INPLACE_ARRAY1, DIM_TYPE DIM1) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* INPLACE_ARRAY1, DIM_TYPE DIM1) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* INPLACE_ARRAY1, DIM_TYPE DIM1) + (PyArrayObject* array=NULL, int i=1) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,1) || !require_contiguous(array) + || !require_native(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = 1; + for (i=0; i < array_numdims(array); ++i) $2 *= array_size(array,i); +} + +/* Typemap suite for (DIM_TYPE DIM1, DATA_TYPE* INPLACE_ARRAY1) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DATA_TYPE* INPLACE_ARRAY1) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DATA_TYPE* INPLACE_ARRAY1) + (PyArrayObject* array=NULL, int i=0) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,1) || !require_contiguous(array) + || !require_native(array)) SWIG_fail; + $1 = 1; + for (i=0; i < array_numdims(array); ++i) $1 *= array_size(array,i); + $2 = (DATA_TYPE*) array_data(array); +} + +/* Typemap suite for (DATA_TYPE INPLACE_ARRAY2[ANY][ANY]) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE INPLACE_ARRAY2[ANY][ANY]) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE INPLACE_ARRAY2[ANY][ANY]) + (PyArrayObject* array=NULL) +{ + npy_intp size[2] = { $1_dim0, $1_dim1 }; + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,2) || !require_size(array, size, 2) || + !require_contiguous(array) || !require_native(array)) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} + +/* Typemap suite for (DATA_TYPE* INPLACE_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* INPLACE_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* INPLACE_ARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,2) || !require_contiguous(array) + || !require_native(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* INPLACE_ARRAY2) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* INPLACE_ARRAY2) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* INPLACE_ARRAY2) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,2) || !require_contiguous(array) || + !require_native(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DATA_TYPE*) array_data(array); +} + +/* Typemap suite for (DATA_TYPE* INPLACE_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* INPLACE_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* INPLACE_FARRAY2, DIM_TYPE DIM1, DIM_TYPE DIM2) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,2) || !require_contiguous(array) + || !require_native(array) || !require_fortran(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* INPLACE_FARRAY2) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* INPLACE_FARRAY2) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DATA_TYPE* INPLACE_FARRAY2) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,2) || !require_contiguous(array) || + !require_native(array) || !require_fortran(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DATA_TYPE*) array_data(array); +} + +/* Typemap suite for (DATA_TYPE INPLACE_ARRAY3[ANY][ANY][ANY]) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE INPLACE_ARRAY3[ANY][ANY][ANY]) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE INPLACE_ARRAY3[ANY][ANY][ANY]) + (PyArrayObject* array=NULL) +{ + npy_intp size[3] = { $1_dim0, $1_dim1, $1_dim2 }; + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,3) || !require_size(array, size, 3) || + !require_contiguous(array) || !require_native(array)) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} + +/* Typemap suite for (DATA_TYPE* INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,3) || !require_contiguous(array) || + !require_native(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); + $4 = (DIM_TYPE) array_size(array,2); +} + +/* Typemap suite for (DATA_TYPE** INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE** INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + $1 = PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE** INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + (DATA_TYPE** array=NULL, PyArrayObject** object_array=NULL) +{ + npy_intp size[2] = { -1, -1 }; + PyArrayObject* temp_array; + Py_ssize_t i; + + /* length of the list */ + $2 = PyList_Size($input); + + /* the arrays */ + array = (DATA_TYPE **)malloc($2*sizeof(DATA_TYPE *)); + object_array = (PyArrayObject **)calloc($2,sizeof(PyArrayObject *)); + + if (array == NULL || object_array == NULL) + { + SWIG_fail; + } + + for (i=0; i<$2; i++) + { + temp_array = obj_to_array_no_conversion(PySequence_GetItem($input,i), DATA_TYPECODE); + + /* the new array must be stored so that it can be destroyed in freearg */ + object_array[i] = temp_array; + + if ( !temp_array || !require_dimensions(temp_array, 2) || + !require_contiguous(temp_array) || + !require_native(temp_array) || + !PyArray_EquivTypenums(array_type(temp_array), DATA_TYPECODE) + ) SWIG_fail; + + /* store the size of the first array in the list, then use that for comparison. */ + if (i == 0) + { + size[0] = array_size(temp_array,0); + size[1] = array_size(temp_array,1); + } + + if (!require_size(temp_array, size, 2)) SWIG_fail; + + array[i] = (DATA_TYPE*) array_data(temp_array); + } + + $1 = (DATA_TYPE**) array; + $3 = (DIM_TYPE) size[0]; + $4 = (DIM_TYPE) size[1]; +} +%typemap(freearg) + (DATA_TYPE** INPLACE_ARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + if (array$argnum!=NULL) free(array$argnum); + if (object_array$argnum!=NULL) free(object_array$argnum); +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, + * DATA_TYPE* INPLACE_ARRAY3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* INPLACE_ARRAY3) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* INPLACE_ARRAY3) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,3) || !require_contiguous(array) + || !require_native(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DIM_TYPE) array_size(array,2); + $4 = (DATA_TYPE*) array_data(array); +} + +/* Typemap suite for (DATA_TYPE* INPLACE_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* INPLACE_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* INPLACE_FARRAY3, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,3) || !require_contiguous(array) || + !require_native(array) || !require_fortran(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); + $4 = (DIM_TYPE) array_size(array,2); +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, + * DATA_TYPE* INPLACE_FARRAY3) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* INPLACE_FARRAY3) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DATA_TYPE* INPLACE_FARRAY3) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,3) || !require_contiguous(array) + || !require_native(array) || !require_fortran(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DIM_TYPE) array_size(array,2); + $4 = (DATA_TYPE*) array_data(array); +} + +/* Typemap suite for (DATA_TYPE INPLACE_ARRAY4[ANY][ANY][ANY][ANY]) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE INPLACE_ARRAY4[ANY][ANY][ANY][ANY]) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE INPLACE_ARRAY4[ANY][ANY][ANY][ANY]) + (PyArrayObject* array=NULL) +{ + npy_intp size[4] = { $1_dim0, $1_dim1, $1_dim2 , $1_dim3 }; + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,4) || !require_size(array, size, 4) || + !require_contiguous(array) || !require_native(array)) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} + +/* Typemap suite for (DATA_TYPE* INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3, DIM_TYPE DIM4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,4) || !require_contiguous(array) || + !require_native(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); + $4 = (DIM_TYPE) array_size(array,2); + $5 = (DIM_TYPE) array_size(array,3); +} + +/* Typemap suite for (DATA_TYPE** INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3, DIM_TYPE DIM4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE** INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + $1 = PySequence_Check($input); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE** INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + (DATA_TYPE** array=NULL, PyArrayObject** object_array=NULL) +{ + npy_intp size[3] = { -1, -1, -1 }; + PyArrayObject* temp_array; + Py_ssize_t i; + + /* length of the list */ + $2 = PyList_Size($input); + + /* the arrays */ + array = (DATA_TYPE **)malloc($2*sizeof(DATA_TYPE *)); + object_array = (PyArrayObject **)calloc($2,sizeof(PyArrayObject *)); + + if (array == NULL || object_array == NULL) + { + SWIG_fail; + } + + for (i=0; i<$2; i++) + { + temp_array = obj_to_array_no_conversion(PySequence_GetItem($input,i), DATA_TYPECODE); + + /* the new array must be stored so that it can be destroyed in freearg */ + object_array[i] = temp_array; + + if ( !temp_array || !require_dimensions(temp_array, 3) || + !require_contiguous(temp_array) || + !require_native(temp_array) || + !PyArray_EquivTypenums(array_type(temp_array), DATA_TYPECODE) + ) SWIG_fail; + + /* store the size of the first array in the list, then use that for comparison. */ + if (i == 0) + { + size[0] = array_size(temp_array,0); + size[1] = array_size(temp_array,1); + size[2] = array_size(temp_array,2); + } + + if (!require_size(temp_array, size, 3)) SWIG_fail; + + array[i] = (DATA_TYPE*) array_data(temp_array); + } + + $1 = (DATA_TYPE**) array; + $3 = (DIM_TYPE) size[0]; + $4 = (DIM_TYPE) size[1]; + $5 = (DIM_TYPE) size[2]; +} +%typemap(freearg) + (DATA_TYPE** INPLACE_ARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + if (array$argnum!=NULL) free(array$argnum); + if (object_array$argnum!=NULL) free(object_array$argnum); +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, + * DATA_TYPE* INPLACE_ARRAY4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* INPLACE_ARRAY4) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* INPLACE_ARRAY4) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,4) || !require_contiguous(array) + || !require_native(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DIM_TYPE) array_size(array,2); + $4 = (DIM_TYPE) array_size(array,3); + $5 = (DATA_TYPE*) array_data(array); +} + +/* Typemap suite for (DATA_TYPE* INPLACE_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, + * DIM_TYPE DIM3, DIM_TYPE DIM4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* INPLACE_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* INPLACE_FARRAY4, DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,4) || !require_contiguous(array) || + !require_native(array) || !require_fortran(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = (DIM_TYPE) array_size(array,0); + $3 = (DIM_TYPE) array_size(array,1); + $4 = (DIM_TYPE) array_size(array,2); + $5 = (DIM_TYPE) array_size(array,3); +} + +/* Typemap suite for (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, + * DATA_TYPE* INPLACE_FARRAY4) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* INPLACE_FARRAY4) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DIM_TYPE DIM2, DIM_TYPE DIM3, DIM_TYPE DIM4, DATA_TYPE* INPLACE_FARRAY4) + (PyArrayObject* array=NULL) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_dimensions(array,4) || !require_contiguous(array) + || !require_native(array) || !require_fortran(array)) SWIG_fail; + $1 = (DIM_TYPE) array_size(array,0); + $2 = (DIM_TYPE) array_size(array,1); + $3 = (DIM_TYPE) array_size(array,2); + $4 = (DIM_TYPE) array_size(array,3); + $5 = (DATA_TYPE*) array_data(array); +} + +/*************************/ +/* Argout Array Typemaps */ +/*************************/ + +/* Typemap suite for (DATA_TYPE ARGOUT_ARRAY1[ANY]) + */ +%typemap(in,numinputs=0, + fragment="NumPy_Backward_Compatibility,NumPy_Macros") + (DATA_TYPE ARGOUT_ARRAY1[ANY]) + (PyObject* array = NULL) +{ + npy_intp dims[1] = { $1_dim0 }; + array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); + if (!array) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} +%typemap(argout) + (DATA_TYPE ARGOUT_ARRAY1[ANY]) +{ + $result = SWIG_Python_AppendOutput($result,(PyObject*)array$argnum); +} + +/* Typemap suite for (DATA_TYPE* ARGOUT_ARRAY1, DIM_TYPE DIM1) + */ +%typemap(in,numinputs=1, + fragment="NumPy_Fragments") + (DATA_TYPE* ARGOUT_ARRAY1, DIM_TYPE DIM1) + (PyObject* array = NULL) +{ + npy_intp dims[1]; + if (!PyLong_Check($input)) + { + const char* typestring = pytype_string($input); + PyErr_Format(PyExc_TypeError, + "Int dimension expected. '%s' given.", + typestring); + SWIG_fail; + } + $2 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($2 == -1 && PyErr_Occurred()) SWIG_fail; + dims[0] = (npy_intp) $2; + array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); + if (!array) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); +} +%typemap(argout) + (DATA_TYPE* ARGOUT_ARRAY1, DIM_TYPE DIM1) +{ + $result = SWIG_Python_AppendOutput($result,(PyObject*)array$argnum); +} + +/* Typemap suite for (DIM_TYPE DIM1, DATA_TYPE* ARGOUT_ARRAY1) + */ +%typemap(in,numinputs=1, + fragment="NumPy_Fragments") + (DIM_TYPE DIM1, DATA_TYPE* ARGOUT_ARRAY1) + (PyObject* array = NULL) +{ + npy_intp dims[1]; + if (!PyLong_Check($input)) + { + const char* typestring = pytype_string($input); + PyErr_Format(PyExc_TypeError, + "Int dimension expected. '%s' given.", + typestring); + SWIG_fail; + } + $1 = (DIM_TYPE) PyLong_AsSsize_t($input); + if ($1 == -1 && PyErr_Occurred()) SWIG_fail; + dims[0] = (npy_intp) $1; + array = PyArray_SimpleNew(1, dims, DATA_TYPECODE); + if (!array) SWIG_fail; + $2 = (DATA_TYPE*) array_data(array); +} +%typemap(argout) + (DIM_TYPE DIM1, DATA_TYPE* ARGOUT_ARRAY1) +{ + $result = SWIG_Python_AppendOutput($result,(PyObject*)array$argnum); +} + +/* Typemap suite for (DATA_TYPE ARGOUT_ARRAY2[ANY][ANY]) + */ +%typemap(in,numinputs=0, + fragment="NumPy_Backward_Compatibility,NumPy_Macros") + (DATA_TYPE ARGOUT_ARRAY2[ANY][ANY]) + (PyObject* array = NULL) +{ + npy_intp dims[2] = { $1_dim0, $1_dim1 }; + array = PyArray_SimpleNew(2, dims, DATA_TYPECODE); + if (!array) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} +%typemap(argout) + (DATA_TYPE ARGOUT_ARRAY2[ANY][ANY]) +{ + $result = SWIG_Python_AppendOutput($result,(PyObject*)array$argnum); +} + +/* Typemap suite for (DATA_TYPE ARGOUT_ARRAY3[ANY][ANY][ANY]) + */ +%typemap(in,numinputs=0, + fragment="NumPy_Backward_Compatibility,NumPy_Macros") + (DATA_TYPE ARGOUT_ARRAY3[ANY][ANY][ANY]) + (PyObject* array = NULL) +{ + npy_intp dims[3] = { $1_dim0, $1_dim1, $1_dim2 }; + array = PyArray_SimpleNew(3, dims, DATA_TYPECODE); + if (!array) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} +%typemap(argout) + (DATA_TYPE ARGOUT_ARRAY3[ANY][ANY][ANY]) +{ + $result = SWIG_Python_AppendOutput($result,(PyObject*)array$argnum); +} + +/* Typemap suite for (DATA_TYPE ARGOUT_ARRAY4[ANY][ANY][ANY][ANY]) + */ +%typemap(in,numinputs=0, + fragment="NumPy_Backward_Compatibility,NumPy_Macros") + (DATA_TYPE ARGOUT_ARRAY4[ANY][ANY][ANY][ANY]) + (PyObject* array = NULL) +{ + npy_intp dims[4] = { $1_dim0, $1_dim1, $1_dim2, $1_dim3 }; + array = PyArray_SimpleNew(4, dims, DATA_TYPECODE); + if (!array) SWIG_fail; + $1 = ($1_ltype) array_data(array); +} +%typemap(argout) + (DATA_TYPE ARGOUT_ARRAY4[ANY][ANY][ANY][ANY]) +{ + $result = SWIG_Python_AppendOutput($result,(PyObject*)array$argnum); +} + +/*****************************/ +/* Argoutview Array Typemaps */ +/*****************************/ + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEW_ARRAY1, DIM_TYPE* DIM1) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEW_ARRAY1, DIM_TYPE* DIM1 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim_temp) +{ + $1 = &data_temp; + $2 = &dim_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility") + (DATA_TYPE** ARGOUTVIEW_ARRAY1, DIM_TYPE* DIM1) +{ + npy_intp dims[1] = { *$2 }; + PyObject* obj = PyArray_SimpleNewFromData(1, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DATA_TYPE** ARGOUTVIEW_ARRAY1) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DATA_TYPE** ARGOUTVIEW_ARRAY1) + (DIM_TYPE dim_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim_temp; + $2 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility") + (DIM_TYPE* DIM1, DATA_TYPE** ARGOUTVIEW_ARRAY1) +{ + npy_intp dims[1] = { *$1 }; + PyObject* obj = PyArray_SimpleNewFromData(1, dims, DATA_TYPECODE, (void*)(*$2)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEW_ARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEW_ARRAY2, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility") + (DATA_TYPE** ARGOUTVIEW_ARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) +{ + npy_intp dims[2] = { *$2, *$3 }; + PyObject* obj = PyArray_SimpleNewFromData(2, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEW_ARRAY2) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DATA_TYPE** ARGOUTVIEW_ARRAY2) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEW_ARRAY2) +{ + npy_intp dims[2] = { *$1, *$2 }; + PyObject* obj = PyArray_SimpleNewFromData(2, dims, DATA_TYPECODE, (void*)(*$3)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEW_FARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEW_FARRAY2, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements") + (DATA_TYPE** ARGOUTVIEW_FARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) +{ + npy_intp dims[2] = { *$2, *$3 }; + PyObject* obj = PyArray_SimpleNewFromData(2, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEW_FARRAY2) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DATA_TYPE** ARGOUTVIEW_FARRAY2) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEW_FARRAY2) +{ + npy_intp dims[2] = { *$1, *$2 }; + PyObject* obj = PyArray_SimpleNewFromData(2, dims, DATA_TYPECODE, (void*)(*$3)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEW_ARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, + DIM_TYPE* DIM3) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEW_ARRAY3, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; + $4 = &dim3_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility") + (DATA_TYPE** ARGOUTVIEW_ARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) +{ + npy_intp dims[3] = { *$2, *$3, *$4 }; + PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, + DATA_TYPE** ARGOUTVIEW_ARRAY3) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEW_ARRAY3) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DATA_TYPE* data_temp = NULL) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &dim3_temp; + $4 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEW_ARRAY3) +{ + npy_intp dims[3] = { *$1, *$2, *$3 }; + PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$4)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEW_FARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, + DIM_TYPE* DIM3) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEW_FARRAY3, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; + $4 = &dim3_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements") + (DATA_TYPE** ARGOUTVIEW_FARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) +{ + npy_intp dims[3] = { *$2, *$3, *$4 }; + PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, + DATA_TYPE** ARGOUTVIEW_FARRAY3) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DATA_TYPE** ARGOUTVIEW_FARRAY3) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &dim3_temp; + $4 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEW_FARRAY3) +{ + npy_intp dims[3] = { *$1, *$2, *$3 }; + PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$4)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEW_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, + DIM_TYPE* DIM3, DIM_TYPE* DIM4) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEW_ARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; + $4 = &dim3_temp; + $5 = &dim4_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility") + (DATA_TYPE** ARGOUTVIEW_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) +{ + npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; + PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, + DATA_TYPE** ARGOUTVIEW_ARRAY4) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEW_ARRAY4) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &dim3_temp; + $4 = &dim4_temp; + $5 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEW_ARRAY4) +{ + npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; + PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEW_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, + DIM_TYPE* DIM3, DIM_TYPE* DIM4) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEW_FARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; + $4 = &dim3_temp; + $5 = &dim4_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements") + (DATA_TYPE** ARGOUTVIEW_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) +{ + npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; + PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, + DATA_TYPE** ARGOUTVIEW_FARRAY4) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEW_FARRAY4) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &dim3_temp; + $4 = &dim4_temp; + $5 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEW_FARRAY4) +{ + npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; + PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + $result = SWIG_Python_AppendOutput($result,obj); +} + +/*************************************/ +/* Managed Argoutview Array Typemaps */ +/*************************************/ + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_ARRAY1, DIM_TYPE* DIM1) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEWM_ARRAY1, DIM_TYPE* DIM1 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim_temp) +{ + $1 = &data_temp; + $2 = &dim_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Utilities") + (DATA_TYPE** ARGOUTVIEWM_ARRAY1, DIM_TYPE* DIM1) +{ + npy_intp dims[1] = { *$2 }; + PyObject* obj = PyArray_SimpleNewFromData(1, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DATA_TYPE** ARGOUTVIEWM_ARRAY1) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DATA_TYPE** ARGOUTVIEWM_ARRAY1) + (DIM_TYPE dim_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim_temp; + $2 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Utilities") + (DIM_TYPE* DIM1, DATA_TYPE** ARGOUTVIEWM_ARRAY1) +{ + npy_intp dims[1] = { *$1 }; + PyObject* obj = PyArray_SimpleNewFromData(1, dims, DATA_TYPECODE, (void*)(*$2)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$2), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$2), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_ARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEWM_ARRAY2, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Utilities") + (DATA_TYPE** ARGOUTVIEWM_ARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) +{ + npy_intp dims[2] = { *$2, *$3 }; + PyObject* obj = PyArray_SimpleNewFromData(2, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEWM_ARRAY2) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DATA_TYPE** ARGOUTVIEWM_ARRAY2) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Utilities") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEWM_ARRAY2) +{ + npy_intp dims[2] = { *$1, *$2 }; + PyObject* obj = PyArray_SimpleNewFromData(2, dims, DATA_TYPECODE, (void*)(*$3)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_FARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEWM_FARRAY2, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") + (DATA_TYPE** ARGOUTVIEWM_FARRAY2, DIM_TYPE* DIM1, DIM_TYPE* DIM2) +{ + npy_intp dims[2] = { *$2, *$3 }; + PyObject* obj = PyArray_SimpleNewFromData(2, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEWM_FARRAY2) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DATA_TYPE** ARGOUTVIEWM_FARRAY2) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DATA_TYPE** ARGOUTVIEWM_FARRAY2) +{ + npy_intp dims[2] = { *$1, *$2 }; + PyObject* obj = PyArray_SimpleNewFromData(2, dims, DATA_TYPECODE, (void*)(*$3)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$3), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$3), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_ARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, + DIM_TYPE* DIM3) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEWM_ARRAY3, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; + $4 = &dim3_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Utilities") + (DATA_TYPE** ARGOUTVIEWM_ARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) +{ + npy_intp dims[3] = { *$2, *$3, *$4 }; + PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, + DATA_TYPE** ARGOUTVIEWM_ARRAY3) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DATA_TYPE** ARGOUTVIEWM_ARRAY3) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &dim3_temp; + $4 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Utilities") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEWM_ARRAY3) +{ + npy_intp dims[3] = { *$1, *$2, *$3 }; + PyObject* obj= PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$4)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_FARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, + DIM_TYPE* DIM3) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEWM_FARRAY3, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; + $4 = &dim3_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") + (DATA_TYPE** ARGOUTVIEWM_FARRAY3, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) +{ + npy_intp dims[3] = { *$2, *$3, *$4 }; + PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, + DATA_TYPE** ARGOUTVIEWM_FARRAY3) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DATA_TYPE** ARGOUTVIEWM_FARRAY3) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &dim3_temp; + $4 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DATA_TYPE** ARGOUTVIEWM_FARRAY3) +{ + npy_intp dims[3] = { *$1, *$2, *$3 }; + PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$4)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$4), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$4), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, + DIM_TYPE* DIM3, DIM_TYPE* DIM4) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; + $4 = &dim3_temp; + $5 = &dim4_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Utilities") + (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) +{ + npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; + PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, + DATA_TYPE** ARGOUTVIEWM_ARRAY4) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEWM_ARRAY4) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &dim3_temp; + $4 = &dim4_temp; + $5 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Utilities") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_ARRAY4) +{ + npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; + PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, + DIM_TYPE* DIM3, DIM_TYPE* DIM4) + */ +%typemap(in,numinputs=0) + (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) + (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) +{ + $1 = &data_temp; + $2 = &dim1_temp; + $3 = &dim2_temp; + $4 = &dim3_temp; + $5 = &dim4_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") + (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) +{ + npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; + PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, + DATA_TYPE** ARGOUTVIEWM_FARRAY4) + */ +%typemap(in,numinputs=0) + (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEWM_FARRAY4) + (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) +{ + $1 = &dim1_temp; + $2 = &dim2_temp; + $3 = &dim3_temp; + $4 = &dim4_temp; + $5 = &data_temp; +} +%typemap(argout, + fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") + (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_FARRAY4) +{ + npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; + PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); + PyArrayObject* array = (PyArrayObject*) obj; + + if (!array || !require_fortran(array)) SWIG_fail; + +%#ifdef SWIGPY_USE_CAPSULE + PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); +%#else + PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); +%#endif + +%#if NPY_API_VERSION < 0x00000007 + PyArray_BASE(array) = cap; +%#else + PyArray_SetBaseObject(array,cap); +%#endif + + $result = SWIG_Python_AppendOutput($result,obj); +} + +/**************************************/ +/* In-Place Array Typemap - flattened */ +/**************************************/ + +/* Typemap suite for (DATA_TYPE* INPLACE_ARRAY_FLAT, DIM_TYPE DIM_FLAT) + */ +%typecheck(SWIG_TYPECHECK_DOUBLE_ARRAY, + fragment="NumPy_Macros") + (DATA_TYPE* INPLACE_ARRAY_FLAT, DIM_TYPE DIM_FLAT) +{ + $1 = is_array($input) && PyArray_EquivTypenums(array_type($input), + DATA_TYPECODE); +} +%typemap(in, + fragment="NumPy_Fragments") + (DATA_TYPE* INPLACE_ARRAY_FLAT, DIM_TYPE DIM_FLAT) + (PyArrayObject* array=NULL, int i=1) +{ + array = obj_to_array_no_conversion($input, DATA_TYPECODE); + if (!array || !require_c_or_f_contiguous(array) + || !require_native(array)) SWIG_fail; + $1 = (DATA_TYPE*) array_data(array); + $2 = 1; + for (i=0; i < array_numdims(array); ++i) $2 *= array_size(array,i); +} + +%enddef /* %numpy_typemaps() macro */ +/* *************************************************************** */ + +/* Concrete instances of the %numpy_typemaps() macro: Each invocation + * below applies all of the typemaps above to the specified data type. + */ +%numpy_typemaps(signed char , NPY_BYTE , int) +%numpy_typemaps(unsigned char , NPY_UBYTE , int) +%numpy_typemaps(short , NPY_SHORT , int) +%numpy_typemaps(unsigned short , NPY_USHORT , int) +%numpy_typemaps(int , NPY_INT , int) +%numpy_typemaps(unsigned int , NPY_UINT , int) +%numpy_typemaps(long , NPY_LONG , int) +%numpy_typemaps(unsigned long , NPY_ULONG , int) +%numpy_typemaps(long long , NPY_LONGLONG , int) +%numpy_typemaps(unsigned long long, NPY_ULONGLONG, int) +%numpy_typemaps(float , NPY_FLOAT , int) +%numpy_typemaps(double , NPY_DOUBLE , int) +%numpy_typemaps(int8_t , NPY_INT8 , int) +%numpy_typemaps(int16_t , NPY_INT16 , int) +%numpy_typemaps(int32_t , NPY_INT32 , int) +%numpy_typemaps(int64_t , NPY_INT64 , int) +%numpy_typemaps(uint8_t , NPY_UINT8 , int) +%numpy_typemaps(uint16_t , NPY_UINT16 , int) +%numpy_typemaps(uint32_t , NPY_UINT32 , int) +%numpy_typemaps(uint64_t , NPY_UINT64 , int) + + +/* *************************************************************** + * The follow macro expansion does not work, because C++ bool is 4 + * bytes and NPY_BOOL is 1 byte + * + * %numpy_typemaps(bool, NPY_BOOL, int) + */ + +/* *************************************************************** + * On my Mac, I get the following warning for this macro expansion: + * 'swig/python detected a memory leak of type 'long double *', no destructor found.' + * + * %numpy_typemaps(long double, NPY_LONGDOUBLE, int) + */ + +#ifdef __cplusplus + +%include + +%numpy_typemaps(std::complex, NPY_CFLOAT , int) +%numpy_typemaps(std::complex, NPY_CDOUBLE, int) + +#endif + +#endif /* SWIGPYTHON */ diff --git a/tools/prepare_python.sh b/tools/prepare_python.sh new file mode 100755 index 0000000..0e66a6c --- /dev/null +++ b/tools/prepare_python.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Check that script is executed from tools directory +if [[ $(basename $PWD) != "tools" ]] ; then + echo "This script should run in the tools directory" + exit -1 +fi + +TREXIO_ROOT=$(dirname "${PWD}../") + +# First define readonly global variables. +readonly SRC=${TREXIO_ROOT}/src +readonly INCLUDIR=${TREXIO_ROOT}/include +readonly TOOLS=${TREXIO_ROOT}/tools +readonly PYDIR=${TREXIO_ROOT}/python + +# We want the script to crash on the 1st error: +set -e + +# Create src directory in the python folder if not yet done +mkdir -p ${PYDIR}/src + +# Copy all the source code and header files in the corresponding python directory +cp ${SRC}/*.py ${PYDIR} +cp ${SRC}/*.c ${PYDIR}/src +cp ${SRC}/*.h ${PYDIR}/src +cp ${INCLUDIR}/trexio.h ${PYDIR}/src + +# Copy additional info +cp ${TREXIO_ROOT}/AUTHORS ${TREXIO_ROOT}/LICENSE ${TREXIO_ROOT}/README.md ${PYDIR} + From 1388ddef8087654f97c6eb39db32d6a2ed361acc Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 6 Aug 2021 12:06:42 +0300 Subject: [PATCH 033/107] fix warnings produced during compilation of the Python extension module --- src/pytrexio.i | 2 +- src/templates_front/templator_front.org | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index 47f60a4..78a5e43 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -60,7 +60,7 @@ import_array(); %} -%apply (double* ARGOUT_ARRAY1, int DIM1) {(double * const dset_out, const uint64_t dim_out)}; +%apply (double* ARGOUT_ARRAY1, int DIM1) {(double * const dset_out, const int64_t dim_out)}; /* This tells SWIG to treat char ** dset_in pattern as a special case Enables access to trexio_[...]_write_dset_str set of functions directly, i.e. diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index f6d3072..d29258d 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -539,7 +539,7 @@ trexio_open(const char* file_name, const char mode, /* Data for the parent type */ - strncpy(result->file_name, file_name, TREXIO_MAX_FILENAME_LENGTH); + strncpy(result->file_name, file_name, TREXIO_MAX_FILENAME_LENGTH-1); if (result->file_name[TREXIO_MAX_FILENAME_LENGTH-1] != '\0') { free(result); return NULL; @@ -1154,8 +1154,8 @@ trexio_exit_code trexio_read_$group_dset$_32(trexio_t* const file, $group_dset_d trexio_exit_code trexio_write_$group_dset$_32(trexio_t* const file, const $group_dset_dtype_single$* $group_dset$); trexio_exit_code trexio_read_$group_dset$_64(trexio_t* const file, $group_dset_dtype_double$* const $group_dset$); trexio_exit_code trexio_write_$group_dset$_64(trexio_t* const file, const $group_dset_dtype_double$* $group_dset$); -trexio_exit_code trexio_read_safe_$group_dset$(trexio_t* const file, $group_dset_dtype_default$* const dset_out, const uint64_t dim_out); -trexio_exit_code trexio_write_safe_$group_dset$(trexio_t* const file, const $group_dset_dtype_default$* dset_in, const uint64_t dim_in); +trexio_exit_code trexio_read_safe_$group_dset$(trexio_t* const file, $group_dset_dtype_default$* const dset_out, const int64_t dim_out); +trexio_exit_code trexio_write_safe_$group_dset$(trexio_t* const file, const $group_dset_dtype_default$* dset_in, const int64_t dim_in); #+end_src **** Source code for double precision functions @@ -1423,7 +1423,7 @@ trexio_write_$group_dset$_32 (trexio_t* const file, const $group_dset_dtype_sing #+begin_src c :tangle read_dset_data_safe_front.c trexio_exit_code -trexio_read_safe_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const dset_out, const uint64_t dim_out) +trexio_read_safe_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const dset_out, const int64_t dim_out) { if (file == NULL) return TREXIO_INVALID_ARG_1; @@ -1446,7 +1446,7 @@ trexio_read_safe_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* dim_size *= dims[i]; } - if (dim_out > dim_size) return TREXIO_UNSAFE_ARRAY_DIM; + if (dim_out > (int64_t) dim_size) return TREXIO_UNSAFE_ARRAY_DIM; /* */ return trexio_read_$group_dset$_$default_prec$(file, dset_out); @@ -1456,7 +1456,7 @@ trexio_read_safe_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* #+begin_src c :tangle write_dset_data_safe_front.c trexio_exit_code -trexio_write_safe_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default$* dset_in, const uint64_t dim_in) +trexio_write_safe_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default$* dset_in, const int64_t dim_in) { if (file == NULL) return TREXIO_INVALID_ARG_1; @@ -1480,7 +1480,7 @@ trexio_write_safe_$group_dset$ (trexio_t* const file, const $group_dset_dtype_de dim_size *= dims[i]; } - if (dim_in > dim_size) return TREXIO_UNSAFE_ARRAY_DIM; + if (dim_in > (int64_t) dim_size) return TREXIO_UNSAFE_ARRAY_DIM; /* */ return trexio_write_$group_dset$_$default_prec$(file, dset_in); From 808adc7cd80abfa189d17c64ab53cf84691070c8 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 6 Aug 2021 14:04:29 +0300 Subject: [PATCH 034/107] add typemaps and tests for numpy int arrays + some cleaning in the Python test and SWIG interface files --- python/test/test_py.py | 91 +++++++++++++++++++++++++----------------- src/pytrexio.i | 70 ++++++++++---------------------- 2 files changed, 74 insertions(+), 87 deletions(-) diff --git a/python/test/test_py.py b/python/test/test_py.py index d0649b5..b00da59 100644 --- a/python/test/test_py.py +++ b/python/test/test_py.py @@ -1,13 +1,12 @@ import os import shutil -#import numpy as np +import numpy as np from pytrexio import * -# TODO: make a user-friendly more pythonic API that will have to be autogenerated -# add Exception handling -# check of dimensions and call to safe API -# conversion to and from numpy arrays +# TODO: +# 1) make a user-friendly more pythonic API that will have to be autogenerated +# 2) add Exception handling (can be done easily in the front end python-ic API e.g. try: if function_call(...) == TREXIO_SUCCESS # automatically download (hopefully the latest version) numpy.i using # wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i @@ -37,7 +36,7 @@ try: elif TEST_TREXIO_BACKEND == TREXIO_TEXT: shutil.rmtree(output_filename) except: - print (f'Test file {output_filename} does not exist') + print ('Nothing to remove.') #=========================================================# #============ WRITE THE DATA IN THE TEST FILE ============# @@ -50,14 +49,32 @@ nucleus_num = 12 rc = trexio_write_nucleus_num(test_file, nucleus_num) assert rc==TREXIO_SUCCESS -charges = doubleArray(nucleus_num) -for i in range(nucleus_num): - if i < nucleus_num/2: - charges[i] = 6. - else: - charges[i] = 1. +# initialize charge arrays as a list and convert it to numpy array +charges = [6., 6., 6., 6., 6., 6., 1., 1., 1., 1., 1., 1.] +charges_np = np.array(charges, dtype=np.float64) -rc = trexio_write_nucleus_charge(test_file, charges) +# function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived +# from the size of the list/array by SWIG using typemacs from numpy.i +rc = trexio_write_safe_nucleus_charge(test_file, charges_np) +assert rc==TREXIO_SUCCESS + +# less Python-ic way to read/write arrays using Array classes (probably more portable to other languages) +#charges = doubleArray(nucleus_num) +#for i in range(nucleus_num): +# if i < nucleus_num/2: +# charges[i] = 6. +# else: +# charges[i] = 1. +#rc = trexio_write_nucleus_charge(test_file, charges) + +# initialize arrays of nuclear indices as a list and convert it to numpy array +indices = [i for i in range(nucleus_num)] +# type cast is important here because by default numpy transforms a list of integers into int64 array +indices_np = np.array(indices, dtype=np.int32) + +# function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived +# from the size of the list/array by SWIG using typemacs from numpy.i +rc = trexio_write_safe_basis_nucleus_index(test_file, indices_np) assert rc==TREXIO_SUCCESS point_group = 'B3U' @@ -97,43 +114,43 @@ test_file2 = trexio_open(output_filename, 'r', TEST_TREXIO_BACKEND) result = trexio_read_nucleus_num(test_file2) assert result[0]==TREXIO_SUCCESS assert result[1]==nucleus_num -#print(result) -charges2 = doubleArray(nucleus_num) -print(charges2[3]) -#for i in range(nucleus_num): -# charges2[i] = -1. - -rc = trexio_read_nucleus_charge(test_file2, charges2) -assert rc==TREXIO_SUCCESS -for i in range(nucleus_num): - assert charges2[i]==charges[i] - -#charge_numpy = np.zeros(nucleus_num, dtype=np.float64) -#print(charge_numpy) - -rc, charge_numpy = trexio_read_safe_nucleus_charge(test_file2, 12) - -print(charge_numpy) -print(charge_numpy[11]) +# safe call to read_safe array of float values +rc, rcharges_np = trexio_read_safe_nucleus_charge(test_file2, nucleus_num) assert rc==TREXIO_SUCCESS +assert rcharges_np.dtype is np.dtype(np.float64) +np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) # unsafe call to read_safe should not only have return code = TREXIO_UNSAFE_ARRAY_DIM -# but also should not return numpy array filled with garbage -rc, charge_numpy = trexio_read_safe_nucleus_charge(test_file2, 12*5) - -#print(charge_numpy) +# TODO: it should not return numpy array filled with garbage +rc, rcharges_fail = trexio_read_safe_nucleus_charge(test_file2, nucleus_num*5) assert rc==TREXIO_UNSAFE_ARRAY_DIM +# less Python-ic way to read/write arrays using Array classes (probably more portable to other languages) +#charges2 = doubleArray(nucleus_num) +#for i in range(nucleus_num): +# charges2[i] = -1. +#rc = trexio_read_nucleus_charge(test_file2, charges2) +#assert rc==TREXIO_SUCCESS +#for i in range(nucleus_num): +# assert charges2[i]==charges[i] + +# safe call to read_safe array of int values +rc, rindices_np = trexio_read_safe_basis_nucleus_index(test_file2, nucleus_num) +assert rc==TREXIO_SUCCESS +assert rindices_np.dtype is np.dtype(np.int32) +for i in range(nucleus_num): + assert rindices_np[i]==indices_np[i] + + # [WIP]: ideally, the list of strings should be returned as below #rc, label_2d = trexio_read_nucleus_label(test_file2, 10) # [WIP]: currently only low-level routines (return one long string instead of an array of strings) work - rc, labels_1d = trexio_read_nucleus_label_low(test_file2, 10) assert rc==TREXIO_SUCCESS labels_2d = [label for label in labels_1d.split(TREXIO_DELIM) if label] -print(labels_2d) +print('Read and parsed nuclear labels:\n', labels_2d) for i in range(nucleus_num): assert labels_2d[i]==labels[i] diff --git a/src/pytrexio.i b/src/pytrexio.i index 78a5e43..1503382 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -10,32 +10,37 @@ /* Include stdint to recognize types from stdint.h */ %include -/* Include carrays to work with C pointers to arrays */ + +/* NOTE: + carrays was useful before numpy.i was introduced. + For Python interface it's better to use numpy arrays instead of carrays, because the latter are less python-ic. + On the other hand, carrays might be more portable to other target languages. +// Include carrays to work with C pointers to arrays %include "carrays.i" -/* Include classes that correspond to integer and float arrays */ +// Include classes that correspond to integer and float arrays %array_class(double, doubleArray); %array_class(float, floatArray); %array_class(int32_t, int32Array); %array_class(int64_t, int64Array); +*/ + /* Include typemaps to play with input/output re-casting Useful when working with C pointers */ %include typemaps.i - /* Redefine the int32_t* and int64_t* num to be output Useful for TREXIO read_num functions where the num variable is modified by address */ %apply int *OUTPUT { int32_t* const num}; %apply int *OUTPUT { int64_t* const num}; -/* Does not work for arrays (SIGSEGV) -%apply double *OUTPUT { double* const dataset }; -*/ +/* Does not work for arrays (SIGSEGV) */ + /* This enables access to trexio_[...]_read_dset_str_low set of functions in order to return one long string with TREXIO_DELIM delimeter as 2-nd argument of output tuple */ %include -%cstring_bounded_output(char* dset_out, 1024); +%cstring_bounded_output(char* dset_out, 4096); /* [WIP] TREXIO back ends and exit codes can be redefined in the SWIG target language using %ignore and further #define statements (instead of disabling the type cast in the trexio.h file) @@ -47,11 +52,11 @@ #define TREXIO_TEXT 0 */ -/* This is an attempt to make SWIG treat double * dset_out, const uint64_t dim_out pattern +/* This is an attempt to make SWIG treat double * dset_out|_in, int64_t dim_out|_in pattern as a special case in order to return the NumPy array to Python from C pointer to array provided by trexio_read_safe_[dset_num] function. NOTE: numpy.i is currently not part of SWIG but included in the numpy distribution (under numpy/tools/swig/numpy.i) - This means that the interface file have to be provided to SWIG upon compilation either by + This means that the interface file have to be provided to SWIG during compilation either by copying it to the local working directory or by providing -l/path/to/numpy.i flag upon SWIG compilation */ %include "numpy.i" @@ -60,7 +65,12 @@ import_array(); %} +/* Enable write|read_safe functions to convert numpy arrays from/to double arrays */ %apply (double* ARGOUT_ARRAY1, int DIM1) {(double * const dset_out, const int64_t dim_out)}; +%apply (double* IN_ARRAY1, int DIM1) {(const double * dset_in, const int64_t dim_in)}; +/* Enable write|read_safe functions to convert numpy arrays from/to int32 arrays */ +%apply (int* ARGOUT_ARRAY1, int DIM1) {(int32_t * const dset_out, const int64_t dim_out)}; +%apply (int* IN_ARRAY1, int DIM1) {(const int32_t * dset_in, const int64_t dim_in)}; /* This tells SWIG to treat char ** dset_in pattern as a special case Enables access to trexio_[...]_write_dset_str set of functions directly, i.e. @@ -93,47 +103,7 @@ import_array(); free((char *) $1); } -/* [WIP] This is an attempt to make SWIG treat char ** dset_out as a special case - In order to return list of string to Python from C-native char ** dset_out, - which is modified (but not allocated) within the trexio_[...]_read_dset_str function -*/ -%typemap(in, numinputs=0) char ** dset_out (char * temp) { - /*temp = (char *) malloc(1028*sizeof(char));*/ - $1 = &temp; -} - -%typemap(argout) char ** dset_out { - - Py_ssize_t i = 0; - Py_ssize_t mysize = 12; - PyObject *o_res = PyList_New(mysize); - PyObject *o; - for (i = 0; i < mysize; i++) { - //printf("%s\n", $1[i]); - o = PyString_FromString($1[i]); - PyList_SetItem(o_res, i, o); - } - - PyObject *o2, *o3; - - if ((!$result) || ($result == Py_None)) { - $result = o_res; - } else { - if (!PyTuple_Check($result)) { - PyObject *o2 = $result; - $result = PyTuple_New(1); - PyTuple_SetItem($result, 0, o2); - } - o3 = PyTuple_New(1); - PyTuple_SetItem(o3, 0, o_res); - o2 = $result; - $result = PySequence_Concat(o2, o3); - Py_DECREF(o2); - Py_DECREF(o3); - Py_DECREF(o_res); - } -} - /* Parse the header files to generate wrappers */ %include "trexio_s.h" %include "trexio.h" + From ec7a39f6c24eddff78f25b2b01cb148293e5d7c9 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 6 Aug 2021 16:24:11 +0300 Subject: [PATCH 035/107] strict int types for DIM1 and arrays --- src/pytrexio.i | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index 1503382..f19cb6d 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -65,12 +65,15 @@ import_array(); %} +/* Typemaps below change the type of numpy array dimensions from int to int64_t */ +%numpy_typemaps(double, NPY_DOUBLE, int64_t) +%numpy_typemaps(int32_t, NPY_INT, int64_t) /* Enable write|read_safe functions to convert numpy arrays from/to double arrays */ -%apply (double* ARGOUT_ARRAY1, int DIM1) {(double * const dset_out, const int64_t dim_out)}; -%apply (double* IN_ARRAY1, int DIM1) {(const double * dset_in, const int64_t dim_in)}; +%apply (double* ARGOUT_ARRAY1, int64_t DIM1) {(double * const dset_out, const int64_t dim_out)}; +%apply (double* IN_ARRAY1, int64_t DIM1) {(const double * dset_in, const int64_t dim_in)}; /* Enable write|read_safe functions to convert numpy arrays from/to int32 arrays */ -%apply (int* ARGOUT_ARRAY1, int DIM1) {(int32_t * const dset_out, const int64_t dim_out)}; -%apply (int* IN_ARRAY1, int DIM1) {(const int32_t * dset_in, const int64_t dim_in)}; +%apply (int32_t* ARGOUT_ARRAY1, int64_t DIM1) {(int32_t * const dset_out, const int64_t dim_out)}; +%apply (int32_t* IN_ARRAY1, int64_t DIM1) {(const int32_t * dset_in, const int64_t dim_in)}; /* This tells SWIG to treat char ** dset_in pattern as a special case Enables access to trexio_[...]_write_dset_str set of functions directly, i.e. From 66a790717ab7531f38af6ce73e7284a160cb14a0 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 10 Aug 2021 15:43:24 +0300 Subject: [PATCH 036/107] Optional check and update of numpy.i file if outdated --- Makefile.am | 4 +++- tools/check_numpy_i.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100755 tools/check_numpy_i.sh diff --git a/Makefile.am b/Makefile.am index f5798b7..a8544a1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -213,6 +213,8 @@ $(pytrexio_c): $(ORG_FILES) $(trexio_h) $(pytrexio_i) $(numpy_i) $(numpy_i): wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i -O $(numpy_i) +check-numpy: + cd tools && ./check_numpy_i.sh CLEANFILES += src/pytrexio_wrap.c \ src/pytrexio.py \ @@ -220,7 +222,7 @@ CLEANFILES += src/pytrexio_wrap.c \ python/src/*.c \ python/src/*.h -.PHONY: cppcheck python-test python-install +.PHONY: cppcheck python-test python-install check-numpy endif diff --git a/tools/check_numpy_i.sh b/tools/check_numpy_i.sh new file mode 100755 index 0000000..49f5686 --- /dev/null +++ b/tools/check_numpy_i.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Check that script is executed from tools directory +if [[ $(basename $PWD) != "tools" ]] ; then + echo "This script should run in the tools directory" + exit -1 +fi + +TREXIO_ROOT=$(dirname "${PWD}../") + +# First define readonly global variables. +readonly SRC=${TREXIO_ROOT}/src +readonly TOOLS=${TREXIO_ROOT}/tools + +# We want the script to crash on the 1st error: +set -e + +NUMPY_SRC=${SRC}/numpy.i +NUMPY_LATEST=${TOOLS}/numpy.i + +# Download the latest numpy.i file from NumPy GitHub +wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i -O ${NUMPY_LATEST} + +# Execute diff to check if the numpy.i file in the src/ directory is updated +if ! diff -q ${NUMPY_LATEST} ${NUMPY_SRC} &>/dev/null; then + >&2 echo "numpy.i SWIG interface file in the source tree is outdated; replacing ..." + mv ${NUMPY_LATEST} ${NUMPY_SRC} +else + >&2 echo "numpy.i SWIG interface file in the source tree is up-to-date." + rm ${NUMPY_LATEST} +fi + From 2be1abd571255f2db7f9fc843c18322eb722ee2e Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 11 Aug 2021 15:59:58 +0300 Subject: [PATCH 037/107] add Python front end --- src/templates_front/build.sh | 4 + src/templates_front/templator_front.org | 265 +++++++++++++++++++++++- 2 files changed, 263 insertions(+), 6 deletions(-) diff --git a/src/templates_front/build.sh b/src/templates_front/build.sh index 61cde7b..5853c27 100644 --- a/src/templates_front/build.sh +++ b/src/templates_front/build.sh @@ -10,6 +10,7 @@ echo "" >> trexio.h cat prefix_s_front.h > trexio_s.h cat prefix_fortran.f90 > trexio_f.f90 +cat prefix_python.py > trexio.py # c front end cat populated/pop_*.c >> trexio.c @@ -21,6 +22,9 @@ cat populated/pop_*.f90 >> trexio_f.f90 cat helper_fortran.f90 >> trexio_f.f90 cat populated/pop_*.fh_90 >> trexio_f.f90 +# python front end +cat populated/pop_*.py >> trexio.py + # suffixes cat suffix_s_front.h >> trexio_s.h cat suffix_front.h >> trexio.h diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index d29258d..7cbf128 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -12,13 +12,29 @@ #+NAME:header #+begin_src c /* This file was generated from the templator_front.org org-mode file. - To generate it, open trexio.org in Emacs and execute + To generate it, open templator_front.org in Emacs and execute M-x org-babel-tangle */ #+end_src - #+begin_src f90 :tangle prefix_fortran.f90 :noweb yes +** Python + #+begin_src python :tangle prefix_python.py + +# define TREXIO back ends +TREXIO_HDF5 = 0 +TREXIO_TEXT = 1 +#TREXIO_JSON = 2 +TREXIO_INVALID_BACK_END = 2 + +# define max length of string item when reading arrays of strings +# this is needed for the low-level C routines +PYTREXIO_MAX_STR_LENGTH = 2048 + + #+end_src + +** Fortran + #+begin_src f90 :tangle prefix_fortran.f90 module trexio use, intrinsic :: iso_c_binding @@ -33,10 +49,10 @@ module trexio integer(trexio_backend), parameter :: TREXIO_INVALID_BACK_END = 2 character(kind=c_char), parameter :: TREXIO_DELIM = c_new_line - #+end_src #+end_src +** C #+begin_src c :tangle prefix_front.h :noweb yes <
> #ifndef TREXIO_H @@ -119,7 +135,7 @@ typedef int32_t trexio_exit_code; #+end_src * Front end - + All calls to TREXIO are thread-safe. TREXIO front end is modular, which simplifies implementation of new back ends. @@ -177,6 +193,15 @@ for (text, code,_) in table: result += [ f" integer(trexio_exit_code), parameter :: {text:30s} = {code:d}" ] result += [ "#+end_src" ] +result += [ "" ] + +result += [ "#+begin_src python :tangle prefix_python.py :exports none" ] +result += [ "# define TREXIO exit codes" ] +for (text, code,_) in table: + text=text.replace("~","") + result += [ f"{text:30s} = {code:d}" ] +result += [ "#+end_src" ] + return '\n'.join(result) #+end_src @@ -241,14 +266,44 @@ return '\n'.join(result) integer(trexio_exit_code), parameter :: TREXIO_UNSAFE_ARRAY_DIM = 23 integer(trexio_exit_code), parameter :: TREXIO_INVALID_STR_LEN = 30 #+end_src + + #+begin_src python :tangle prefix_python.py :exports none + TREXIO_FAILURE = -1 + TREXIO_SUCCESS = 0 + TREXIO_INVALID_ARG_1 = 1 + TREXIO_INVALID_ARG_2 = 2 + TREXIO_INVALID_ARG_3 = 3 + TREXIO_INVALID_ARG_4 = 4 + TREXIO_INVALID_ARG_5 = 5 + TREXIO_END = 6 + TREXIO_READONLY = 7 + TREXIO_ERRNO = 8 + TREXIO_INVALID_ID = 9 + TREXIO_ALLOCATION_FAILED = 10 + TREXIO_HAS_NOT = 11 + TREXIO_INVALID_NUM = 12 + TREXIO_ATTR_ALREADY_EXISTS = 13 + TREXIO_DSET_ALREADY_EXISTS = 14 + TREXIO_OPEN_ERROR = 15 + TREXIO_LOCK_ERROR = 16 + TREXIO_UNLOCK_ERROR = 17 + TREXIO_FILE_ERROR = 18 + TREXIO_GROUP_READ_ERROR = 19 + TREXIO_GROUP_WRITE_ERROR = 20 + TREXIO_ELEM_READ_ERROR = 21 + TREXIO_ELEM_WRITE_ERROR = 22 + TREXIO_UNSAFE_ARRAY_DIM = 23 + TREXIO_INVALID_STR_LEN = 30 + #+end_src :END: + +*** Decoding errors + The ~trexio_string_of_error~ converts an exit code into a string. The string is assumed to be large enough to contain the error message (typically 128 characters). -â—‰ Decoding errors - To decode the error messages, ~trexio_string_of_error~ converts an error code into a string. @@ -659,6 +714,22 @@ interface end interface #+end_src + #+begin_src c :tangle prefix_python.py +def open(file_name: str, mode: str, back_end: int): + try: + from pytrexio import trexio_open + except ImportError: + raise + + try: + trexio_file = trexio_open(file_name, mode, back_end) + except: + raise + + return trexio_file + + #+end_src + Because arrays are zero-based in Fortran, we need to set a flag to know if we need to shift by 1 arrays of indices. @@ -776,6 +847,21 @@ interface end interface #+end_src + #+begin_src c :tangle prefix_python.py +def close(trexio_file): + try: + from pytrexio import trexio_close, trexio_string_of_error, TREXIO_SUCCESS + except ImportError: + raise + + try: + rc = trexio_close(trexio_file) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + + #+end_src + * Templates for front end ** Description @@ -1119,6 +1205,43 @@ interface end interface #+end_src +*** Python templates for front end + + #+begin_src python :tangle write_num_front.py +def write_$group_num$(trexio_file, num_w) -> None: + try: + from pytrexio import trexio_write_$group_num$, trexio_string_of_error, TREXIO_SUCCESS + except ImportError: + raise + + try: + rc = trexio_write_$group_num$(trexio_file, num_w) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + except: + raise + + #+end_src + + #+begin_src python :tangle read_num_front.py +def read_$group_num$(trexio_file): + try: + from pytrexio import trexio_read_$group_num$, trexio_string_of_error, TREXIO_SUCCESS + except ImportError: + raise + + try: + rc, num_r = trexio_read_$group_num$(trexio_file) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + except: + raise + + return num_r + #+end_src + ** Templates for front end has/read/write a dataset of numerical data This section concerns API calls related to datasets. @@ -1610,6 +1733,42 @@ interface end interface #+end_src +*** Python templates for front end + + #+begin_src python :tangle write_dset_data_front.py +def write_$group_dset$(trexio_file, dset_w) -> None: + try: + from pytrexio import trexio_write_$group_dset$, trexio_string_of_error, TREXIO_SUCCESS + except ImportError: + raise + + try: + rc = trexio_write_$group_dset$(trexio_file, dset_w) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + except: + raise + + #+end_src + + #+begin_src python :tangle read_dset_data_front.py +def read_$group_dset$(trexio_file): + try: + from pytrexio import trexio_read_$group_dset$, trexio_string_of_error, TREXIO_SUCCESS + except ImportError: + raise + + try: + rc, dset_r = trexio_read_$group_dset$(trexio_file) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + except: + raise + + return dset_r + #+end_src ** Sparse data structures Sparse data structures are used typically for large tensors such as @@ -2065,6 +2224,56 @@ end interface end function trexio_write_$group_dset$ #+end_src +*** Python templates for front end + + #+begin_src python :tangle write_dset_str_front.py +def write_$group_dset$(trexio_file, dset_w) -> None: + try: + from pytrexio import (trexio_write_$group_dset$, + trexio_string_of_error, + TREXIO_SUCCESS + ) + except ImportError: + raise + + max_str_length = len(max(dset_w, key=len)) + 1 + + try: + rc = trexio_write_$group_dset$(trexio_file, dset_w, max_str_length) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + except: + raise + + #+end_src + + #+begin_src python :tangle read_dset_str_front.py +def read_$group_dset$(trexio_file): + try: + from pytrexio import (trexio_read_$group_dset$, + trexio_string_of_error, + TREXIO_DELIM + ) + except ImportError: + raise + + try: + rc, dset_1d_r = trexio_read_$group_dset$_low(trexio_file, PYTREXIO_MAX_STR_LENGTH) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + except: + raise + + try: + dset_2d_r = [item for item in dset_1d_r.split(TREXIO_DELIM) if item] + assert dset_2d_r + except AssertionError: + raise TypeError(f"Output of {read_$group_dset$.__name__} function cannot be an empty list.") + + return dset_2d_r + #+end_src ** Templates for front end has/read/write a single string attribute *** Introduction @@ -2243,6 +2452,50 @@ end interface end function trexio_write_$group_str$ #+end_src +*** Python templates for front end + + #+begin_src python :tangle write_attr_str_front.py +def write_$group_str$(trexio_file, str_w) -> None: + try: + from pytrexio import (trexio_write_$group_str$, + trexio_string_of_error, + TREXIO_SUCCESS + ) + except ImportError: + raise + + max_str_length = len(str_w) + 1 + + try: + rc = trexio_write_$group_str$(trexio_file, str_w, max_str_length) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + except: + raise + + #+end_src + + #+begin_src python :tangle read_attr_str_front.py +def read_$group_str$(trexio_file): + try: + from pytrexio import (trexio_read_$group_str$, + trexio_string_of_error, + PYTREXIO_MAX_STR_LENGTH + ) + except ImportError: + raise + + try: + rc, str_r = trexio_read_$group_str$(trexio_file, PYTREXIO_MAX_STR_LENGTH) + assert rc==TREXIO_SUCCESS + except AssertionError: + raise Exception(trexio_string_of_error(rc)) + except: + raise + + return str_r + #+end_src * Fortran helper/wrapper functions The function below adapts the original C-based ~trexio_open~ for Fortran. From 209f4506c8aa7fef9d2c7908464ea7e049ca4f2a Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 11 Aug 2021 16:01:04 +0300 Subject: [PATCH 038/107] update gitignore --- src/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/src/.gitignore b/src/.gitignore index 977c220..8847478 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -2,6 +2,7 @@ templates_front/*.c templates_front/*.h templates_front/*.f90 templates_front/*.fh_90 +templates_front/*.py templates_front/*.dump templates_front/populated/ From 721352cb898c464de2cc7ba7fdfc029897f6ef1f Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 11 Aug 2021 16:01:36 +0300 Subject: [PATCH 039/107] add missing support for reading single string attributes --- src/pytrexio.i | 6 ++++++ src/templates_front/templator_front.org | 10 +++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index f19cb6d..df460a7 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -34,13 +34,19 @@ */ %apply int *OUTPUT { int32_t* const num}; %apply int *OUTPUT { int64_t* const num}; + /* Does not work for arrays (SIGSEGV) */ /* This enables access to trexio_[...]_read_dset_str_low set of functions in order to return one long string with TREXIO_DELIM delimeter as 2-nd argument of output tuple */ %include +/* This enables read of long strings with TREXIO_DELIM delimeters that can be further converted into an array of string */ %cstring_bounded_output(char* dset_out, 4096); +/* This enables read of single string attributes with pre-defined max_str_len + for Python we pre-define max_str_len = PYTREXIO_MAX_STR_LENGTH everywhere for simplicity +*/ +%cstring_output_maxsize(char* const str_out, const uint32_t max_str_len); /* [WIP] TREXIO back ends and exit codes can be redefined in the SWIG target language using %ignore and further #define statements (instead of disabling the type cast in the trexio.h file) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 7cbf128..e9a2ef9 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -2289,27 +2289,27 @@ def read_$group_dset$(trexio_file): #+begin_src c :tangle hrw_attr_str_front.h :exports none trexio_exit_code trexio_has_$group_str$(trexio_t* const file); -trexio_exit_code trexio_read_$group_str$(trexio_t* const file, char* const str, const uint32_t max_str_len); +trexio_exit_code trexio_read_$group_str$(trexio_t* const file, char* const str_out, const uint32_t max_str_len); trexio_exit_code trexio_write_$group_str$(trexio_t* const file, const char* str, const uint32_t max_str_len); #+end_src #+begin_src c :tangle read_attr_str_front.c trexio_exit_code -trexio_read_$group_str$ (trexio_t* const file, char* const str, const uint32_t max_str_len) +trexio_read_$group_str$ (trexio_t* const file, char* const str_out, const uint32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; - if (str == NULL) return TREXIO_INVALID_ARG_2; + if (str_out == NULL) return TREXIO_INVALID_ARG_2; if (max_str_len <= 0) return TREXIO_INVALID_ARG_3; switch (file->back_end) { case TREXIO_TEXT: - return trexio_text_read_$group_str$(file, str, max_str_len); + return trexio_text_read_$group_str$(file, str_out, max_str_len); break; case TREXIO_HDF5: - return trexio_hdf5_read_$group_str$(file, str, max_str_len); + return trexio_hdf5_read_$group_str$(file, str_out, max_str_len); break; /* case TREXIO_JSON: From 0b223b9d6e9ec6df28da272f8593be242d5eae4d Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 14 Aug 2021 12:27:11 +0300 Subject: [PATCH 040/107] return type cast of exit codes and back ends in C add top-level string_of_error function in Python --- src/templates_front/templator_front.org | 92 +++++++++++++++---------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index e9a2ef9..b7a449a 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -174,7 +174,7 @@ typedef int32_t trexio_exit_code; # We need to force Emacs not to indent the Python code: # -*- org-src-preserve-indentation: t - #+begin_src python :var table=table-exit-codes :results drawer :exports none + #+begin_src python :var table=table-exit-codes :results drawer """ This script generates the C and Fortran constants for the error codes from the org-mode table. """ @@ -182,7 +182,7 @@ typedef int32_t trexio_exit_code; result = [ "#+begin_src c :tangle prefix_front.h :exports none" ] for (text, code,_) in table: text=text.replace("~","") - result += [ f"#define {text:30s} {code:d} //((trexio_exit_code) {code:d})" ] + result += [ f"#define {text:30s} ((trexio_exit_code) {code:d})" ] result += [ "#+end_src" ] result += [ "" ] @@ -210,32 +210,32 @@ return '\n'.join(result) #+RESULTS: :RESULTS: #+begin_src c :tangle prefix_front.h :exports none - #define TREXIO_FAILURE -1 //((trexio_exit_code) -1) - #define TREXIO_SUCCESS 0 //((trexio_exit_code) 0) - #define TREXIO_INVALID_ARG_1 1 //((trexio_exit_code) 1) - #define TREXIO_INVALID_ARG_2 2 //((trexio_exit_code) 2) - #define TREXIO_INVALID_ARG_3 3 //((trexio_exit_code) 3) - #define TREXIO_INVALID_ARG_4 4 //((trexio_exit_code) 4) - #define TREXIO_INVALID_ARG_5 5 //((trexio_exit_code) 5) - #define TREXIO_END 6 //((trexio_exit_code) 6) - #define TREXIO_READONLY 7 //((trexio_exit_code) 7) - #define TREXIO_ERRNO 8 //((trexio_exit_code) 8) - #define TREXIO_INVALID_ID 9 //((trexio_exit_code) 9) - #define TREXIO_ALLOCATION_FAILED 10 //((trexio_exit_code) 10) - #define TREXIO_HAS_NOT 11 //((trexio_exit_code) 11) - #define TREXIO_INVALID_NUM 12 //((trexio_exit_code) 12) - #define TREXIO_ATTR_ALREADY_EXISTS 13 //((trexio_exit_code) 13) - #define TREXIO_DSET_ALREADY_EXISTS 14 //((trexio_exit_code) 14) - #define TREXIO_OPEN_ERROR 15 //((trexio_exit_code) 15) - #define TREXIO_LOCK_ERROR 16 //((trexio_exit_code) 16) - #define TREXIO_UNLOCK_ERROR 17 //((trexio_exit_code) 17) - #define TREXIO_FILE_ERROR 18 //((trexio_exit_code) 18) - #define TREXIO_GROUP_READ_ERROR 19 //((trexio_exit_code) 19) - #define TREXIO_GROUP_WRITE_ERROR 20 //((trexio_exit_code) 20) - #define TREXIO_ELEM_READ_ERROR 21 //((trexio_exit_code) 21) - #define TREXIO_ELEM_WRITE_ERROR 22 //((trexio_exit_code) 22) - #define TREXIO_UNSAFE_ARRAY_DIM 23 //((trexio_exit_code) 23) - #define TREXIO_INVALID_STR_LEN 30 //((trexio_exit_code) 30) + #define TREXIO_FAILURE ((trexio_exit_code) -1) + #define TREXIO_SUCCESS ((trexio_exit_code) 0) + #define TREXIO_INVALID_ARG_1 ((trexio_exit_code) 1) + #define TREXIO_INVALID_ARG_2 ((trexio_exit_code) 2) + #define TREXIO_INVALID_ARG_3 ((trexio_exit_code) 3) + #define TREXIO_INVALID_ARG_4 ((trexio_exit_code) 4) + #define TREXIO_INVALID_ARG_5 ((trexio_exit_code) 5) + #define TREXIO_END ((trexio_exit_code) 6) + #define TREXIO_READONLY ((trexio_exit_code) 7) + #define TREXIO_ERRNO ((trexio_exit_code) 8) + #define TREXIO_INVALID_ID ((trexio_exit_code) 9) + #define TREXIO_ALLOCATION_FAILED ((trexio_exit_code) 10) + #define TREXIO_HAS_NOT ((trexio_exit_code) 11) + #define TREXIO_INVALID_NUM ((trexio_exit_code) 12) + #define TREXIO_ATTR_ALREADY_EXISTS ((trexio_exit_code) 13) + #define TREXIO_DSET_ALREADY_EXISTS ((trexio_exit_code) 14) + #define TREXIO_OPEN_ERROR ((trexio_exit_code) 15) + #define TREXIO_LOCK_ERROR ((trexio_exit_code) 16) + #define TREXIO_UNLOCK_ERROR ((trexio_exit_code) 17) + #define TREXIO_FILE_ERROR ((trexio_exit_code) 18) + #define TREXIO_GROUP_READ_ERROR ((trexio_exit_code) 19) + #define TREXIO_GROUP_WRITE_ERROR ((trexio_exit_code) 20) + #define TREXIO_ELEM_READ_ERROR ((trexio_exit_code) 21) + #define TREXIO_ELEM_WRITE_ERROR ((trexio_exit_code) 22) + #define TREXIO_UNSAFE_ARRAY_DIM ((trexio_exit_code) 23) + #define TREXIO_INVALID_STR_LEN ((trexio_exit_code) 30) #+end_src #+begin_src f90 :tangle prefix_fortran.f90 :exports none @@ -268,6 +268,7 @@ return '\n'.join(result) #+end_src #+begin_src python :tangle prefix_python.py :exports none + # define TREXIO exit codes TREXIO_FAILURE = -1 TREXIO_SUCCESS = 0 TREXIO_INVALID_ARG_1 = 1 @@ -296,8 +297,7 @@ return '\n'.join(result) TREXIO_INVALID_STR_LEN = 30 #+end_src :END: - - + *** Decoding errors The ~trexio_string_of_error~ converts an exit code into a string. The @@ -414,7 +414,8 @@ return '\n'.join(result) break; #+end_example - # Source +**** C source code + #+begin_src c :tangle prefix_front.c :noweb yes const char* trexio_string_of_error (const trexio_exit_code error) @@ -432,7 +433,8 @@ trexio_string_of_error_f (const trexio_exit_code error, char result[< str: + try: + from pytrexio import trexio_string_of_error + except ImportError: + raise + + try: + error_str = trexio_string_of_error(trexio_return_code) + except: + raise + + return error_str + #+end_src + ** Back ends TREXIO has several back ends: @@ -460,10 +479,10 @@ end interface #+begin_src c :tangle prefix_front.h typedef int32_t back_end_t; -#define TREXIO_HDF5 0 //( (back_end_t) 0 ) -#define TREXIO_TEXT 1 //( (back_end_t) 1 ) +#define TREXIO_HDF5 ( (back_end_t) 0 ) +#define TREXIO_TEXT ( (back_end_t) 1 ) /*#define TREXIO_JSON ( (back_end_t) 2 )*/ -#define TREXIO_INVALID_BACK_END 2 //( (back_end_t) 2 ) +#define TREXIO_INVALID_BACK_END ( (back_end_t) 2 ) #define TREXIO_DELIM "\n" #+end_src @@ -2480,8 +2499,7 @@ def write_$group_str$(trexio_file, str_w) -> None: def read_$group_str$(trexio_file): try: from pytrexio import (trexio_read_$group_str$, - trexio_string_of_error, - PYTREXIO_MAX_STR_LENGTH + trexio_string_of_error ) except ImportError: raise From 9f8b6838df3c0b2c746358007cb15eca94b8b76e Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 13:15:23 +0300 Subject: [PATCH 041/107] minor cleaning --- python/.gitignore | 1 - python/MANIFEST.in | 2 +- python/pyproject.toml | 1 - python/setup.cfg | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/python/.gitignore b/python/.gitignore index b968958..a4b23d2 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,7 +1,6 @@ AUTHORS LICENSE README.md -pytrexio.py src/ build/ diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 10daee9..eab38bc 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1 +1 @@ -include src/*.c src/trexio*.h src/*.py +include src/*.c src/trexio*.h diff --git a/python/pyproject.toml b/python/pyproject.toml index d484910..374b58c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,6 @@ [build-system] requires = [ "setuptools>=42", - "h5py", "wheel" ] build-backend = "setuptools.build_meta" diff --git a/python/setup.cfg b/python/setup.cfg index 9549416..b4d9c49 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -8,5 +8,4 @@ license = BSD 3-Clause License [options] zip_safe = False python_requires = >=3.6 -install_requires = h5py From 31b14a891a00b684eb0f3c35f1e19affb4f4106e Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 13:26:57 +0300 Subject: [PATCH 042/107] make trexio a Python package instead of a module to fix some installation issues --- Makefile.am | 11 ++++++----- python/setup.py | 11 +++++------ python/trexio/__init__.py | 0 3 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 python/trexio/__init__.py diff --git a/Makefile.am b/Makefile.am index a8544a1..f1ede0d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -186,15 +186,15 @@ cppcheck.out: $(trexio_h) setup_py = $(srcdir)/python/setup.py setup_cfg = $(srcdir)/python/setup.cfg -pytrexio_py = $(srcdir)/python/pytrexio.py -TESTS_PY = $(srcdir)/python/test/test_py.py +pytrexio_py = $(srcdir)/python/trexio/pytrexio.py +TEST_PY = $(srcdir)/python/test/test_full.py pytrexio_c = $(srcdir)/src/pytrexio_wrap.c pytrexio_i = $(srcdir)/src/pytrexio.i numpy_i = $(srcdir)/src/numpy.i -python-test: $(pytrexio_py) $(TESTS_PY) - python3 $(TESTS_PY) +python-test: $(pytrexio_py) $(TEST_PY) + python3 $(TEST_PY) $(RM) -r -- __pycache__ python-install: $(pytrexio_py) $(setup_py) $(setup_cfg) @@ -218,7 +218,8 @@ check-numpy: CLEANFILES += src/pytrexio_wrap.c \ src/pytrexio.py \ - python/pytrexio.py \ + python/trexio/pytrexio.py \ + python/trexio/trexio.py \ python/src/*.c \ python/src/*.h diff --git a/python/setup.py b/python/setup.py index 29ae023..a837fe1 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """ -setup.py file for pytrexio +setup.py file for TREXIO Python package """ -from setuptools import setup, Extension, find_packages +from setuptools import setup, Extension import os rootpath = os.path.dirname(os.path.abspath(__file__)) @@ -15,7 +15,7 @@ with open("README.md", "r") as fh: long_description = fh.read() -pytrexio_module = Extension('_pytrexio', +pytrexio_module = Extension('trexio._pytrexio', sources = [os.path.join(srcpath, code) for code in c_files], include_dirs = ['/usr/include/hdf5/serial', srcpath], libraries = ['hdf5', 'hdf5_hl'], @@ -24,7 +24,7 @@ pytrexio_module = Extension('_pytrexio', ) -setup(name = 'pytrexio', +setup(name = 'trexio', version = '0.1', author = "TREX-CoE", author_email = "posenitskiy@irsamc.ups-tlse.fr", @@ -32,10 +32,9 @@ setup(name = 'pytrexio', long_description = long_description, long_description_content_type = "text/markdown", ext_modules = [pytrexio_module], - py_modules = ["pytrexio"], + packages = ['trexio'], url = 'https://github.com/TREX-CoE/trexio', license = 'BSD', - packages = find_packages(), classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: C", diff --git a/python/trexio/__init__.py b/python/trexio/__init__.py new file mode 100644 index 0000000..e69de29 From f0bccee32d99e6c9258ade6430994b2fcf641ab2 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 13:28:37 +0300 Subject: [PATCH 043/107] single top-level import of pytrexio resolves bug with caching --- src/templates_front/templator_front.org | 123 ++++++++++++------------ 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index b7a449a..89fc466 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -20,6 +20,11 @@ ** Python #+begin_src python :tangle prefix_python.py +try: + from trexio.pytrexio import * +except ImportError: + raise Exception("Could not import pytrexio module from trexio package") + # define TREXIO back ends TREXIO_HDF5 = 0 @@ -450,10 +455,10 @@ end interface #+begin_src python :tangle prefix_python.py :noexport def string_of_error(return_code: int) -> str: - try: - from pytrexio import trexio_string_of_error - except ImportError: - raise + #try: + # from trexio.pytrexio import trexio_string_of_error + #except ImportError: + # raise try: error_str = trexio_string_of_error(trexio_return_code) @@ -735,10 +740,10 @@ end interface #+begin_src c :tangle prefix_python.py def open(file_name: str, mode: str, back_end: int): - try: - from pytrexio import trexio_open - except ImportError: - raise + #try: + # from trexio.pytrexio import trexio_open + #except ImportError: + # raise try: trexio_file = trexio_open(file_name, mode, back_end) @@ -868,10 +873,10 @@ end interface #+begin_src c :tangle prefix_python.py def close(trexio_file): - try: - from pytrexio import trexio_close, trexio_string_of_error, TREXIO_SUCCESS - except ImportError: - raise + #try: + # from trexio.pytrexio import trexio_close, trexio_string_of_error + #except ImportError: + # raise try: rc = trexio_close(trexio_file) @@ -1228,10 +1233,10 @@ end interface #+begin_src python :tangle write_num_front.py def write_$group_num$(trexio_file, num_w) -> None: - try: - from pytrexio import trexio_write_$group_num$, trexio_string_of_error, TREXIO_SUCCESS - except ImportError: - raise + #try: + # from trexio.pytrexio import trexio_write_$group_num$, trexio_string_of_error + #except ImportError: + # raise try: rc = trexio_write_$group_num$(trexio_file, num_w) @@ -1245,10 +1250,10 @@ def write_$group_num$(trexio_file, num_w) -> None: #+begin_src python :tangle read_num_front.py def read_$group_num$(trexio_file): - try: - from pytrexio import trexio_read_$group_num$, trexio_string_of_error, TREXIO_SUCCESS - except ImportError: - raise + #try: + # from trexio.pytrexio import trexio_read_$group_num$, trexio_string_of_error + #except ImportError: + # raise try: rc, num_r = trexio_read_$group_num$(trexio_file) @@ -1755,14 +1760,14 @@ end interface *** Python templates for front end #+begin_src python :tangle write_dset_data_front.py -def write_$group_dset$(trexio_file, dset_w) -> None: - try: - from pytrexio import trexio_write_$group_dset$, trexio_string_of_error, TREXIO_SUCCESS - except ImportError: - raise +def write_safe_$group_dset$(trexio_file, dset_w) -> None: + #try: + # from trexio.pytrexio import trexio_write_safe_$group_dset$, trexio_string_of_error + #except ImportError: + # raise try: - rc = trexio_write_$group_dset$(trexio_file, dset_w) + rc = trexio_write_safe_$group_dset$(trexio_file, dset_w) assert rc==TREXIO_SUCCESS except AssertionError: raise Exception(trexio_string_of_error(rc)) @@ -1772,20 +1777,22 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+end_src #+begin_src python :tangle read_dset_data_front.py -def read_$group_dset$(trexio_file): - try: - from pytrexio import trexio_read_$group_dset$, trexio_string_of_error, TREXIO_SUCCESS - except ImportError: - raise +def read_safe_$group_dset$(trexio_file, dim): + #try: + # from trexio.pytrexio import trexio_read_safe_$group_dset$, trexio_string_of_error + #except ImportError: + # raise try: - rc, dset_r = trexio_read_$group_dset$(trexio_file) + rc, dset_r = trexio_read_safe_$group_dset$(trexio_file, dim) assert rc==TREXIO_SUCCESS except AssertionError: raise Exception(trexio_string_of_error(rc)) except: raise + # additional assert can be added here to check that read_safe functions returns numpy array of proper dimension + return dset_r #+end_src ** Sparse data structures @@ -2247,13 +2254,12 @@ end interface #+begin_src python :tangle write_dset_str_front.py def write_$group_dset$(trexio_file, dset_w) -> None: - try: - from pytrexio import (trexio_write_$group_dset$, - trexio_string_of_error, - TREXIO_SUCCESS - ) - except ImportError: - raise + #try: + # from trexio.pytrexio import (trexio_write_$group_dset$, + # trexio_string_of_error + # ) + #except ImportError: + # raise max_str_length = len(max(dset_w, key=len)) + 1 @@ -2269,13 +2275,13 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+begin_src python :tangle read_dset_str_front.py def read_$group_dset$(trexio_file): - try: - from pytrexio import (trexio_read_$group_dset$, - trexio_string_of_error, - TREXIO_DELIM - ) - except ImportError: - raise + #try: + # from trexio.pytrexio import (trexio_read_$group_dset$_low, + # trexio_string_of_error, + # TREXIO_DELIM + # ) + #except ImportError: + # raise try: rc, dset_1d_r = trexio_read_$group_dset$_low(trexio_file, PYTREXIO_MAX_STR_LENGTH) @@ -2475,13 +2481,12 @@ end interface #+begin_src python :tangle write_attr_str_front.py def write_$group_str$(trexio_file, str_w) -> None: - try: - from pytrexio import (trexio_write_$group_str$, - trexio_string_of_error, - TREXIO_SUCCESS - ) - except ImportError: - raise + #try: + # from trexio.pytrexio import (trexio_write_$group_str$, + # trexio_string_of_error, + # ) + #except ImportError: + # raise max_str_length = len(str_w) + 1 @@ -2497,12 +2502,12 @@ def write_$group_str$(trexio_file, str_w) -> None: #+begin_src python :tangle read_attr_str_front.py def read_$group_str$(trexio_file): - try: - from pytrexio import (trexio_read_$group_str$, - trexio_string_of_error - ) - except ImportError: - raise + #try: + # from trexio.pytrexio import (trexio_read_$group_str$, + # trexio_string_of_error + # ) + #except ImportError: + # raise try: rc, str_r = trexio_read_$group_str$(trexio_file, PYTREXIO_MAX_STR_LENGTH) From 9cc552409fd4c94e3668cae95e58d4220b6e016a Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 13:30:03 +0300 Subject: [PATCH 044/107] rename the front-end Python module --- src/templates_front/build.sh | 4 ++-- tools/prepare_python.sh | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/templates_front/build.sh b/src/templates_front/build.sh index 5853c27..aa0d114 100644 --- a/src/templates_front/build.sh +++ b/src/templates_front/build.sh @@ -10,7 +10,7 @@ echo "" >> trexio.h cat prefix_s_front.h > trexio_s.h cat prefix_fortran.f90 > trexio_f.f90 -cat prefix_python.py > trexio.py +cat prefix_python.py > trexio_api.py # c front end cat populated/pop_*.c >> trexio.c @@ -23,7 +23,7 @@ cat helper_fortran.f90 >> trexio_f.f90 cat populated/pop_*.fh_90 >> trexio_f.f90 # python front end -cat populated/pop_*.py >> trexio.py +cat populated/pop_*.py >> trexio_api.py # suffixes cat suffix_s_front.h >> trexio_s.h diff --git a/tools/prepare_python.sh b/tools/prepare_python.sh index 0e66a6c..7224e2b 100755 --- a/tools/prepare_python.sh +++ b/tools/prepare_python.sh @@ -13,15 +13,18 @@ readonly SRC=${TREXIO_ROOT}/src readonly INCLUDIR=${TREXIO_ROOT}/include readonly TOOLS=${TREXIO_ROOT}/tools readonly PYDIR=${TREXIO_ROOT}/python +readonly PYTREXIODIR=${PYDIR}/trexio # We want the script to crash on the 1st error: set -e -# Create src directory in the python folder if not yet done +# Create src and trexio directories in the python folder if not yet done mkdir -p ${PYDIR}/src +mkdir -p ${PYTREXIODIR} # Copy all the source code and header files in the corresponding python directory -cp ${SRC}/*.py ${PYDIR} +mv ${SRC}/pytrexio.py ${PYTREXIODIR} +mv ${SRC}/trexio_api.py ${PYTREXIODIR} cp ${SRC}/*.c ${PYDIR}/src cp ${SRC}/*.h ${PYDIR}/src cp ${INCLUDIR}/trexio.h ${PYDIR}/src From cb1e83ca5722631bff5672ba0c7dd2485a2dfa32 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 13:40:27 +0300 Subject: [PATCH 045/107] adapt tests to check both pytrexio and trexio_api modules --- python/test/test_api.py | 131 +++++++++++++++++++ python/test/{test_py.py => test_pytrexio.py} | 53 ++++---- 2 files changed, 160 insertions(+), 24 deletions(-) create mode 100644 python/test/test_api.py rename python/test/{test_py.py => test_pytrexio.py} (83%) diff --git a/python/test/test_api.py b/python/test/test_api.py new file mode 100644 index 0000000..f5bf278 --- /dev/null +++ b/python/test/test_api.py @@ -0,0 +1,131 @@ +import os +import shutil +import numpy as np + +import trexio.trexio_api as tr + +#=========================================================# +#======== SETUP THE BACK END AND OUTPUT FILE NAME ========# +#=========================================================# + +# 0: TREXIO_HDF5 ; 1: TREXIO_TEXT +TEST_TREXIO_BACKEND = tr.TREXIO_TEXT +OUTPUT_FILENAME_TEXT = 'test_py_swig.dir' +OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' + + +if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: + output_filename = OUTPUT_FILENAME_HDF5 +elif TEST_TREXIO_BACKEND == tr.TREXIO_TEXT: + output_filename = OUTPUT_FILENAME_TEXT +else: + raise ValueError ('Specify one of the supported back ends as TEST_TREXIO_BACKEND') + + +try: + if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: + os.remove(output_filename) + elif TEST_TREXIO_BACKEND == tr.TREXIO_TEXT: + shutil.rmtree(output_filename) +except: + print ('Nothing to remove.') + +#=========================================================# +#============ WRITE THE DATA IN THE TEST FILE ============# +#=========================================================# + +test_file = tr.open(output_filename, 'w', TEST_TREXIO_BACKEND) + +print(test_file) + +nucleus_num = 12 + +tr.write_nucleus_num(test_file, nucleus_num) +#rc = pytr.trexio_write_nucleus_num(test_file, nucleus_num) +#assert rc==tr.TREXIO_SUCCESS + +# initialize charge arrays as a list and convert it to numpy array +charges = [6., 6., 6., 6., 6., 6., 1., 1., 1., 1., 1., 1.] +charges_np = np.array(charges, dtype=np.float64) + +# function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived +# from the size of the list/array by SWIG using typemaps from numpy.i +rc = tr.write_safe_nucleus_charge(test_file, charges_np) + +# initialize arrays of nuclear indices as a list and convert it to numpy array +indices = [i for i in range(nucleus_num)] +# type cast is important here because by default numpy transforms a list of integers into int64 array +indices_np = np.array(indices, dtype=np.int32) + +# function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived +# from the size of the list/array by SWIG using typemacs from numpy.i +tr.write_safe_basis_nucleus_index(test_file, indices_np) + +point_group = 'B3U' + +tr.write_nucleus_point_group(test_file, point_group) + +labels = [ + 'C', + 'C', + 'C', + 'C', + 'C', + 'C', + 'H', + 'H', + 'H', + 'H', + 'H', + 'H'] + +tr.write_nucleus_label(test_file,labels) + +rc = tr.close(test_file) + +#==========================================================# +#============ READ THE DATA FROM THE TEST FILE ============# +#==========================================================# + +test_file2 = tr.open(output_filename, 'r', TEST_TREXIO_BACKEND) + +rnum = tr.read_nucleus_num(test_file2) +assert rnum==nucleus_num + +# safe call to read_safe array of float values +rcharges_np = tr.read_safe_nucleus_charge(test_file2, nucleus_num) +assert rcharges_np.dtype is np.dtype(np.float64) +np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) + +# unsafe call to read_safe should not only have return code = TREXIO_UNSAFE_ARRAY_DIM +# TODO: it should not return numpy array filled with garbage +# rc, rcharges_fail = tr.read_safe_nucleus_charge(test_file2, nucleus_num*5) +# assert rc==TREXIO_UNSAFE_ARRAY_DIM + +# safe call to read_safe array of int values +rindices_np = tr.read_safe_basis_nucleus_index(test_file2, nucleus_num) +assert rindices_np.dtype is np.dtype(np.int32) +for i in range(nucleus_num): + assert rindices_np[i]==indices_np[i] + +rlabels_2d = tr.read_nucleus_label(test_file2) +print(rlabels_2d) +for i in range(nucleus_num): + assert rlabels_2d[i]==labels[i] + +rpoint_group = tr.read_nucleus_point_group(test_file2) +print(rpoint_group) +assert rpoint_group==point_group + +tr.close(test_file2) + +try: + if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: + os.remove(output_filename) + elif TEST_TREXIO_BACKEND == tr.TREXIO_TEXT: + shutil.rmtree(output_filename) +except: + print (f'No output file {output_filename} has been produced') + +#==========================================================# + diff --git a/python/test/test_py.py b/python/test/test_pytrexio.py similarity index 83% rename from python/test/test_py.py rename to python/test/test_pytrexio.py index b00da59..76c0e96 100644 --- a/python/test/test_py.py +++ b/python/test/test_pytrexio.py @@ -2,11 +2,11 @@ import os import shutil import numpy as np -from pytrexio import * +from trexio.pytrexio import * # TODO: # 1) make a user-friendly more pythonic API that will have to be autogenerated -# 2) add Exception handling (can be done easily in the front end python-ic API e.g. try: if function_call(...) == TREXIO_SUCCESS +# 2) add Exception handling (can be done easily in the front end python-ic API e.g. try: if function_call(...) == 0 # automatically download (hopefully the latest version) numpy.i using # wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i @@ -17,23 +17,23 @@ from pytrexio import * #=========================================================# # 0: TREXIO_HDF5 ; 1: TREXIO_TEXT -TEST_TREXIO_BACKEND = TREXIO_TEXT +TEST_TREXIO_BACKEND = 1 OUTPUT_FILENAME_TEXT = 'test_py_swig.dir' OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' -if TEST_TREXIO_BACKEND == TREXIO_HDF5: +if TEST_TREXIO_BACKEND == 0: output_filename = OUTPUT_FILENAME_HDF5 -elif TEST_TREXIO_BACKEND == TREXIO_TEXT: +elif TEST_TREXIO_BACKEND == 1: output_filename = OUTPUT_FILENAME_TEXT else: raise ValueError ('Specify one of the supported back ends as TEST_TREXIO_BACKEND') try: - if TEST_TREXIO_BACKEND == TREXIO_HDF5: + if TEST_TREXIO_BACKEND == 0: os.remove(output_filename) - elif TEST_TREXIO_BACKEND == TREXIO_TEXT: + elif TEST_TREXIO_BACKEND == 1: shutil.rmtree(output_filename) except: print ('Nothing to remove.') @@ -47,16 +47,16 @@ test_file = trexio_open(output_filename, 'w', TEST_TREXIO_BACKEND) nucleus_num = 12 rc = trexio_write_nucleus_num(test_file, nucleus_num) -assert rc==TREXIO_SUCCESS +assert rc==0 # initialize charge arrays as a list and convert it to numpy array charges = [6., 6., 6., 6., 6., 6., 1., 1., 1., 1., 1., 1.] charges_np = np.array(charges, dtype=np.float64) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived -# from the size of the list/array by SWIG using typemacs from numpy.i +# from the size of the list/array by SWIG using typemaps from numpy.i rc = trexio_write_safe_nucleus_charge(test_file, charges_np) -assert rc==TREXIO_SUCCESS +assert rc==0 # less Python-ic way to read/write arrays using Array classes (probably more portable to other languages) #charges = doubleArray(nucleus_num) @@ -75,12 +75,12 @@ indices_np = np.array(indices, dtype=np.int32) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived # from the size of the list/array by SWIG using typemacs from numpy.i rc = trexio_write_safe_basis_nucleus_index(test_file, indices_np) -assert rc==TREXIO_SUCCESS +assert rc==0 point_group = 'B3U' rc = trexio_write_nucleus_point_group(test_file, point_group, 10) -assert rc==TREXIO_SUCCESS +assert rc==0 labels = [ 'C', @@ -97,10 +97,10 @@ labels = [ 'H'] rc = trexio_write_nucleus_label(test_file, labels, 10) -assert rc==TREXIO_SUCCESS +assert rc==0 rc = trexio_close(test_file) -assert rc==TREXIO_SUCCESS +assert rc==0 #==========================================================# #============ READ THE DATA FROM THE TEST FILE ============# @@ -108,36 +108,36 @@ assert rc==TREXIO_SUCCESS test_file2 = trexio_open(output_filename, 'r', TEST_TREXIO_BACKEND) -# TODO: think about adding exception handling if rc != TREXIO_SUCCESS throw an exception but do not return the code itself and then there is less arguments +# TODO: think about adding exception handling if rc != 0 throw an exception but do not return the code itself and then there is less arguments # TODO: maybe also for the write result = trexio_read_nucleus_num(test_file2) -assert result[0]==TREXIO_SUCCESS +assert result[0]==0 assert result[1]==nucleus_num # safe call to read_safe array of float values rc, rcharges_np = trexio_read_safe_nucleus_charge(test_file2, nucleus_num) -assert rc==TREXIO_SUCCESS +assert rc==0 assert rcharges_np.dtype is np.dtype(np.float64) np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) # unsafe call to read_safe should not only have return code = TREXIO_UNSAFE_ARRAY_DIM # TODO: it should not return numpy array filled with garbage rc, rcharges_fail = trexio_read_safe_nucleus_charge(test_file2, nucleus_num*5) -assert rc==TREXIO_UNSAFE_ARRAY_DIM +#assert rc==TREXIO_UNSAFE_ARRAY_DIM # less Python-ic way to read/write arrays using Array classes (probably more portable to other languages) #charges2 = doubleArray(nucleus_num) #for i in range(nucleus_num): # charges2[i] = -1. #rc = trexio_read_nucleus_charge(test_file2, charges2) -#assert rc==TREXIO_SUCCESS +#assert rc==0 #for i in range(nucleus_num): # assert charges2[i]==charges[i] # safe call to read_safe array of int values rc, rindices_np = trexio_read_safe_basis_nucleus_index(test_file2, nucleus_num) -assert rc==TREXIO_SUCCESS +assert rc==0 assert rindices_np.dtype is np.dtype(np.int32) for i in range(nucleus_num): assert rindices_np[i]==indices_np[i] @@ -147,20 +147,25 @@ for i in range(nucleus_num): #rc, label_2d = trexio_read_nucleus_label(test_file2, 10) # [WIP]: currently only low-level routines (return one long string instead of an array of strings) work rc, labels_1d = trexio_read_nucleus_label_low(test_file2, 10) -assert rc==TREXIO_SUCCESS +assert rc==0 labels_2d = [label for label in labels_1d.split(TREXIO_DELIM) if label] print('Read and parsed nuclear labels:\n', labels_2d) for i in range(nucleus_num): assert labels_2d[i]==labels[i] +rc, rpoint_group = trexio_read_nucleus_point_group(test_file2, 3) +print(f'Read point group: {rpoint_group}') +assert rc==0 +assert rpoint_group==point_group + rc = trexio_close(test_file2) -assert rc==TREXIO_SUCCESS +assert rc==0 try: - if TEST_TREXIO_BACKEND == TREXIO_HDF5: + if TEST_TREXIO_BACKEND == 0: os.remove(output_filename) - elif TEST_TREXIO_BACKEND == TREXIO_TEXT: + elif TEST_TREXIO_BACKEND == 1: shutil.rmtree(output_filename) except: print (f'No output file {output_filename} has been produced') From 4ebd13b0400294680e840485df174c760353d571 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 13:46:16 +0300 Subject: [PATCH 046/107] change pytrexio to trexio + comments on compatibility tags and editable installations --- python/install_pytrexio.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index 65fb3a0..d85b8a3 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -5,7 +5,6 @@ set -e # This script should update the version of the Python package #source version.py -#exit 0 # Install/upgrade packages required for the installation python3 -m pip install --upgrade setuptools wheel twine @@ -25,23 +24,28 @@ python3 setup.py sdist bdist_wheel # Install pytrexio in the current environment from the aforementioned wheel # --force-reinstall is needed here because build-system pre-installs pytrexio in the environment # but does not install things in the corresponding site-packages directory -python3 -m pip install dist/pytrexio-0.1-*.whl --force-reinstall +python3 -m pip install dist/trexio-0.1-*.whl --force-reinstall + +# Run the command below in the root directory to install the package in 'editable' (-e) mode without dependencies (--no-deps) +#python -m pip install -e . --no-deps # Uninstall pytrexio: this only works from the pytrexio root directory (more likely python/ folder) and only after --force-reinstall -#python3 -m pip uninstall pytrexio +#python3 -m pip uninstall trexio # Test the current release by uploading to TestPyPI sandbox -#python3 -m twine upload --repository testpypi dist/pytrexio-*.tar.gz +#python3 -m twine upload --repository testpypi dist/trexio-*.tar.gz # NOTE:I get an error: -# Binary wheel 'pytrexio-0.1-cp38-cp38-linux_x86_64.whl' has an unsupported platform tag 'linux_x86_64'. +# Binary wheel 'trexio-0.1-cp38-cp38-linux_x86_64.whl' has an unsupported platform tag 'linux_x86_64'. # when uploading the wheel instead of the tar.gz file to testpypi # This is a well-known issue, see https://stackoverflow.com/questions/59451069/binary-wheel-cant-be-uploaded-on-pypi-using-twine -# Once fixed both .tar.gz and .whl distribution can be uploaded, meaning that dist/pytrexio-*.tar.gz can be replaced by dist/* +# Once fixed both .tar.gz and .whl distribution can be uploaded, meaning that dist/trexio-*.tar.gz can be replaced by dist/* +# +# Compatibility tags for Linux have been discussed in PEP-513 https://www.python.org/dev/peps/pep-0513/ # Upload updated version of PyTREXIO to PyPI -#python3 -m twine upload dist/pytrexio-*.tar.gz +#python3 -m twine upload dist/trexio-*.tar.gz # Cleaning -rm -rf build dist pytrexio.egg-info +rm -rf build dist trexio.egg-info From df91b9e7e5bba036f3b35f438587392d66c9caea Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 15:02:54 +0300 Subject: [PATCH 047/107] better cleaning of the python part --- Makefile.am | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile.am b/Makefile.am index f1ede0d..2bda741 100644 --- a/Makefile.am +++ b/Makefile.am @@ -184,16 +184,17 @@ cppcheck.out: $(trexio_h) -I../include *.c *.h 2>../$@ -setup_py = $(srcdir)/python/setup.py -setup_cfg = $(srcdir)/python/setup.cfg -pytrexio_py = $(srcdir)/python/trexio/pytrexio.py -TEST_PY = $(srcdir)/python/test/test_full.py -pytrexio_c = $(srcdir)/src/pytrexio_wrap.c -pytrexio_i = $(srcdir)/src/pytrexio.i -numpy_i = $(srcdir)/src/numpy.i +setup_py = $(srcdir)/python/setup.py +setup_cfg = $(srcdir)/python/setup.cfg +pytrexio_py = $(srcdir)/python/trexio/pytrexio.py +trexio_api_py = $(srcdir)/python/trexio/trexio_api.py +TEST_PY = $(srcdir)/python/test/test_api.py +pytrexio_c = $(srcdir)/src/pytrexio_wrap.c +pytrexio_i = $(srcdir)/src/pytrexio.i +numpy_i = $(srcdir)/src/numpy.i -python-test: $(pytrexio_py) $(TEST_PY) +python-test: $(TEST_PY) python3 $(TEST_PY) $(RM) -r -- __pycache__ @@ -216,12 +217,11 @@ $(numpy_i): check-numpy: cd tools && ./check_numpy_i.sh -CLEANFILES += src/pytrexio_wrap.c \ - src/pytrexio.py \ - python/trexio/pytrexio.py \ - python/trexio/trexio.py \ - python/src/*.c \ - python/src/*.h +CLEANFILES += $(pytrexio_c) \ + $(pytrexio_py) \ + $(trexio_api_py) \ + python/src/*.c \ + python/src/*.h .PHONY: cppcheck python-test python-install check-numpy From 902147b45b555e9821479c1f8df8d7aa1299ce18 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 15:40:42 +0300 Subject: [PATCH 048/107] better structure of the python directory --- Makefile.am | 6 +++--- python/{trexio => pytrexio}/__init__.py | 0 python/setup.py | 5 +++-- src/templates_front/build.sh | 4 ++-- src/templates_front/templator_front.org | 2 +- tools/prepare_python.sh | 4 ++-- 6 files changed, 11 insertions(+), 10 deletions(-) rename python/{trexio => pytrexio}/__init__.py (100%) diff --git a/Makefile.am b/Makefile.am index 2bda741..bc6fa1e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -186,8 +186,8 @@ cppcheck.out: $(trexio_h) setup_py = $(srcdir)/python/setup.py setup_cfg = $(srcdir)/python/setup.cfg -pytrexio_py = $(srcdir)/python/trexio/pytrexio.py -trexio_api_py = $(srcdir)/python/trexio/trexio_api.py +pytrexio_py = $(srcdir)/python/pytrexio/pytrexio.py +trexio_py = $(srcdir)/python/trexio.py TEST_PY = $(srcdir)/python/test/test_api.py pytrexio_c = $(srcdir)/src/pytrexio_wrap.c pytrexio_i = $(srcdir)/src/pytrexio.i @@ -219,7 +219,7 @@ check-numpy: CLEANFILES += $(pytrexio_c) \ $(pytrexio_py) \ - $(trexio_api_py) \ + $(trexio_py) \ python/src/*.c \ python/src/*.h diff --git a/python/trexio/__init__.py b/python/pytrexio/__init__.py similarity index 100% rename from python/trexio/__init__.py rename to python/pytrexio/__init__.py diff --git a/python/setup.py b/python/setup.py index a837fe1..6967a1c 100644 --- a/python/setup.py +++ b/python/setup.py @@ -15,7 +15,7 @@ with open("README.md", "r") as fh: long_description = fh.read() -pytrexio_module = Extension('trexio._pytrexio', +pytrexio_module = Extension('pytrexio._pytrexio', sources = [os.path.join(srcpath, code) for code in c_files], include_dirs = ['/usr/include/hdf5/serial', srcpath], libraries = ['hdf5', 'hdf5_hl'], @@ -32,7 +32,8 @@ setup(name = 'trexio', long_description = long_description, long_description_content_type = "text/markdown", ext_modules = [pytrexio_module], - packages = ['trexio'], + py_modules = ['trexio'], + packages = ['pytrexio'], url = 'https://github.com/TREX-CoE/trexio', license = 'BSD', classifiers=[ diff --git a/src/templates_front/build.sh b/src/templates_front/build.sh index aa0d114..5853c27 100644 --- a/src/templates_front/build.sh +++ b/src/templates_front/build.sh @@ -10,7 +10,7 @@ echo "" >> trexio.h cat prefix_s_front.h > trexio_s.h cat prefix_fortran.f90 > trexio_f.f90 -cat prefix_python.py > trexio_api.py +cat prefix_python.py > trexio.py # c front end cat populated/pop_*.c >> trexio.c @@ -23,7 +23,7 @@ cat helper_fortran.f90 >> trexio_f.f90 cat populated/pop_*.fh_90 >> trexio_f.f90 # python front end -cat populated/pop_*.py >> trexio_api.py +cat populated/pop_*.py >> trexio.py # suffixes cat suffix_s_front.h >> trexio_s.h diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 89fc466..5bf2d43 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -21,7 +21,7 @@ ** Python #+begin_src python :tangle prefix_python.py try: - from trexio.pytrexio import * + from pytrexio.pytrexio import * except ImportError: raise Exception("Could not import pytrexio module from trexio package") diff --git a/tools/prepare_python.sh b/tools/prepare_python.sh index 7224e2b..8b21506 100755 --- a/tools/prepare_python.sh +++ b/tools/prepare_python.sh @@ -13,7 +13,7 @@ readonly SRC=${TREXIO_ROOT}/src readonly INCLUDIR=${TREXIO_ROOT}/include readonly TOOLS=${TREXIO_ROOT}/tools readonly PYDIR=${TREXIO_ROOT}/python -readonly PYTREXIODIR=${PYDIR}/trexio +readonly PYTREXIODIR=${PYDIR}/pytrexio # We want the script to crash on the 1st error: set -e @@ -24,7 +24,7 @@ mkdir -p ${PYTREXIODIR} # Copy all the source code and header files in the corresponding python directory mv ${SRC}/pytrexio.py ${PYTREXIODIR} -mv ${SRC}/trexio_api.py ${PYTREXIODIR} +mv ${SRC}/trexio.py ${PYDIR} cp ${SRC}/*.c ${PYDIR}/src cp ${SRC}/*.h ${PYDIR}/src cp ${INCLUDIR}/trexio.h ${PYDIR}/src From 8d8658fabf72b7dbc15e5c75d7d85cdec049993e Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 15:42:10 +0300 Subject: [PATCH 049/107] minor cleaning + exception handling for an unsafe call to the safe API --- python/test/test_api.py | 15 +++++++-------- python/test/test_pytrexio.py | 20 +++----------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index f5bf278..f5bfc4b 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -2,7 +2,7 @@ import os import shutil import numpy as np -import trexio.trexio_api as tr +import trexio as tr #=========================================================# #======== SETUP THE BACK END AND OUTPUT FILE NAME ========# @@ -41,8 +41,6 @@ print(test_file) nucleus_num = 12 tr.write_nucleus_num(test_file, nucleus_num) -#rc = pytr.trexio_write_nucleus_num(test_file, nucleus_num) -#assert rc==tr.TREXIO_SUCCESS # initialize charge arrays as a list and convert it to numpy array charges = [6., 6., 6., 6., 6., 6., 1., 1., 1., 1., 1., 1.] @@ -81,7 +79,7 @@ labels = [ tr.write_nucleus_label(test_file,labels) -rc = tr.close(test_file) +tr.close(test_file) #==========================================================# #============ READ THE DATA FROM THE TEST FILE ============# @@ -97,10 +95,11 @@ rcharges_np = tr.read_safe_nucleus_charge(test_file2, nucleus_num) assert rcharges_np.dtype is np.dtype(np.float64) np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) -# unsafe call to read_safe should not only have return code = TREXIO_UNSAFE_ARRAY_DIM -# TODO: it should not return numpy array filled with garbage -# rc, rcharges_fail = tr.read_safe_nucleus_charge(test_file2, nucleus_num*5) -# assert rc==TREXIO_UNSAFE_ARRAY_DIM +# unsafe call to read_safe should fail with error message corresponding to TREXIO_UNSAFE_ARRAY_DIM +try: + rcharges_fail = tr.read_safe_nucleus_charge(test_file2, nucleus_num*5) +except Exception: + print("Unsafe call to safe API: successful") # safe call to read_safe array of int values rindices_np = tr.read_safe_basis_nucleus_index(test_file2, nucleus_num) diff --git a/python/test/test_pytrexio.py b/python/test/test_pytrexio.py index 76c0e96..b048bfe 100644 --- a/python/test/test_pytrexio.py +++ b/python/test/test_pytrexio.py @@ -2,15 +2,7 @@ import os import shutil import numpy as np -from trexio.pytrexio import * - -# TODO: -# 1) make a user-friendly more pythonic API that will have to be autogenerated -# 2) add Exception handling (can be done easily in the front end python-ic API e.g. try: if function_call(...) == 0 - -# automatically download (hopefully the latest version) numpy.i using -# wget https://raw.githubusercontent.com/numpy/numpy/main/tools/swig/numpy.i -# but also keep the original copy in case if wget fails +from pytrexio.pytrexio import * #=========================================================# #======== SETUP THE BACK END AND OUTPUT FILE NAME ========# @@ -108,9 +100,6 @@ assert rc==0 test_file2 = trexio_open(output_filename, 'r', TEST_TREXIO_BACKEND) -# TODO: think about adding exception handling if rc != 0 throw an exception but do not return the code itself and then there is less arguments -# TODO: maybe also for the write - result = trexio_read_nucleus_num(test_file2) assert result[0]==0 assert result[1]==nucleus_num @@ -124,7 +113,7 @@ np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) # unsafe call to read_safe should not only have return code = TREXIO_UNSAFE_ARRAY_DIM # TODO: it should not return numpy array filled with garbage rc, rcharges_fail = trexio_read_safe_nucleus_charge(test_file2, nucleus_num*5) -#assert rc==TREXIO_UNSAFE_ARRAY_DIM +assert rc==TREXIO_UNSAFE_ARRAY_DIM # less Python-ic way to read/write arrays using Array classes (probably more portable to other languages) #charges2 = doubleArray(nucleus_num) @@ -142,10 +131,7 @@ assert rindices_np.dtype is np.dtype(np.int32) for i in range(nucleus_num): assert rindices_np[i]==indices_np[i] - -# [WIP]: ideally, the list of strings should be returned as below -#rc, label_2d = trexio_read_nucleus_label(test_file2, 10) -# [WIP]: currently only low-level routines (return one long string instead of an array of strings) work +# currently only low-level routines (return one long string instead of an array of strings) work rc, labels_1d = trexio_read_nucleus_label_low(test_file2, 10) assert rc==0 From 831973fc8ed2e9cb24f3b4f86157e816aa2ace4d Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 18 Aug 2021 16:28:43 +0300 Subject: [PATCH 050/107] better python compilation --- src/templates_front/build.sh | 1 + src/templates_front/templator_front.org | 64 ++++--------------------- tools/prepare_python.sh | 4 +- 3 files changed, 13 insertions(+), 56 deletions(-) diff --git a/src/templates_front/build.sh b/src/templates_front/build.sh index 5853c27..b0b0ddb 100644 --- a/src/templates_front/build.sh +++ b/src/templates_front/build.sh @@ -23,6 +23,7 @@ cat helper_fortran.f90 >> trexio_f.f90 cat populated/pop_*.fh_90 >> trexio_f.f90 # python front end +cat basic_python.py >> trexio.py cat populated/pop_*.py >> trexio.py # suffixes diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 5bf2d43..2a06d0a 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -25,7 +25,6 @@ try: except ImportError: raise Exception("Could not import pytrexio module from trexio package") - # define TREXIO back ends TREXIO_HDF5 = 0 TREXIO_TEXT = 1 @@ -35,7 +34,6 @@ TREXIO_INVALID_BACK_END = 2 # define max length of string item when reading arrays of strings # this is needed for the low-level C routines PYTREXIO_MAX_STR_LENGTH = 2048 - #+end_src ** Fortran @@ -573,6 +571,8 @@ struct trexio_back_end_s { are hard-coded), which is why the user should tend to avoid renaming the ~.txt~ data files. +*** C + #+begin_src c :tangle prefix_front.h :exports none trexio_t* trexio_open(const char* file_name, const char mode, const back_end_t back_end); #+end_src @@ -726,6 +726,7 @@ trexio_open(const char* file_name, const char mode, } #+end_src +*** Fortran #+begin_src f90 :tangle prefix_fortran.f90 interface integer(8) function trexio_open_c (filename, mode, backend) bind(C, name="trexio_open") @@ -738,12 +739,9 @@ interface end interface #+end_src - #+begin_src c :tangle prefix_python.py +*** Python + #+begin_src c :tangle basic_python.py def open(file_name: str, mode: str, back_end: int): - #try: - # from trexio.pytrexio import trexio_open - #except ImportError: - # raise try: trexio_file = trexio_open(file_name, mode, back_end) @@ -751,9 +749,9 @@ def open(file_name: str, mode: str, back_end: int): raise return trexio_file - #+end_src +*** Zero-based versus one-based arrays of indices Because arrays are zero-based in Fortran, we need to set a flag to know if we need to shift by 1 arrays of indices. @@ -792,6 +790,7 @@ end interface output: ~trexio_exit_code~ exit code. +*** C #+begin_src c :tangle prefix_front.h :exports none trexio_exit_code trexio_close(trexio_t* file); #+end_src @@ -862,6 +861,7 @@ trexio_close (trexio_t* file) } #+end_src +*** Fortran #+begin_src f90 :tangle prefix_fortran.f90 interface integer function trexio_close (trex_file) bind(C) @@ -871,12 +871,9 @@ interface end interface #+end_src - #+begin_src c :tangle prefix_python.py +*** Python + #+begin_src c :tangle basic_python.py def close(trexio_file): - #try: - # from trexio.pytrexio import trexio_close, trexio_string_of_error - #except ImportError: - # raise try: rc = trexio_close(trexio_file) @@ -1233,10 +1230,6 @@ end interface #+begin_src python :tangle write_num_front.py def write_$group_num$(trexio_file, num_w) -> None: - #try: - # from trexio.pytrexio import trexio_write_$group_num$, trexio_string_of_error - #except ImportError: - # raise try: rc = trexio_write_$group_num$(trexio_file, num_w) @@ -1250,10 +1243,6 @@ def write_$group_num$(trexio_file, num_w) -> None: #+begin_src python :tangle read_num_front.py def read_$group_num$(trexio_file): - #try: - # from trexio.pytrexio import trexio_read_$group_num$, trexio_string_of_error - #except ImportError: - # raise try: rc, num_r = trexio_read_$group_num$(trexio_file) @@ -1761,10 +1750,6 @@ end interface #+begin_src python :tangle write_dset_data_front.py def write_safe_$group_dset$(trexio_file, dset_w) -> None: - #try: - # from trexio.pytrexio import trexio_write_safe_$group_dset$, trexio_string_of_error - #except ImportError: - # raise try: rc = trexio_write_safe_$group_dset$(trexio_file, dset_w) @@ -1778,10 +1763,6 @@ def write_safe_$group_dset$(trexio_file, dset_w) -> None: #+begin_src python :tangle read_dset_data_front.py def read_safe_$group_dset$(trexio_file, dim): - #try: - # from trexio.pytrexio import trexio_read_safe_$group_dset$, trexio_string_of_error - #except ImportError: - # raise try: rc, dset_r = trexio_read_safe_$group_dset$(trexio_file, dim) @@ -2254,12 +2235,6 @@ end interface #+begin_src python :tangle write_dset_str_front.py def write_$group_dset$(trexio_file, dset_w) -> None: - #try: - # from trexio.pytrexio import (trexio_write_$group_dset$, - # trexio_string_of_error - # ) - #except ImportError: - # raise max_str_length = len(max(dset_w, key=len)) + 1 @@ -2275,13 +2250,6 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+begin_src python :tangle read_dset_str_front.py def read_$group_dset$(trexio_file): - #try: - # from trexio.pytrexio import (trexio_read_$group_dset$_low, - # trexio_string_of_error, - # TREXIO_DELIM - # ) - #except ImportError: - # raise try: rc, dset_1d_r = trexio_read_$group_dset$_low(trexio_file, PYTREXIO_MAX_STR_LENGTH) @@ -2481,12 +2449,6 @@ end interface #+begin_src python :tangle write_attr_str_front.py def write_$group_str$(trexio_file, str_w) -> None: - #try: - # from trexio.pytrexio import (trexio_write_$group_str$, - # trexio_string_of_error, - # ) - #except ImportError: - # raise max_str_length = len(str_w) + 1 @@ -2502,12 +2464,6 @@ def write_$group_str$(trexio_file, str_w) -> None: #+begin_src python :tangle read_attr_str_front.py def read_$group_str$(trexio_file): - #try: - # from trexio.pytrexio import (trexio_read_$group_str$, - # trexio_string_of_error - # ) - #except ImportError: - # raise try: rc, str_r = trexio_read_$group_str$(trexio_file, PYTREXIO_MAX_STR_LENGTH) diff --git a/tools/prepare_python.sh b/tools/prepare_python.sh index 8b21506..b700af4 100755 --- a/tools/prepare_python.sh +++ b/tools/prepare_python.sh @@ -23,8 +23,8 @@ mkdir -p ${PYDIR}/src mkdir -p ${PYTREXIODIR} # Copy all the source code and header files in the corresponding python directory -mv ${SRC}/pytrexio.py ${PYTREXIODIR} -mv ${SRC}/trexio.py ${PYDIR} +mv ${SRC}/pytrexio.py ${PYTREXIODIR}/pytrexio.py +mv ${SRC}/trexio.py ${PYDIR}/trexio.py cp ${SRC}/*.c ${PYDIR}/src cp ${SRC}/*.h ${PYDIR}/src cp ${INCLUDIR}/trexio.h ${PYDIR}/src From 5d5025ae1dfea84abbbb446633d77e9f3220b8f2 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 20 Aug 2021 13:43:15 +0300 Subject: [PATCH 051/107] get rid of _safe suffix and add dimension to read_dset_str functions --- python/test/test_api.py | 12 ++++++------ src/templates_front/templator_front.org | 9 +++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index f5bfc4b..93f2e12 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -48,7 +48,7 @@ charges_np = np.array(charges, dtype=np.float64) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived # from the size of the list/array by SWIG using typemaps from numpy.i -rc = tr.write_safe_nucleus_charge(test_file, charges_np) +rc = tr.write_nucleus_charge(test_file, charges_np) # initialize arrays of nuclear indices as a list and convert it to numpy array indices = [i for i in range(nucleus_num)] @@ -57,7 +57,7 @@ indices_np = np.array(indices, dtype=np.int32) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived # from the size of the list/array by SWIG using typemacs from numpy.i -tr.write_safe_basis_nucleus_index(test_file, indices_np) +tr.write_basis_nucleus_index(test_file, indices_np) point_group = 'B3U' @@ -91,23 +91,23 @@ rnum = tr.read_nucleus_num(test_file2) assert rnum==nucleus_num # safe call to read_safe array of float values -rcharges_np = tr.read_safe_nucleus_charge(test_file2, nucleus_num) +rcharges_np = tr.read_nucleus_charge(test_file2, dim=nucleus_num) assert rcharges_np.dtype is np.dtype(np.float64) np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) # unsafe call to read_safe should fail with error message corresponding to TREXIO_UNSAFE_ARRAY_DIM try: - rcharges_fail = tr.read_safe_nucleus_charge(test_file2, nucleus_num*5) + rcharges_fail = tr.read_nucleus_charge(test_file2, dim=nucleus_num*5) except Exception: print("Unsafe call to safe API: successful") # safe call to read_safe array of int values -rindices_np = tr.read_safe_basis_nucleus_index(test_file2, nucleus_num) +rindices_np = tr.read_basis_nucleus_index(test_file2, dim=nucleus_num) assert rindices_np.dtype is np.dtype(np.int32) for i in range(nucleus_num): assert rindices_np[i]==indices_np[i] -rlabels_2d = tr.read_nucleus_label(test_file2) +rlabels_2d = tr.read_nucleus_label(test_file2, dim=nucleus_num) print(rlabels_2d) for i in range(nucleus_num): assert rlabels_2d[i]==labels[i] diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 2a06d0a..91898cf 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -1749,7 +1749,7 @@ end interface *** Python templates for front end #+begin_src python :tangle write_dset_data_front.py -def write_safe_$group_dset$(trexio_file, dset_w) -> None: +def write_$group_dset$(trexio_file, dset_w) -> None: try: rc = trexio_write_safe_$group_dset$(trexio_file, dset_w) @@ -1762,7 +1762,7 @@ def write_safe_$group_dset$(trexio_file, dset_w) -> None: #+end_src #+begin_src python :tangle read_dset_data_front.py -def read_safe_$group_dset$(trexio_file, dim): +def read_$group_dset$(trexio_file, dim): try: rc, dset_r = trexio_read_safe_$group_dset$(trexio_file, dim) @@ -2249,7 +2249,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+end_src #+begin_src python :tangle read_dset_str_front.py -def read_$group_dset$(trexio_file): +def read_$group_dset$(trexio_file, dim): try: rc, dset_1d_r = trexio_read_$group_dset$_low(trexio_file, PYTREXIO_MAX_STR_LENGTH) @@ -2260,7 +2260,8 @@ def read_$group_dset$(trexio_file): raise try: - dset_2d_r = [item for item in dset_1d_r.split(TREXIO_DELIM) if item] + dset_full = dset_1d_r.split(TREXIO_DELIM) + dset_2d_r = [dset_full[i] for i in range(dim) if dset_full[i]] assert dset_2d_r except AssertionError: raise TypeError(f"Output of {read_$group_dset$.__name__} function cannot be an empty list.") From aee68dd619699482de7817099d31b4a03f686138 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 20 Aug 2021 15:20:09 +0300 Subject: [PATCH 052/107] add Autoconf macro for SWIG --- configure.ac | 4 ++ m4/ax_pkg_swig.m4 | 139 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 m4/ax_pkg_swig.m4 diff --git a/configure.ac b/configure.ac index 5c1233a..9057ce6 100644 --- a/configure.ac +++ b/configure.ac @@ -126,6 +126,10 @@ if test "x${TREXIO_DEVEL}" != "x"; then AC_PROG_AWK AM_PATH_PYTHON([3.0]) + AX_PKG_SWIG(4.0.0, [], AC_MSG_WARN([SWIG is required to build Python API.]) ) + # The macro below performs Python benchmarks, but overlapping with AM_PATH_PYTHON + #AX_SWIG_PYTHON + AC_CHECK_PROGS([EMACS],[emacs26 emacs],[no]) if test x${EMACS} == xno ; then AC_MSG_ERROR([ diff --git a/m4/ax_pkg_swig.m4 b/m4/ax_pkg_swig.m4 new file mode 100644 index 0000000..b1f77f0 --- /dev/null +++ b/m4/ax_pkg_swig.m4 @@ -0,0 +1,139 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_pkg_swig.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_PKG_SWIG([major.minor.micro], [action-if-found], [action-if-not-found]) +# +# DESCRIPTION +# +# This macro searches for a SWIG installation on your system. If found, +# then SWIG is AC_SUBST'd; if not found, then $SWIG is empty. If SWIG is +# found, then SWIG_LIB is set to the SWIG library path, and AC_SUBST'd. +# +# You can use the optional first argument to check if the version of the +# available SWIG is greater than or equal to the value of the argument. It +# should have the format: N[.N[.N]] (N is a number between 0 and 999. Only +# the first N is mandatory.) If the version argument is given (e.g. +# 1.3.17), AX_PKG_SWIG checks that the swig package is this version number +# or higher. +# +# As usual, action-if-found is executed if SWIG is found, otherwise +# action-if-not-found is executed. +# +# In configure.in, use as: +# +# AX_PKG_SWIG(1.3.17, [], [ AC_MSG_ERROR([SWIG is required to build..]) ]) +# AX_SWIG_ENABLE_CXX +# AX_SWIG_MULTI_MODULE_SUPPORT +# AX_SWIG_PYTHON +# +# LICENSE +# +# Copyright (c) 2008 Sebastian Huber +# Copyright (c) 2008 Alan W. Irwin +# Copyright (c) 2008 Rafael Laboissiere +# Copyright (c) 2008 Andrew Collier +# Copyright (c) 2011 Murray Cumming +# Copyright (c) 2021 Vincent Danjean +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 14 + +AC_DEFUN([AX_PKG_SWIG],[ + # Find path to the "swig" executable. + AC_PATH_PROGS([SWIG],[swig swig3.0 swig2.0]) + if test -z "$SWIG" ; then + m4_ifval([$3],[$3],[:]) + elif test -z "$1" ; then + m4_ifval([$2],[$2],[:]) + else + AC_MSG_CHECKING([SWIG version]) + [swig_version=`$SWIG -version 2>&1 | grep 'SWIG Version' | sed 's/.*\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/g'`] + AC_MSG_RESULT([$swig_version]) + if test -n "$swig_version" ; then + # Calculate the required version number components + [required=$1] + [required_major=`echo $required | sed 's/[^0-9].*//'`] + if test -z "$required_major" ; then + [required_major=0] + fi + [required=`echo $required. | sed 's/[0-9]*[^0-9]//'`] + [required_minor=`echo $required | sed 's/[^0-9].*//'`] + if test -z "$required_minor" ; then + [required_minor=0] + fi + [required=`echo $required. | sed 's/[0-9]*[^0-9]//'`] + [required_patch=`echo $required | sed 's/[^0-9].*//'`] + if test -z "$required_patch" ; then + [required_patch=0] + fi + # Calculate the available version number components + [available=$swig_version] + [available_major=`echo $available | sed 's/[^0-9].*//'`] + if test -z "$available_major" ; then + [available_major=0] + fi + [available=`echo $available | sed 's/[0-9]*[^0-9]//'`] + [available_minor=`echo $available | sed 's/[^0-9].*//'`] + if test -z "$available_minor" ; then + [available_minor=0] + fi + [available=`echo $available | sed 's/[0-9]*[^0-9]//'`] + [available_patch=`echo $available | sed 's/[^0-9].*//'`] + if test -z "$available_patch" ; then + [available_patch=0] + fi + # Convert the version tuple into a single number for easier comparison. + # Using base 100 should be safe since SWIG internally uses BCD values + # to encode its version number. + required_swig_vernum=`expr $required_major \* 10000 \ + \+ $required_minor \* 100 \+ $required_patch` + available_swig_vernum=`expr $available_major \* 10000 \ + \+ $available_minor \* 100 \+ $available_patch` + + if test $available_swig_vernum -lt $required_swig_vernum; then + AC_MSG_WARN([SWIG version >= $1 is required. You have $swig_version.]) + SWIG='' + m4_ifval([$3],[$3],[]) + else + AC_MSG_CHECKING([for SWIG library]) + SWIG_LIB=`$SWIG -swiglib` + AC_MSG_RESULT([$SWIG_LIB]) + m4_ifval([$2],[$2],[]) + fi + else + AC_MSG_WARN([cannot determine SWIG version]) + SWIG='' + m4_ifval([$3],[$3],[]) + fi + fi + AC_SUBST([SWIG_LIB]) +]) + From 09823dd6a038952c5685998254c3f71bee8bc773 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 20 Aug 2021 15:21:49 +0300 Subject: [PATCH 053/107] more portable build of the Python extension module with HDF5 paths propagated from configure script --- Makefile.am | 11 ++++++++--- python/install_pytrexio.sh | 5 ++++- python/setup.py | 7 +++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Makefile.am b/Makefile.am index bc6fa1e..c3402cd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -157,6 +157,10 @@ $(HTML_FILES): docs/index.html # =============== DEVELOPER MODE =============== # +SWIG = @SWIG@ +HDF5_LDFLAGS = @HDF5_LDFLAGS@ +HDF5_CFLAGS = @HDF5_CFLAGS@ + if TREXIO_DEVEL CLEANFILES += $(SOURCES) $(trexio_f) $(trexio_h) @@ -183,7 +187,6 @@ cppcheck.out: $(trexio_h) --language=c --std=c99 -rp --platform=unix64 \ -I../include *.c *.h 2>../$@ - setup_py = $(srcdir)/python/setup.py setup_cfg = $(srcdir)/python/setup.cfg pytrexio_py = $(srcdir)/python/pytrexio/pytrexio.py @@ -199,7 +202,8 @@ python-test: $(TEST_PY) $(RM) -r -- __pycache__ python-install: $(pytrexio_py) $(setup_py) $(setup_cfg) - cd python && ./install_pytrexio.sh + cd python && \ + ./install_pytrexio.sh $(HDF5_LDFLAGS) $(HDF5_CFLAGS) $(pytrexio_py): $(pytrexio_c) cd tools && ./prepare_python.sh @@ -208,7 +212,8 @@ $(pytrexio_py): $(pytrexio_c) # [?] swig -python -threads pytrexio.i ----> Add thread support for all the interface $(pytrexio_c): $(ORG_FILES) $(trexio_h) $(pytrexio_i) $(numpy_i) cp $(trexio_h) src/ - cd src/ && swig -python -py3 -o pytrexio_wrap.c pytrexio.i + cd src/ && \ + $(SWIG) -python -py3 -o pytrexio_wrap.c pytrexio.i $(RM) -- src/trexio.h $(numpy_i): diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index d85b8a3..148ef9d 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -3,6 +3,9 @@ set -x set -e +H5_LDFLAGS_LOCAL=$1 +H5_CFLAGS_LOCAL=$2 + # This script should update the version of the Python package #source version.py @@ -11,7 +14,7 @@ python3 -m pip install --upgrade setuptools wheel twine # Create build directory and compile extension files (*.c) # --no-user-cfg disables custom .cfg files of the user machine, so that only setup.cfg is used -python3 -s setup.py --no-user-cfg build +H5_LDFLAGS=${H5_LDFLAGS_LOCAL} H5_CFLAGS=${H5_CFLAGS_LOCAL} python3 -s setup.py --no-user-cfg build # Local inplace build of the .so module with SWIG-produced pytrexio_wrap.c (from the SWIG documentation) #python3 setup.py build_ext --inplace --swig-opts="-modern" diff --git a/python/setup.py b/python/setup.py index 6967a1c..367adbe 100644 --- a/python/setup.py +++ b/python/setup.py @@ -14,13 +14,16 @@ c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] with open("README.md", "r") as fh: long_description = fh.read() +h5_ldflags = str(os.environ.get("H5_LDFLAGS", None )) +h5_cflags_withI = str(os.environ.get("H5_CFLAGS", None )) +h5_cflags = h5_cflags_withI.replace("-I","") pytrexio_module = Extension('pytrexio._pytrexio', sources = [os.path.join(srcpath, code) for code in c_files], - include_dirs = ['/usr/include/hdf5/serial', srcpath], + include_dirs = [h5_cflags, srcpath], libraries = ['hdf5', 'hdf5_hl'], extra_compile_args = ['-Wno-discarded-qualifiers'], - extra_link_args = ['-L/usr/lib/x86_64-linux-gnu/hdf5/serial'] + extra_link_args = [h5_ldflags] ) From 9f674b1add960a799743973ae3d52546562186e2 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 20 Aug 2021 16:30:05 +0300 Subject: [PATCH 054/107] fix bug due to HDF5 name duplication in PKG_CHECK_MODULES --- configure.ac | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 9057ce6..4a4c0ca 100644 --- a/configure.ac +++ b/configure.ac @@ -94,18 +94,31 @@ CPPFLAGS="${HDF5_CPPFLAGS} ${CPPFLAGS}" LDFLAGS="${HDF5_LDFLAGS} ${LDFLAGS}" LIBS="${HDF5_LIBS} ${LIBS}" -# Check if HDF5 if available with pkg-config +# Check if HDF5 is available with pkg-config. +# First argument has to be named differently than HDF5 (e.g. HDF5_MOD) because for pkg FOO +# PKG_CHECK_MODULES(FOO,...) sets FOO_LIBS but we have already used HDF5_LIBS above. -PKG_CHECK_MODULES([HDF5], [hdf5 >= 1.8], [ +PKG_CHECK_MODULES([HDF5_MOD], [hdf5 >= 1.8], [ PKG_HDF5="hdf5" ],[ PKG_HDF5="" ]) - PKG_CFLAGS="${PKG_CFLAGS}" +PKG_CFLAGS="${HDF5_MOD_CFLAGS}" +PKG_LIBS="${HDF5_MOD_LIBS}" AC_SUBST([PKG_HDF5]) AC_SUBST([PKG_CFLAGS]) +AC_SUBST([PKG_LIBS]) +# The block below should only execute if the ax_lib_hdf5.m4 macro failed to find HDF5. +# It is only needed to manually build Python API because setup.py depends on HDF5. + +if test "$HDF5_LDFLAGS" = "" || "$HDF5_CFLAGS" = ""; then + HDF5_LDFLAGS="${PKG_LIBS}" + HDF5_CFLAGS="${PKG_CFLAGS}" + AC_SUBST([HDF5_CFLAGS]) + AC_SUBST([HDF5_LDFLAGS]) +fi # Checks for typedefs, structures, and compiler characteristics. AC_TYPE_INT32_T From 8b22494f2f16f244d0012b042481a6d1a8ff27b9 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 20 Aug 2021 16:31:22 +0300 Subject: [PATCH 055/107] LDFLAGS may contain several items and should be passed second --- Makefile.am | 2 +- python/install_pytrexio.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index c3402cd..e833fab 100644 --- a/Makefile.am +++ b/Makefile.am @@ -203,7 +203,7 @@ python-test: $(TEST_PY) python-install: $(pytrexio_py) $(setup_py) $(setup_cfg) cd python && \ - ./install_pytrexio.sh $(HDF5_LDFLAGS) $(HDF5_CFLAGS) + ./install_pytrexio.sh $(HDF5_CFLAGS) $(HDF5_LDFLAGS) $(pytrexio_py): $(pytrexio_c) cd tools && ./prepare_python.sh diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index 148ef9d..fd3dd09 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -3,8 +3,8 @@ set -x set -e -H5_LDFLAGS_LOCAL=$1 -H5_CFLAGS_LOCAL=$2 +H5_CFLAGS_LOCAL=$1 +H5_LDFLAGS_LOCAL=$2 # This script should update the version of the Python package #source version.py From ed47bad05684157cceedd0b39e9c3e55af98903a Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 21 Aug 2021 11:58:42 +0300 Subject: [PATCH 056/107] roll back names in PKG_CHECK_MODULES --- configure.ac | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 4a4c0ca..2c87548 100644 --- a/configure.ac +++ b/configure.ac @@ -50,6 +50,7 @@ AC_FC_LIBRARY_LDFLAGS # pkg-config PKG_PROG_PKG_CONFIG() PKG_CFLAGS="" +PKG_LIBS="" AC_PROG_INSTALL AC_PROG_LIBTOOL @@ -95,16 +96,14 @@ LDFLAGS="${HDF5_LDFLAGS} ${LDFLAGS}" LIBS="${HDF5_LIBS} ${LIBS}" # Check if HDF5 is available with pkg-config. -# First argument has to be named differently than HDF5 (e.g. HDF5_MOD) because for pkg FOO -# PKG_CHECK_MODULES(FOO,...) sets FOO_LIBS but we have already used HDF5_LIBS above. -PKG_CHECK_MODULES([HDF5_MOD], [hdf5 >= 1.8], [ +PKG_CHECK_MODULES([HDF5], [hdf5 >= 1.8], [ PKG_HDF5="hdf5" ],[ PKG_HDF5="" ]) -PKG_CFLAGS="${HDF5_MOD_CFLAGS}" -PKG_LIBS="${HDF5_MOD_LIBS}" +PKG_CFLAGS="${HDF5_CFLAGS}" +PKG_LIBS="${HDF5_LIBS}" AC_SUBST([PKG_HDF5]) AC_SUBST([PKG_CFLAGS]) From 4fe86ee980d86912064385815c82da6c8a695070 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 21 Aug 2021 12:47:36 +0300 Subject: [PATCH 057/107] add Python docstrings for API functions + move definition of Fortran and Python back ends from Constant Prefixes to Back Ends section --- src/templates_front/templator_front.org | 265 ++++++++++++++++++++---- 1 file changed, 220 insertions(+), 45 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 91898cf..2f64514 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -17,45 +17,9 @@ */ #+end_src - -** Python - #+begin_src python :tangle prefix_python.py -try: - from pytrexio.pytrexio import * -except ImportError: - raise Exception("Could not import pytrexio module from trexio package") - -# define TREXIO back ends -TREXIO_HDF5 = 0 -TREXIO_TEXT = 1 -#TREXIO_JSON = 2 -TREXIO_INVALID_BACK_END = 2 - -# define max length of string item when reading arrays of strings -# this is needed for the low-level C routines -PYTREXIO_MAX_STR_LENGTH = 2048 - #+end_src - -** Fortran - #+begin_src f90 :tangle prefix_fortran.f90 -module trexio - - use, intrinsic :: iso_c_binding - implicit none - - integer, parameter :: trexio_exit_code = 4 - integer, parameter :: trexio_backend = 4 - - integer(trexio_backend), parameter :: TREXIO_HDF5 = 0 - integer(trexio_backend), parameter :: TREXIO_TEXT = 1 -! integer(trexio_backend), parameter :: TREXIO_JSON = 2 - integer(trexio_backend), parameter :: TREXIO_INVALID_BACK_END = 2 - - character(kind=c_char), parameter :: TREXIO_DELIM = c_new_line - - #+end_src - + ** C + #+begin_src c :tangle prefix_front.h :noweb yes <
> #ifndef TREXIO_H @@ -102,6 +66,40 @@ typedef int32_t trexio_exit_code; #define _TREXIO_PRIVATE_H #+end_src +** Fortran + + #+begin_src f90 :tangle prefix_fortran.f90 +module trexio + + use, intrinsic :: iso_c_binding + implicit none + + integer, parameter :: trexio_exit_code = 4 + integer, parameter :: trexio_backend = 4 + + character(kind=c_char), parameter :: TREXIO_DELIM = c_new_line + #+end_src + +** Python + + #+begin_src python :tangle prefix_python.py +"""The Python API of the TREXIO library. + +This module contains wrappers of the pytrexio.py module produced by SWIG. +Top-level wrapper like this allows to extend functionality of SWIG-generated codes. +Moreover, exception handling becomes trivial thanks to the use of TREXIO exit codes. +""" + + +try: + from pytrexio.pytrexio import * +except ImportError: + raise Exception("Could not import pytrexio module from trexio package") + +# define max length of a string to be read, required for the low-level C routines +PYTREXIO_MAX_STR_LENGTH = 2048 + #+end_src + * Coding conventions - integer types will be defined using types given in ~stdint.h~ @@ -479,6 +477,10 @@ def string_of_error(return_code: int) -> str: Then the corresponding back-end ~has/read/write~ functions has to be implemented. For example, see the commented lines that correspond to the ~TREXIO_JSON~ back end (not implemented yet). + _Note_: It is important to increment the value of TREXIO_INVALID_BACK_END when a new back end is added. Otherwise, it will not be available. + +*** C + #+begin_src c :tangle prefix_front.h typedef int32_t back_end_t; @@ -489,6 +491,26 @@ typedef int32_t back_end_t; #define TREXIO_DELIM "\n" #+end_src + +*** Fortran + + #+begin_src f90 :tangle prefix_fortran.f90 + integer(trexio_backend), parameter :: TREXIO_HDF5 = 0 + integer(trexio_backend), parameter :: TREXIO_TEXT = 1 +! integer(trexio_backend), parameter :: TREXIO_JSON = 2 + integer(trexio_backend), parameter :: TREXIO_INVALID_BACK_END = 2 + #+end_src + +*** Python + + #+begin_src python :tangle prefix_python.py +# define TREXIO back ends +TREXIO_HDF5 = 0 +TREXIO_TEXT = 1 +#TREXIO_JSON = 2 +TREXIO_INVALID_BACK_END = 2 + #+end_src + ** Read/write behavior Every time a reading function is called, the data is read from the @@ -727,6 +749,7 @@ trexio_open(const char* file_name, const char mode, #+end_src *** Fortran + #+begin_src f90 :tangle prefix_fortran.f90 interface integer(8) function trexio_open_c (filename, mode, backend) bind(C, name="trexio_open") @@ -740,8 +763,30 @@ end interface #+end_src *** Python + #+begin_src c :tangle basic_python.py def open(file_name: str, mode: str, back_end: int): + """Create TREXIO file or open existing one. + + Parameters: + + file_name: str + Name of the TREXIO file + + mode: str + One of the currently supported ~open~ modes (e.g. 'w', 'r') + + back_end: int + One of the currently supported TREXIO back ends (e.g. TREXIO_HDF5, TREXIO_TEXT) + + Return: + SWIG object of type trexio_s. + + Examples: + >>> from trexio import open as tr_open + >>> trex_file = tr_open("example.h5", "w", TREXIO_HDF5) + """ + try: trexio_file = trexio_open(file_name, mode, back_end) @@ -791,6 +836,7 @@ end interface ~trexio_exit_code~ exit code. *** C + #+begin_src c :tangle prefix_front.h :exports none trexio_exit_code trexio_close(trexio_t* file); #+end_src @@ -862,6 +908,7 @@ trexio_close (trexio_t* file) #+end_src *** Fortran + #+begin_src f90 :tangle prefix_fortran.f90 interface integer function trexio_close (trex_file) bind(C) @@ -872,15 +919,20 @@ end interface #+end_src *** Python + #+begin_src c :tangle basic_python.py def close(trexio_file): + """Close TREXIO file. + + Parameter is a ~trexio_file~ object that has been created by a call to ~open~ function. + """ + try: rc = trexio_close(trexio_file) assert rc==TREXIO_SUCCESS except AssertionError: raise Exception(trexio_string_of_error(rc)) - #+end_src * Templates for front end @@ -1230,6 +1282,21 @@ end interface #+begin_src python :tangle write_num_front.py def write_$group_num$(trexio_file, num_w) -> None: + """Write the $group_num$ variable in the TREXIO file. + + Parameters: + + trexio_file: + TREXIO file handle. + + num_w: int + Value of the $group_num$ variable to be written. + + Raises: + - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + try: rc = trexio_write_$group_num$(trexio_file, num_w) @@ -1243,6 +1310,19 @@ def write_$group_num$(trexio_file, num_w) -> None: #+begin_src python :tangle read_num_front.py def read_$group_num$(trexio_file): + """Read the $group_num$ variable from the TREXIO file. + + Parameter is a ~trexio_file~ object that has been created by a call to ~open~ function. + + Returns: + ~num_r~: int + Integer value of $group_num$ variable read from ~trexio_file~. + + Raises: + - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + try: rc, num_r = trexio_read_$group_num$(trexio_file) @@ -1750,6 +1830,21 @@ end interface #+begin_src python :tangle write_dset_data_front.py def write_$group_dset$(trexio_file, dset_w) -> None: + """Write the $group_dset$ array of numbers in the TREXIO file. + + Parameters: + + trexio_file: + TREXIO file handle. + + dset_w: list OR numpy.ndarray + Array of $group_dset$ values to be written. + + Raises: + - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + try: rc = trexio_write_safe_$group_dset$(trexio_file, dset_w) @@ -1762,7 +1857,26 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+end_src #+begin_src python :tangle read_dset_data_front.py -def read_$group_dset$(trexio_file, dim): +def read_$group_dset$(trexio_file, dim: int): + """Read the $group_dset$ array of numbers from the TREXIO file. + + Parameters: + + trexio_file: + TREXIO file handle. + + dim: int + Size of the block to be read from the file (i.e. how many items of $group_dset$ will be returned) + + Returns: + ~dset_r~: numpy.ndarray + 1D NumPy array with ~dim~ elements corresponding to $group_dset$ values read from the TREXIO file. + + Raises: + - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + try: rc, dset_r = trexio_read_safe_$group_dset$(trexio_file, dim) @@ -2234,7 +2348,22 @@ end interface *** Python templates for front end #+begin_src python :tangle write_dset_str_front.py -def write_$group_dset$(trexio_file, dset_w) -> None: +def write_$group_dset$(trexio_file, dset_w: list) -> None: + """Write the $group_dset$ array of strings in the TREXIO file. + + Parameters: + + trexio_file: + TREXIO file handle. + + dset_w: list + Array of $group_dset$ strings to be written. + + Raises: + - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + max_str_length = len(max(dset_w, key=len)) + 1 @@ -2249,7 +2378,25 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+end_src #+begin_src python :tangle read_dset_str_front.py -def read_$group_dset$(trexio_file, dim): +def read_$group_dset$(trexio_file, dim: int) -> list: + """Read the $group_dset$ array of strings from the TREXIO file. + + Parameters: + + trexio_file: + TREXIO file handle. + + dim: int + Size of the block to be read from the file (i.e. how many items of $group_dset$ will be returned) + + Returns: + ~dset_r~: list + 1D list with ~dim~ elements corresponding to $group_dset$ strings read from the TREXIO file. + + Raises: + - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ try: rc, dset_1d_r = trexio_read_$group_dset$_low(trexio_file, PYTREXIO_MAX_STR_LENGTH) @@ -2449,7 +2596,22 @@ end interface *** Python templates for front end #+begin_src python :tangle write_attr_str_front.py -def write_$group_str$(trexio_file, str_w) -> None: +def write_$group_str$(trexio_file, str_w: str) -> None: + """Write the $group_str$ variable in the TREXIO file. + + Parameters: + + trexio_file: + TREXIO file handle. + + str_w: str + String corresponding to the $group_str$ variable to be written. + + Raises: + - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + max_str_length = len(str_w) + 1 @@ -2460,11 +2622,23 @@ def write_$group_str$(trexio_file, str_w) -> None: raise Exception(trexio_string_of_error(rc)) except: raise - #+end_src #+begin_src python :tangle read_attr_str_front.py -def read_$group_str$(trexio_file): +def read_$group_str$(trexio_file) -> str: + """Read the $group_str$ variable from the TREXIO file. + + Parameter is a ~trexio_file~ object that has been created by a call to ~open~ function. + + Returns: + ~str_r~: str + String corresponding to the $group_str$ variable read from ~trexio_file~. + + Raises: + - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + try: rc, str_r = trexio_read_$group_str$(trexio_file, PYTREXIO_MAX_STR_LENGTH) @@ -2476,6 +2650,7 @@ def read_$group_str$(trexio_file): return str_r #+end_src + * Fortran helper/wrapper functions The function below adapts the original C-based ~trexio_open~ for Fortran. From 6f7c23aa11f501b58ef535021125968695a70fe9 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 21 Aug 2021 13:13:55 +0300 Subject: [PATCH 058/107] add missing annotations --- src/templates_front/templator_front.org | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 2f64514..ab56965 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -451,10 +451,13 @@ end interface #+begin_src python :tangle prefix_python.py :noexport def string_of_error(return_code: int) -> str: - #try: - # from trexio.pytrexio import trexio_string_of_error - #except ImportError: - # raise + """Decode the TREXIO exit code. + + Argument is an integer return code that correspond to one of the TREXIO errors. + + Returns string that contains description of TREXIO ~return_code~. + """ + try: error_str = trexio_string_of_error(trexio_return_code) @@ -1281,7 +1284,7 @@ end interface *** Python templates for front end #+begin_src python :tangle write_num_front.py -def write_$group_num$(trexio_file, num_w) -> None: +def write_$group_num$(trexio_file, num_w: int) -> None: """Write the $group_num$ variable in the TREXIO file. Parameters: @@ -1309,7 +1312,7 @@ def write_$group_num$(trexio_file, num_w) -> None: #+end_src #+begin_src python :tangle read_num_front.py -def read_$group_num$(trexio_file): +def read_$group_num$(trexio_file) -> int: """Read the $group_num$ variable from the TREXIO file. Parameter is a ~trexio_file~ object that has been created by a call to ~open~ function. From f30c6f60a67c2b11ddffeb264f063e7fafaa6680 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 21 Aug 2021 13:14:59 +0300 Subject: [PATCH 059/107] document test_api file --- python/test/test_api.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index 93f2e12..ac37bc0 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -14,6 +14,7 @@ OUTPUT_FILENAME_TEXT = 'test_py_swig.dir' OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' +# define TREXIO file name if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: output_filename = OUTPUT_FILENAME_HDF5 elif TEST_TREXIO_BACKEND == tr.TREXIO_TEXT: @@ -22,6 +23,7 @@ else: raise ValueError ('Specify one of the supported back ends as TEST_TREXIO_BACKEND') +# remove TREXIO file if exists in the current directory try: if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: os.remove(output_filename) @@ -34,12 +36,15 @@ except: #============ WRITE THE DATA IN THE TEST FILE ============# #=========================================================# +# create TREXIO file and open it for writing test_file = tr.open(output_filename, 'w', TEST_TREXIO_BACKEND) -print(test_file) +# Print docstring of the tr.open function +#print(tr.open.__doc__) nucleus_num = 12 +# write nucleus_num in the file tr.write_nucleus_num(test_file, nucleus_num) # initialize charge arrays as a list and convert it to numpy array @@ -61,6 +66,7 @@ tr.write_basis_nucleus_index(test_file, indices_np) point_group = 'B3U' +# write nucleus_point_group in the file tr.write_nucleus_point_group(test_file, point_group) labels = [ @@ -77,20 +83,24 @@ labels = [ 'H', 'H'] +# write nucleus_label in the file tr.write_nucleus_label(test_file,labels) +# close TREXIO file tr.close(test_file) #==========================================================# #============ READ THE DATA FROM THE TEST FILE ============# #==========================================================# +# open previously created TREXIO file, now in 'read' mode test_file2 = tr.open(output_filename, 'r', TEST_TREXIO_BACKEND) +# read nucleus_num from file rnum = tr.read_nucleus_num(test_file2) assert rnum==nucleus_num -# safe call to read_safe array of float values +# safe call to read_nucleus_charge array of float values rcharges_np = tr.read_nucleus_charge(test_file2, dim=nucleus_num) assert rcharges_np.dtype is np.dtype(np.float64) np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) @@ -99,25 +109,28 @@ np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) try: rcharges_fail = tr.read_nucleus_charge(test_file2, dim=nucleus_num*5) except Exception: - print("Unsafe call to safe API: successful") + print("Unsafe call to safe API: checked") -# safe call to read_safe array of int values +# safe call to read array of int values (nuclear indices) rindices_np = tr.read_basis_nucleus_index(test_file2, dim=nucleus_num) assert rindices_np.dtype is np.dtype(np.int32) for i in range(nucleus_num): assert rindices_np[i]==indices_np[i] +# read array of nuclear labels rlabels_2d = tr.read_nucleus_label(test_file2, dim=nucleus_num) print(rlabels_2d) for i in range(nucleus_num): assert rlabels_2d[i]==labels[i] +# read a string corresponding to nuclear point group rpoint_group = tr.read_nucleus_point_group(test_file2) -print(rpoint_group) assert rpoint_group==point_group +# close TREXIO file tr.close(test_file2) +# cleaning (remove the TREXIO file) try: if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: os.remove(output_filename) From c3cc76c4922e4e96afd9e77c0f9e8981798a3bc0 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 24 Aug 2021 11:39:46 +0300 Subject: [PATCH 060/107] fix a typo and add SWIG to requirements for developers --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9980e04..4c5aaf2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![build](https://github.com/TREX-CoE/trexio/actions/workflows/actions.yml/badge.svg)](https://github.com/TREX-CoE/trexio/actions/workflows/actions.yml) ![GitHub release (latest by date)](https://img.shields.io/github/v/release/TREX-CoE/trexio) -TREX library fo efficient I/O. +TREX library for efficient I/O. ## Minimal requirements (for users): @@ -33,7 +33,7 @@ TREX library fo efficient I/O. - python3 (>= 3.6) - Emacs (>= 26.0) - +- SWIG (>= 4.0) ## Installation procedure from the GitHub repo clone (for developers): From 1b05157316ddf31e0564bf3b921d20682ef36f1a Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 24 Aug 2021 11:48:46 +0300 Subject: [PATCH 061/107] get HDF5 compilation flags from pythonic pkgconfig --- python/pyproject.toml | 3 ++- python/setup.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 374b58c..fa03e4c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,7 @@ [build-system] requires = [ "setuptools>=42", - "wheel" + "wheel", + "pkgconfig" ] build-backend = "setuptools.build_meta" diff --git a/python/setup.py b/python/setup.py index 367adbe..a18ff79 100644 --- a/python/setup.py +++ b/python/setup.py @@ -14,10 +14,37 @@ c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] with open("README.md", "r") as fh: long_description = fh.read() -h5_ldflags = str(os.environ.get("H5_LDFLAGS", None )) -h5_cflags_withI = str(os.environ.get("H5_CFLAGS", None )) -h5_cflags = h5_cflags_withI.replace("-I","") +# =========================== Start of the HDF5 block =========================== # +# The block below is needed to derive additional flags related to the HDF5 library, +# which is required to build pytrexio extension module during the setup.py execution +h5_ldflags_withl = os.environ.get("H5_LDFLAGS", None) +h5_cflags_withI = os.environ.get("H5_CFLAGS", None) + +h5_ldflags_isUndefined = h5_ldflags_withl is None or h5_ldflags_withl=="" +h5_cflags_isUndefined = h5_cflags_withI is None or h5_cflags_withI=="" + +if h5_ldflags_isUndefined or h5_cflags_isUndefined: + + try: + import pkgconfig as pk + except ImportError: + raise Exception("pkgconfig Python package has not been found") + + try: + assert pk.exists('hdf5') + except AssertionError: + raise Exception("pkg-config could not locate HDF5") + + h5_cflags_withI = pk.cflags('hdf5') + h5_ldflags_withl = pk.libs('hdf5') + +h5_cflags = h5_cflags_withI.replace("-I","").split(" ")[0] +h5_ldflags = h5_ldflags_withl.split(" ")[0] + +# ============================ End of the HDF5 block ============================ # + +# Define pytrexio extension module based on TREXIO source codes + SWIG-generated wrapper pytrexio_module = Extension('pytrexio._pytrexio', sources = [os.path.join(srcpath, code) for code in c_files], include_dirs = [h5_cflags, srcpath], From c7565e9dece056d79f02dc2d54c12ea394e1ad90 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 24 Aug 2021 11:51:29 +0300 Subject: [PATCH 062/107] minor changes --- .gitignore | 1 + tools/prepare_python.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 1347eb2..8b733a2 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ config.h.in config.h depcomp include/config.h.in +include/config.h.in~ include/stamp-h1 install-sh libtool diff --git a/tools/prepare_python.sh b/tools/prepare_python.sh index b700af4..4808d15 100755 --- a/tools/prepare_python.sh +++ b/tools/prepare_python.sh @@ -23,8 +23,8 @@ mkdir -p ${PYDIR}/src mkdir -p ${PYTREXIODIR} # Copy all the source code and header files in the corresponding python directory -mv ${SRC}/pytrexio.py ${PYTREXIODIR}/pytrexio.py -mv ${SRC}/trexio.py ${PYDIR}/trexio.py +cp ${SRC}/pytrexio.py ${PYTREXIODIR}/pytrexio.py +cp ${SRC}/trexio.py ${PYDIR}/trexio.py cp ${SRC}/*.c ${PYDIR}/src cp ${SRC}/*.h ${PYDIR}/src cp ${INCLUDIR}/trexio.h ${PYDIR}/src From 2911e91941427c00a51ad97e43e30dc67ebe9a4e Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 24 Aug 2021 12:51:43 +0300 Subject: [PATCH 063/107] make dim an optional argument in read_ functions if dim is None - the function read all necessary dimensions from the TREXIO file --- python/test/test_api.py | 25 ++++++++++++++++++- src/templates_front/templator_front.org | 32 +++++++++++++++++++++---- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index ac37bc0..802e48a 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -53,7 +53,7 @@ charges_np = np.array(charges, dtype=np.float64) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived # from the size of the list/array by SWIG using typemaps from numpy.i -rc = tr.write_nucleus_charge(test_file, charges_np) +tr.write_nucleus_charge(test_file, charges_np) # initialize arrays of nuclear indices as a list and convert it to numpy array indices = [i for i in range(nucleus_num)] @@ -64,6 +64,25 @@ indices_np = np.array(indices, dtype=np.int32) # from the size of the list/array by SWIG using typemacs from numpy.i tr.write_basis_nucleus_index(test_file, indices_np) +# initialize a list of nuclear coordinates +coords = [ + 0.00000000 , 1.39250319 , 0.00000000 , + -1.20594314 , 0.69625160 , 0.00000000 , + -1.20594314 , -0.69625160 , 0.00000000 , + 0.00000000 , -1.39250319 , 0.00000000 , + 1.20594314 , -0.69625160 , 0.00000000 , + 1.20594314 , 0.69625160 , 0.00000000 , + -2.14171677 , 1.23652075 , 0.00000000 , + -2.14171677 , -1.23652075 , 0.00000000 , + 0.00000000 , -2.47304151 , 0.00000000 , + 2.14171677 , -1.23652075 , 0.00000000 , + 2.14171677 , 1.23652075 , 0.00000000 , + 0.00000000 , 2.47304151 , 0.00000000 , + ] + +# write coordinates in the file +tr.write_nucleus_coord(test_file, coords) + point_group = 'B3U' # write nucleus_point_group in the file @@ -117,6 +136,10 @@ assert rindices_np.dtype is np.dtype(np.int32) for i in range(nucleus_num): assert rindices_np[i]==indices_np[i] +# read nuclear coordinates without providing optional argument dim +rcoords_np = tr.read_nucleus_coord(test_file2) +assert rcoords_np.size==nucleus_num*3 + # read array of nuclear labels rlabels_2d = tr.read_nucleus_label(test_file2, dim=nucleus_num) print(rlabels_2d) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index ab56965..ccc246e 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -1860,7 +1860,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+end_src #+begin_src python :tangle read_dset_data_front.py -def read_$group_dset$(trexio_file, dim: int): +def read_$group_dset$(trexio_file, dim = None): """Read the $group_dset$ array of numbers from the TREXIO file. Parameters: @@ -1868,8 +1868,9 @@ def read_$group_dset$(trexio_file, dim: int): trexio_file: TREXIO file handle. - dim: int + dim (Optional): int Size of the block to be read from the file (i.e. how many items of $group_dset$ will be returned) + If None, the function will read all necessary array dimensions from the file. Returns: ~dset_r~: numpy.ndarray @@ -1880,6 +1881,16 @@ def read_$group_dset$(trexio_file, dim: int): - Exception from some other error (e.g. RuntimeError). """ + + # if dim is not specified, read dimensions from the TREXIO file + if dim is None: + $group_dset_dim$ = read_$group_dset_dim$(trexio_file) + + dims_list = [$group_dset_dim_list$] + dim = 1 + for i in range($group_dset_rank$): + dim *= dims_list[i] + try: rc, dset_r = trexio_read_safe_$group_dset$(trexio_file, dim) @@ -2381,7 +2392,7 @@ def write_$group_dset$(trexio_file, dset_w: list) -> None: #+end_src #+begin_src python :tangle read_dset_str_front.py -def read_$group_dset$(trexio_file, dim: int) -> list: +def read_$group_dset$(trexio_file, dim = None) -> list: """Read the $group_dset$ array of strings from the TREXIO file. Parameters: @@ -2389,8 +2400,9 @@ def read_$group_dset$(trexio_file, dim: int) -> list: trexio_file: TREXIO file handle. - dim: int + dim (Optional): int Size of the block to be read from the file (i.e. how many items of $group_dset$ will be returned) + If None, the function will read all necessary array dimensions from the file. Returns: ~dset_r~: list @@ -2401,6 +2413,17 @@ def read_$group_dset$(trexio_file, dim: int) -> list: - Exception from some other error (e.g. RuntimeError). """ + + # if dim is not specified, read dimensions from the TREXIO file + if dim is None: + $group_dset_dim$ = read_$group_dset_dim$(trexio_file) + + dims_list = [$group_dset_dim_list$] + dim = 1 + for i in range($group_dset_rank$): + dim *= dims_list[i] + + try: rc, dset_1d_r = trexio_read_$group_dset$_low(trexio_file, PYTREXIO_MAX_STR_LENGTH) assert rc==TREXIO_SUCCESS @@ -2409,6 +2432,7 @@ def read_$group_dset$(trexio_file, dim: int) -> list: except: raise + try: dset_full = dset_1d_r.split(TREXIO_DELIM) dset_2d_r = [dset_full[i] for i in range(dim) if dset_full[i]] From ed7e3902e24bae2a1bf290f06ff7da7d442532fb Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 24 Aug 2021 13:32:01 +0300 Subject: [PATCH 064/107] add memory-safe functions in single or double precision --- src/templates_front/templator_front.org | 100 +++++++++++++++++------- 1 file changed, 70 insertions(+), 30 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index ccc246e..6079501 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -1640,14 +1640,8 @@ trexio_write_$group_dset$_32 (trexio_t* const file, const $group_dset_dtype_sing **** Source code for memory-safe functions - #+begin_src c :tangle read_dset_data_safe_front.c -trexio_exit_code -trexio_read_safe_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const dset_out, const int64_t dim_out) -{ - - if (file == NULL) return TREXIO_INVALID_ARG_1; - if (dset_out == NULL) return TREXIO_INVALID_ARG_2; - + #+NAME:dimCheck + #+begin_src c trexio_exit_code rc; int64_t $group_dset_dim$ = 0; @@ -1664,51 +1658,97 @@ trexio_read_safe_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* for (uint32_t i=0; i> + if (dim_out > (int64_t) dim_size) return TREXIO_UNSAFE_ARRAY_DIM; - /* */ - return trexio_read_$group_dset$_$default_prec$(file, dset_out); + return trexio_read_$group_dset$_32(file, dset_out); } #+end_src - #+begin_src c :tangle write_dset_data_safe_front.c + #+begin_src c :tangle write_dset_data_32_safe_front.c :noweb yes trexio_exit_code -trexio_write_safe_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default$* dset_in, const int64_t dim_in) +trexio_write_safe_$group_dset$_32 (trexio_t* const file, const $group_dset_dtype_single$* dset_in, const int64_t dim_in) { if (file == NULL) return TREXIO_INVALID_ARG_1; if (dset_in == NULL) return TREXIO_INVALID_ARG_2; if (trexio_has_$group_dset$(file) == TREXIO_SUCCESS) return TREXIO_DSET_ALREADY_EXISTS; - trexio_exit_code rc; - int64_t $group_dset_dim$ = 0; - - /* Error handling for this call is added by the generator */ - rc = trexio_read_$group_dset_dim$_64(file, &($group_dset_dim$)); - - if ($group_dset_dim$ == 0L) return TREXIO_INVALID_NUM; - - uint32_t rank = $group_dset_rank$; - uint64_t dims[$group_dset_rank$] = {$group_dset_dim_list$}; - - /* The block below is specific to safe API as it checks the boundaries */ - uint64_t dim_size = 1; - for (uint32_t i=0; i> if (dim_in > (int64_t) dim_size) return TREXIO_UNSAFE_ARRAY_DIM; - /* */ - return trexio_write_$group_dset$_$default_prec$(file, dset_in); + return trexio_write_$group_dset$_32(file, dset_in); +} + #+end_src + #+begin_src c :tangle read_dset_data_64_safe_front.c :noweb yes +trexio_exit_code +trexio_read_safe_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* const dset_out, const int64_t dim_out) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + if (dset_out == NULL) return TREXIO_INVALID_ARG_2; + +<> + + if (dim_out > (int64_t) dim_size) return TREXIO_UNSAFE_ARRAY_DIM; + + return trexio_read_$group_dset$_64(file, dset_out); +} + #+end_src + + + #+begin_src c :tangle write_dset_data_64_safe_front.c :noweb yes +trexio_exit_code +trexio_write_safe_$group_dset$_64 (trexio_t* const file, const $group_dset_dtype_double$* dset_in, const int64_t dim_in) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + if (dset_in == NULL) return TREXIO_INVALID_ARG_2; + if (trexio_has_$group_dset$(file) == TREXIO_SUCCESS) return TREXIO_DSET_ALREADY_EXISTS; + +<> + + if (dim_in > (int64_t) dim_size) return TREXIO_UNSAFE_ARRAY_DIM; + + return trexio_write_$group_dset$_64(file, dset_in); } #+end_src **** Source code for default functions + #+begin_src c :tangle read_dset_data_safe_def_front.c +trexio_exit_code +trexio_read_safe_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const $group_dset$, const int64_t dim_out) +{ + return trexio_read_safe_$group_dset$_$default_prec$(file, $group_dset$, dim_out); +} + #+end_src + + + #+begin_src c :tangle write_dset_data_safe_def_front.c +trexio_exit_code +trexio_write_safe_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default$* $group_dset$, const int64_t dim_in) +{ + return trexio_write_safe_$group_dset$_$default_prec$(file, $group_dset$, dim_in); +} + #+end_src + + #+begin_src c :tangle read_dset_data_def_front.c trexio_exit_code trexio_read_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const $group_dset$) From 1dcd32ef7d8b1a700b47867a8f469f83b2dd70ca Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 25 Aug 2021 13:31:17 +0300 Subject: [PATCH 065/107] add SWIG typemaps for safe API calls in single and double precision --- src/pytrexio.i | 10 +++++++++- src/templates_front/templator_front.org | 11 ++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index df460a7..f7fe29e 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -73,13 +73,21 @@ import_array(); /* Typemaps below change the type of numpy array dimensions from int to int64_t */ %numpy_typemaps(double, NPY_DOUBLE, int64_t) -%numpy_typemaps(int32_t, NPY_INT, int64_t) +%numpy_typemaps(float, NPY_FLOAT, int64_t) +%numpy_typemaps(int32_t, NPY_INT32, int64_t) +%numpy_typemaps(int64_t, NPY_INT64, int64_t) /* Enable write|read_safe functions to convert numpy arrays from/to double arrays */ %apply (double* ARGOUT_ARRAY1, int64_t DIM1) {(double * const dset_out, const int64_t dim_out)}; %apply (double* IN_ARRAY1, int64_t DIM1) {(const double * dset_in, const int64_t dim_in)}; +/* Enable write|read_safe functions to convert numpy arrays from/to float arrays */ +%apply (float* ARGOUT_ARRAY1, int64_t DIM1) {(float * const dset_out, const int64_t dim_out)}; +%apply (float* IN_ARRAY1, int64_t DIM1) {(const float * dset_in, const int64_t dim_in)}; /* Enable write|read_safe functions to convert numpy arrays from/to int32 arrays */ %apply (int32_t* ARGOUT_ARRAY1, int64_t DIM1) {(int32_t * const dset_out, const int64_t dim_out)}; %apply (int32_t* IN_ARRAY1, int64_t DIM1) {(const int32_t * dset_in, const int64_t dim_in)}; +/* Enable write|read_safe functions to convert numpy arrays from/to int64 arrays */ +%apply (int64_t* ARGOUT_ARRAY1, int64_t DIM1) {(int64_t * const dset_out, const int64_t dim_out)}; +%apply (int64_t* IN_ARRAY1, int64_t DIM1) {(const int64_t * dset_in, const int64_t dim_in)}; /* This tells SWIG to treat char ** dset_in pattern as a special case Enables access to trexio_[...]_write_dset_str set of functions directly, i.e. diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 6079501..5b1aa4b 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -787,14 +787,15 @@ def open(file_name: str, mode: str, back_end: int): Examples: >>> from trexio import open as tr_open - >>> trex_file = tr_open("example.h5", "w", TREXIO_HDF5) + >>> trex_file = tr_open("example.h5", "w", TREXIO_HDF5) """ try: trexio_file = trexio_open(file_name, mode, back_end) - except: - raise + assert trexio_file is not None + except AssertionError: + raise Exception(f"Could not open TREXIO file {file_name} using trexio_open function. Please make sure that there are no typos in the file name.") return trexio_file #+end_src @@ -1375,6 +1376,10 @@ trexio_exit_code trexio_read_$group_dset$_64(trexio_t* const file, $group_dset_d trexio_exit_code trexio_write_$group_dset$_64(trexio_t* const file, const $group_dset_dtype_double$* $group_dset$); trexio_exit_code trexio_read_safe_$group_dset$(trexio_t* const file, $group_dset_dtype_default$* const dset_out, const int64_t dim_out); trexio_exit_code trexio_write_safe_$group_dset$(trexio_t* const file, const $group_dset_dtype_default$* dset_in, const int64_t dim_in); +trexio_exit_code trexio_read_safe_$group_dset$_32(trexio_t* const file, $group_dset_dtype_single$* const dset_out, const int64_t dim_out); +trexio_exit_code trexio_write_safe_$group_dset$_32(trexio_t* const file, const $group_dset_dtype_single$* dset_in, const int64_t dim_in); +trexio_exit_code trexio_read_safe_$group_dset$_64(trexio_t* const file, $group_dset_dtype_double$* const dset_out, const int64_t dim_out); +trexio_exit_code trexio_write_safe_$group_dset$_64(trexio_t* const file, const $group_dset_dtype_double$* dset_in, const int64_t dim_in); #+end_src **** Source code for double precision functions From 4c28a4cac8d7c47c2fe0dc7fceb10c2bb220c1a0 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 26 Aug 2021 13:14:46 +0300 Subject: [PATCH 066/107] add type converters for numerical arrays in read/write Python functions --- python/test/test_api.py | 25 ++++++--- src/templates_front/templator_front.org | 68 ++++++++++++++++++++++--- tools/generator_tools.py | 6 ++- 3 files changed, 84 insertions(+), 15 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index 802e48a..c962ff7 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -9,7 +9,7 @@ import trexio as tr #=========================================================# # 0: TREXIO_HDF5 ; 1: TREXIO_TEXT -TEST_TREXIO_BACKEND = tr.TREXIO_TEXT +TEST_TREXIO_BACKEND = tr.TREXIO_HDF5 OUTPUT_FILENAME_TEXT = 'test_py_swig.dir' OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' @@ -49,7 +49,8 @@ tr.write_nucleus_num(test_file, nucleus_num) # initialize charge arrays as a list and convert it to numpy array charges = [6., 6., 6., 6., 6., 6., 1., 1., 1., 1., 1., 1.] -charges_np = np.array(charges, dtype=np.float64) +#charges_np = np.array(charges, dtype=np.float32) +charges_np = np.array(charges, dtype=np.int32) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived # from the size of the list/array by SWIG using typemaps from numpy.i @@ -58,7 +59,7 @@ tr.write_nucleus_charge(test_file, charges_np) # initialize arrays of nuclear indices as a list and convert it to numpy array indices = [i for i in range(nucleus_num)] # type cast is important here because by default numpy transforms a list of integers into int64 array -indices_np = np.array(indices, dtype=np.int32) +indices_np = np.array(indices, dtype=np.int64) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived # from the size of the list/array by SWIG using typemacs from numpy.i @@ -108,6 +109,7 @@ tr.write_nucleus_label(test_file,labels) # close TREXIO file tr.close(test_file) + #==========================================================# #============ READ THE DATA FROM THE TEST FILE ============# #==========================================================# @@ -131,10 +133,21 @@ except Exception: print("Unsafe call to safe API: checked") # safe call to read array of int values (nuclear indices) -rindices_np = tr.read_basis_nucleus_index(test_file2, dim=nucleus_num) -assert rindices_np.dtype is np.dtype(np.int32) +rindices_np_16 = tr.read_basis_nucleus_index(test_file2, dim=nucleus_num, dtype=np.int16) +assert rindices_np_16.dtype is np.dtype(np.int16) for i in range(nucleus_num): - assert rindices_np[i]==indices_np[i] + assert rindices_np_16[i]==indices_np[i] + +rindices_np_32 = tr.read_basis_nucleus_index(test_file2, dim=nucleus_num, dtype=np.int32) +assert rindices_np_32.dtype is np.dtype(np.int32) +for i in range(nucleus_num): + assert rindices_np_32[i]==indices_np[i] + +rindices_np_64 = tr.read_basis_nucleus_index(test_file2) +assert rindices_np_64.dtype is np.dtype(np.int64) +assert rindices_np_64.size==nucleus_num +for i in range(nucleus_num): + assert rindices_np_64[i]==indices_np[i] # read nuclear coordinates without providing optional argument dim rcoords_np = tr.read_nucleus_coord(test_file2) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 5b1aa4b..fbbb6ba 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -987,6 +987,7 @@ def close(trexio_file): | ~$group_dset_f_dtype_single$~ | Single precision datatype of the dataset [Fortran] | ~real(4)/integer(4)~ | | ~$group_dset_f_dtype_double$~ | Double precision datatype of the dataset [Fortran] | ~real(8)/integer(8)~ | | ~$group_dset_f_dims$~ | Dimensions in Fortran | ~(:,:)~ | + | ~$group_dset_py_dtype$~ | Standard datatype of the dataset [Python] | ~float/int~ | Note: parent group name is always added to the child objects upon @@ -1886,7 +1887,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: TREXIO file handle. dset_w: list OR numpy.ndarray - Array of $group_dset$ values to be written. + Array of $group_dset$ values to be written. If array data type does not correspond to int64 or float64, the conversion is performed. Raises: - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. @@ -1894,8 +1895,31 @@ def write_$group_dset$(trexio_file, dset_w) -> None: """ + cutPrecision = False + if not isinstance(dset_w, list): + try: + import numpy as np + except ImportError: + raise Exception("NumPy cannot be imported.") + + if isinstance(dset_w, np.ndarray) and (not dset_w.dtype==np.int64 or not dset_w.dtype==np.float64): + cutPrecision = True + + + if cutPrecision: + try: + # TODO: we have to make sure that this conversion does not introduce any noise in the data. + dset_64 = np.$group_dset_py_dtype$64(dset_w) + except: + raise + + try: - rc = trexio_write_safe_$group_dset$(trexio_file, dset_w) + if cutPrecision: + rc = trexio_write_safe_$group_dset$_64(trexio_file, dset_64) + else: + rc = trexio_write_safe_$group_dset$_64(trexio_file, dset_w) + assert rc==TREXIO_SUCCESS except AssertionError: raise Exception(trexio_string_of_error(rc)) @@ -1905,7 +1929,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+end_src #+begin_src python :tangle read_dset_data_front.py -def read_$group_dset$(trexio_file, dim = None): +def read_$group_dset$(trexio_file, dim = None, dtype = None): """Read the $group_dset$ array of numbers from the TREXIO file. Parameters: @@ -1917,8 +1941,11 @@ def read_$group_dset$(trexio_file, dim = None): Size of the block to be read from the file (i.e. how many items of $group_dset$ will be returned) If None, the function will read all necessary array dimensions from the file. + dtype (Optional): type + NumPy data type of the output (e.g. np.int32|int16 or np.float32|float16). If specified, the output array will be converted from the default double precision. + Returns: - ~dset_r~: numpy.ndarray + ~dset_64~ if dtype is None or ~dset_converted~ otherwise: numpy.ndarray 1D NumPy array with ~dim~ elements corresponding to $group_dset$ values read from the TREXIO file. Raises: @@ -1938,16 +1965,41 @@ def read_$group_dset$(trexio_file, dim = None): try: - rc, dset_r = trexio_read_safe_$group_dset$(trexio_file, dim) + rc, dset_64 = trexio_read_safe_$group_dset$_64(trexio_file, dim) assert rc==TREXIO_SUCCESS except AssertionError: raise Exception(trexio_string_of_error(rc)) except: - raise + raise + + + isConverted = False + dset_converted = None + if dtype is not None: + try: + import numpy as np + except ImportError: + raise Exception("NumPy cannot be imported.") + + try: + assert isinstance(dtype, type) + except AssertionError: + raise TypeError("dtype argument has to be an instance of the type class (e.g. np.float32).") + + + if not dtype==np.int64 or not dtype==np.float64: + try: + dset_converted = np.array(dset_64, dtype=dtype) + except: + raise + + isConverted = True # additional assert can be added here to check that read_safe functions returns numpy array of proper dimension - - return dset_r + if isConverted: + return dset_converted + else: + return dset_64 #+end_src ** Sparse data structures diff --git a/tools/generator_tools.py b/tools/generator_tools.py index 7f96175..18fb472 100644 --- a/tools/generator_tools.py +++ b/tools/generator_tools.py @@ -100,7 +100,7 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N fname_new = join('populated',f'pop_{fname}') templ_path = get_template_path(fname, paths) - triggers = ['group_dset_dtype', 'group_dset_h5_dtype', 'default_prec', 'is_index', + triggers = ['group_dset_dtype', 'group_dset_py_dtype', 'group_dset_h5_dtype', 'default_prec', 'is_index', 'group_dset_f_dtype_default', 'group_dset_f_dtype_double', 'group_dset_f_dtype_single', 'group_dset_dtype_default', 'group_dset_dtype_double', 'group_dset_dtype_single', 'group_dset_rank', 'group_dset_dim_list', 'group_dset_f_dims', @@ -542,6 +542,7 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: default_prec = '64' group_dset_std_dtype_out = '24.16e' group_dset_std_dtype_in = 'lf' + group_dset_py_dtype = 'float' elif v[0] in ['int', 'index']: datatype = 'int64_t' group_dset_h5_dtype = 'native_int64' @@ -554,6 +555,7 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: default_prec = '32' group_dset_std_dtype_out = '" PRId64 "' group_dset_std_dtype_in = '" SCNd64 "' + group_dset_py_dtype = 'int' elif v[0] == 'str': datatype = 'char*' group_dset_h5_dtype = '' @@ -566,6 +568,7 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: default_prec = '' group_dset_std_dtype_out = 's' group_dset_std_dtype_in = 's' + group_dset_py_dtype = 'str' # add the dset name for templates tmp_dict['group_dset'] = k @@ -587,6 +590,7 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: tmp_dict['default_prec'] = default_prec tmp_dict['group_dset_std_dtype_in'] = group_dset_std_dtype_in tmp_dict['group_dset_std_dtype_out'] = group_dset_std_dtype_out + tmp_dict['group_dset_py_dtype'] = group_dset_py_dtype # add the rank tmp_dict['rank'] = len(v[1]) tmp_dict['group_dset_rank'] = str(tmp_dict['rank']) From 7b5ebf6272a5f6dc5451f46d4f8059458d79f764 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 26 Aug 2021 17:01:53 +0300 Subject: [PATCH 067/107] implement TREXIO File class and change import of pytrexio module --- python/test/test_api.py | 17 ++- src/templates_front/templator_front.org | 147 +++++++++++++++++------- 2 files changed, 114 insertions(+), 50 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index c962ff7..ae421c5 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -9,7 +9,7 @@ import trexio as tr #=========================================================# # 0: TREXIO_HDF5 ; 1: TREXIO_TEXT -TEST_TREXIO_BACKEND = tr.TREXIO_HDF5 +TEST_TREXIO_BACKEND = tr.TREXIO_TEXT OUTPUT_FILENAME_TEXT = 'test_py_swig.dir' OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' @@ -37,7 +37,8 @@ except: #=========================================================# # create TREXIO file and open it for writing -test_file = tr.open(output_filename, 'w', TEST_TREXIO_BACKEND) +#test_file = tr.open(output_filename, 'w', TEST_TREXIO_BACKEND) +test_file = tr.File(output_filename, mode='w', back_end=TEST_TREXIO_BACKEND) # Print docstring of the tr.open function #print(tr.open.__doc__) @@ -107,15 +108,19 @@ labels = [ tr.write_nucleus_label(test_file,labels) # close TREXIO file -tr.close(test_file) - +# [TODO:] this functional call is no longer needed as we introduced TREXIO_File class which has a desctructor that closes the file +#tr.close(test_file) +# [TODO:] without calling destructor on test_file the TREXIO_FILE is not getting created and the data is not written when using TEXT back end. This, the user still has to explicitly call destructor on test_file object instead +# tr.close function. This is only an issue when the data is getting written and read in the same session (e.g. in Jupyter notebook) +del test_file #==========================================================# #============ READ THE DATA FROM THE TEST FILE ============# #==========================================================# # open previously created TREXIO file, now in 'read' mode -test_file2 = tr.open(output_filename, 'r', TEST_TREXIO_BACKEND) +#test_file2 = tr.open(output_filename, 'r', TEST_TREXIO_BACKEND) +test_file2 = tr.File(output_filename, 'r', TEST_TREXIO_BACKEND) # read nucleus_num from file rnum = tr.read_nucleus_num(test_file2) @@ -164,7 +169,7 @@ rpoint_group = tr.read_nucleus_point_group(test_file2) assert rpoint_group==point_group # close TREXIO file -tr.close(test_file2) +#tr.close(test_file2) # cleaning (remove the TREXIO file) try: diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index fbbb6ba..d308d3c 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -92,7 +92,7 @@ Moreover, exception handling becomes trivial thanks to the use of TREXIO exit co try: - from pytrexio.pytrexio import * + import pytrexio.pytrexio as pytr except ImportError: raise Exception("Could not import pytrexio module from trexio package") @@ -558,6 +558,76 @@ struct trexio_s { char padding[6]; /* Ensures the proper alignment of back ends */ }; #+end_src + +*** TREXIO_File Python class + + #+begin_src python :tangle basic_python.py +class File: + """TREXIO File object. + + General information about the TREXIO file. + + Parameters: + + filename: str + Is a name of the full path to the TREXIO file. + back_end: int + One of the currently supported TREXIO back ends. + For example, TREXIO_HDF5 (0) or TREXIO_TEXT (1). + mode: str + One of the currently supported TREXIO open modes. + For example, 'r' or 'w'. + isOpen: bool + Flag indicating whether the current object is still open for I/O + pytrexio_s: + A PyObject corresponding to SWIG proxy of the trexio_s struct in C. + This argument is in fact a TREXIO file handle, which is required for + communicating with the C back end. + info: dict + Dictionary of key-value pairs with additional information about the file. + """ + + + def __init__(self, filename, mode='r', back_end=TREXIO_HDF5, + pytrexio_s=None, info=None): + """This is a TREXIO File constructor.""" + + self.filename = filename + self.mode = mode + self.back_end = back_end + + self.isOpen = False + if pytrexio_s is None: + self.pytrexio_s = open(filename, mode, back_end) + self.isOpen = True + else: + self.pytrexio_s = pytrexio_s + self.isOpen = None + + self.info = info + + + def close(self): + """Close a TREXIO File.""" + + if self.isOpen: + close(self.pytrexio_s) + self.isOpen = False + else: + raise Exception("TREXIO File object has not been opened.") + + + def __del__(self): + """This is a TREXIO File destructor.""" + + if self.isOpen: + close(self.pytrexio_s) + elif self.isOpen is None: + raise Exception("[WIP]: TREXIO file handle provided but what if the file is already closed?") + else: + pass + #+end_src + ** Polymorphism of the file handle Polymorphism of the ~trexio_t~ type is handled by ensuring that the @@ -767,7 +837,7 @@ end interface *** Python - #+begin_src c :tangle basic_python.py + #+begin_src python :tangle basic_python.py def open(file_name: str, mode: str, back_end: int): """Create TREXIO file or open existing one. @@ -790,9 +860,8 @@ def open(file_name: str, mode: str, back_end: int): >>> trex_file = tr_open("example.h5", "w", TREXIO_HDF5) """ - try: - trexio_file = trexio_open(file_name, mode, back_end) + trexio_file = pytr.trexio_open(file_name, mode, back_end) assert trexio_file is not None except AssertionError: raise Exception(f"Could not open TREXIO file {file_name} using trexio_open function. Please make sure that there are no typos in the file name.") @@ -924,19 +993,18 @@ end interface *** Python - #+begin_src c :tangle basic_python.py + #+begin_src python :tangle basic_python.py def close(trexio_file): """Close TREXIO file. Parameter is a ~trexio_file~ object that has been created by a call to ~open~ function. """ - try: - rc = trexio_close(trexio_file) + rc = pytr.trexio_close(trexio_file) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) #+end_src * Templates for front end @@ -1292,7 +1360,7 @@ def write_$group_num$(trexio_file, num_w: int) -> None: Parameters: trexio_file: - TREXIO file handle. + TREXIO File object. num_w: int Value of the $group_num$ variable to be written. @@ -1302,12 +1370,11 @@ def write_$group_num$(trexio_file, num_w: int) -> None: - Exception from some other error (e.g. RuntimeError). """ - try: - rc = trexio_write_$group_num$(trexio_file, num_w) + rc = pytr.trexio_write_$group_num$(trexio_file.pytrexio_s, num_w) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) except: raise @@ -1317,7 +1384,7 @@ def write_$group_num$(trexio_file, num_w: int) -> None: def read_$group_num$(trexio_file) -> int: """Read the $group_num$ variable from the TREXIO file. - Parameter is a ~trexio_file~ object that has been created by a call to ~open~ function. + Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. Returns: ~num_r~: int @@ -1328,12 +1395,11 @@ def read_$group_num$(trexio_file) -> int: - Exception from some other error (e.g. RuntimeError). """ - try: - rc, num_r = trexio_read_$group_num$(trexio_file) + rc, num_r = pytr.trexio_read_$group_num$(trexio_file.pytrexio_s) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) except: raise @@ -1884,7 +1950,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: Parameters: trexio_file: - TREXIO file handle. + TREXIO File object. dset_w: list OR numpy.ndarray Array of $group_dset$ values to be written. If array data type does not correspond to int64 or float64, the conversion is performed. @@ -1894,9 +1960,8 @@ def write_$group_dset$(trexio_file, dset_w) -> None: - Exception from some other error (e.g. RuntimeError). """ - cutPrecision = False - if not isinstance(dset_w, list): + if not isinstance(dset_w, (list, tuple)): try: import numpy as np except ImportError: @@ -1908,7 +1973,6 @@ def write_$group_dset$(trexio_file, dset_w) -> None: if cutPrecision: try: - # TODO: we have to make sure that this conversion does not introduce any noise in the data. dset_64 = np.$group_dset_py_dtype$64(dset_w) except: raise @@ -1916,13 +1980,13 @@ def write_$group_dset$(trexio_file, dset_w) -> None: try: if cutPrecision: - rc = trexio_write_safe_$group_dset$_64(trexio_file, dset_64) + rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_64) else: - rc = trexio_write_safe_$group_dset$_64(trexio_file, dset_w) + rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_w) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) except: raise @@ -1935,7 +1999,7 @@ def read_$group_dset$(trexio_file, dim = None, dtype = None): Parameters: trexio_file: - TREXIO file handle. + TREXIO File object. dim (Optional): int Size of the block to be read from the file (i.e. how many items of $group_dset$ will be returned) @@ -1952,7 +2016,6 @@ def read_$group_dset$(trexio_file, dim = None, dtype = None): - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. - Exception from some other error (e.g. RuntimeError). """ - # if dim is not specified, read dimensions from the TREXIO file if dim is None: @@ -1965,10 +2028,10 @@ def read_$group_dset$(trexio_file, dim = None, dtype = None): try: - rc, dset_64 = trexio_read_safe_$group_dset$_64(trexio_file, dim) + rc, dset_64 = pytr.trexio_read_safe_$group_dset$_64(trexio_file.pytrexio_s, dim) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) except: raise @@ -2465,7 +2528,7 @@ def write_$group_dset$(trexio_file, dset_w: list) -> None: Parameters: trexio_file: - TREXIO file handle. + TREXIO File object. dset_w: list Array of $group_dset$ strings to be written. @@ -2475,14 +2538,13 @@ def write_$group_dset$(trexio_file, dset_w: list) -> None: - Exception from some other error (e.g. RuntimeError). """ - max_str_length = len(max(dset_w, key=len)) + 1 try: - rc = trexio_write_$group_dset$(trexio_file, dset_w, max_str_length) + rc = pytr.trexio_write_$group_dset$(trexio_file.pytrexio_s, dset_w, max_str_length) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) except: raise @@ -2495,7 +2557,7 @@ def read_$group_dset$(trexio_file, dim = None) -> list: Parameters: trexio_file: - TREXIO file handle. + TREXIO File object. dim (Optional): int Size of the block to be read from the file (i.e. how many items of $group_dset$ will be returned) @@ -2510,7 +2572,6 @@ def read_$group_dset$(trexio_file, dim = None) -> list: - Exception from some other error (e.g. RuntimeError). """ - # if dim is not specified, read dimensions from the TREXIO file if dim is None: $group_dset_dim$ = read_$group_dset_dim$(trexio_file) @@ -2522,16 +2583,16 @@ def read_$group_dset$(trexio_file, dim = None) -> list: try: - rc, dset_1d_r = trexio_read_$group_dset$_low(trexio_file, PYTREXIO_MAX_STR_LENGTH) + rc, dset_1d_r = pytr.trexio_read_$group_dset$_low(trexio_file.pytrexio_s, PYTREXIO_MAX_STR_LENGTH) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) except: raise try: - dset_full = dset_1d_r.split(TREXIO_DELIM) + dset_full = dset_1d_r.split(pytr.TREXIO_DELIM) dset_2d_r = [dset_full[i] for i in range(dim) if dset_full[i]] assert dset_2d_r except AssertionError: @@ -2726,7 +2787,7 @@ def write_$group_str$(trexio_file, str_w: str) -> None: Parameters: trexio_file: - TREXIO file handle. + TREXIO File object. str_w: str String corresponding to the $group_str$ variable to be written. @@ -2736,14 +2797,13 @@ def write_$group_str$(trexio_file, str_w: str) -> None: - Exception from some other error (e.g. RuntimeError). """ - max_str_length = len(str_w) + 1 try: - rc = trexio_write_$group_str$(trexio_file, str_w, max_str_length) + rc = pytr.trexio_write_$group_str$(trexio_file.pytrexio_s, str_w, max_str_length) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) except: raise #+end_src @@ -2752,7 +2812,7 @@ def write_$group_str$(trexio_file, str_w: str) -> None: def read_$group_str$(trexio_file) -> str: """Read the $group_str$ variable from the TREXIO file. - Parameter is a ~trexio_file~ object that has been created by a call to ~open~ function. + Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. Returns: ~str_r~: str @@ -2763,12 +2823,11 @@ def read_$group_str$(trexio_file) -> str: - Exception from some other error (e.g. RuntimeError). """ - try: - rc, str_r = trexio_read_$group_str$(trexio_file, PYTREXIO_MAX_STR_LENGTH) + rc, str_r = pytr.trexio_read_$group_str$(trexio_file.pytrexio_s, PYTREXIO_MAX_STR_LENGTH) assert rc==TREXIO_SUCCESS except AssertionError: - raise Exception(trexio_string_of_error(rc)) + raise Exception(pytr.trexio_string_of_error(rc)) except: raise From 5a2b4d96a7767fde03d2bfa275cc91b48877f5b1 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 27 Aug 2021 16:08:39 +0300 Subject: [PATCH 068/107] reshape output arrays by default when reading from the file --- python/test/test_api.py | 11 +++++- src/templates_front/templator_front.org | 46 +++++++++++++++++++------ 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index ae421c5..a97b87e 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -9,7 +9,7 @@ import trexio as tr #=========================================================# # 0: TREXIO_HDF5 ; 1: TREXIO_TEXT -TEST_TREXIO_BACKEND = tr.TREXIO_TEXT +TEST_TREXIO_BACKEND = 0 OUTPUT_FILENAME_TEXT = 'test_py_swig.dir' OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' @@ -36,6 +36,8 @@ except: #============ WRITE THE DATA IN THE TEST FILE ============# #=========================================================# + + # create TREXIO file and open it for writing #test_file = tr.open(output_filename, 'w', TEST_TREXIO_BACKEND) test_file = tr.File(output_filename, mode='w', back_end=TEST_TREXIO_BACKEND) @@ -114,6 +116,8 @@ tr.write_nucleus_label(test_file,labels) # tr.close function. This is only an issue when the data is getting written and read in the same session (e.g. in Jupyter notebook) del test_file + + #==========================================================# #============ READ THE DATA FROM THE TEST FILE ============# #==========================================================# @@ -156,7 +160,12 @@ for i in range(nucleus_num): # read nuclear coordinates without providing optional argument dim rcoords_np = tr.read_nucleus_coord(test_file2) + assert rcoords_np.size==nucleus_num*3 +np.testing.assert_array_almost_equal(rcoords_np, np.array(coords).reshape(nucleus_num,3), decimal=8) + +# set doReshape to False to get a flat 1D array (e.g. when reading matrices like nuclear coordinates) +#rcoords_reshaped_2 = tr.read_nucleus_coord(test_file2, doReshape=False) # read array of nuclear labels rlabels_2d = tr.read_nucleus_label(test_file2, dim=nucleus_num) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index d308d3c..028ed12 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -1993,7 +1993,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: #+end_src #+begin_src python :tangle read_dset_data_front.py -def read_$group_dset$(trexio_file, dim = None, dtype = None): +def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): """Read the $group_dset$ array of numbers from the TREXIO file. Parameters: @@ -2007,6 +2007,10 @@ def read_$group_dset$(trexio_file, dim = None, dtype = None): dtype (Optional): type NumPy data type of the output (e.g. np.int32|int16 or np.float32|float16). If specified, the output array will be converted from the default double precision. + + doReshape (Optional): bool + Flag to determine whether the output NumPy array has be reshaped or not. Be default, reshaping is performed + based on the dimensions from the ~trex.json~ file. Otherwise, ~shape~ array (list or tuple) is used if provided by the user. Returns: ~dset_64~ if dtype is None or ~dset_converted~ otherwise: numpy.ndarray @@ -2015,17 +2019,30 @@ def read_$group_dset$(trexio_file, dim = None, dtype = None): Raises: - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. - Exception from some other error (e.g. RuntimeError). - """ - +""" + + try: + import numpy as np + except ImportError: + raise Exception("NumPy cannot be imported.") + + if doReshape is None: + doReshape = True + # if dim is not specified, read dimensions from the TREXIO file - if dim is None: + dims_list = None + if dim is None or doReshape: $group_dset_dim$ = read_$group_dset_dim$(trexio_file) dims_list = [$group_dset_dim_list$] dim = 1 for i in range($group_dset_rank$): dim *= dims_list[i] - + + + shape = tuple(dims_list) + if shape is None and doReshape: + raise ValueError("Reshaping failure: shape is None.") try: rc, dset_64 = pytr.trexio_read_safe_$group_dset$_64(trexio_file.pytrexio_s, dim) @@ -2039,10 +2056,6 @@ def read_$group_dset$(trexio_file, dim = None, dtype = None): isConverted = False dset_converted = None if dtype is not None: - try: - import numpy as np - except ImportError: - raise Exception("NumPy cannot be imported.") try: assert isinstance(dtype, type) @@ -2059,8 +2072,21 @@ def read_$group_dset$(trexio_file, dim = None, dtype = None): isConverted = True # additional assert can be added here to check that read_safe functions returns numpy array of proper dimension - if isConverted: + + if doReshape: + try: + # in-place reshaping did not work so I have to make a copy + if isConverted: + dset_reshaped = np.reshape(dset_converted, shape, order='C') + else: + dset_reshaped = np.reshape(dset_64, shape, order='C') + except: + raise + + if isConverted: return dset_converted + elif doReshape: + return dset_reshaped else: return dset_64 #+end_src From b8aff45d3b4c5a91dfccb8bfc5f2b831dab7d348 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 27 Aug 2021 16:14:33 +0300 Subject: [PATCH 069/107] add notes on Python build and installation issues --- python/README_SETUP.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 python/README_SETUP.txt diff --git a/python/README_SETUP.txt b/python/README_SETUP.txt new file mode 100644 index 0000000..b989b27 --- /dev/null +++ b/python/README_SETUP.txt @@ -0,0 +1,24 @@ +# Additional information related to the installation of the TREXIO Python API + +Removing MANIFEST.in leads to issues in the installation. In particular, the .c and .h source files do not get copied +from the src/ directory into the tar.gz which is produced by setup sdist command. +These source files are required to build the pytrexio.so extension module, which is needed for the Python API. + +Then the following commands do the job of installing trexio in virtual environment: + +$ python3 -m pip install --upgrade setuptools wheel twine +$ python3 -s setup.py --no-user-cfg build +$ python3 setup.py sdist bdist_wheel +$ python3 -m pip install dist/trexio-0.1-*.whl --force-reinstall + +BUG: when trying to install trexio from tar.gz, trexio appears to be installed +after user runs [python3 -s setup.py --no-user-cfg build] +but before [python3 -m pip install dist/{WHEELNAME}.whl] +which leads to pip install saying that trexio is already installed but the test fails saying that _trexio was not found, +indicating that probably first installation did not go well. +FIX: Use --force-reinstall option with pip install + +The installed in virtual environment trexio can be uninstalled using [pip uninstall trexio] +but only from the root directory where the initially used (to install) setup.py script is located. +For example, it is most probably the python/ directory of the TREXIO_ROOT folder. + From fdc9a975b97400e1257fb71ee65c040032a33703 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 30 Aug 2021 12:52:36 +0300 Subject: [PATCH 070/107] add empty line between function headers --- tools/generator_tools.py | 42 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/tools/generator_tools.py b/tools/generator_tools.py index 18fb472..18c5395 100644 --- a/tools/generator_tools.py +++ b/tools/generator_tools.py @@ -111,25 +111,27 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N with open(join(templ_path,fname_new), 'a') as f_out : num_written = [] for line in f_in : - # special case to add error handling for read/write of dimensioning variables - if '$group_dset_dim$' in line: - rc_line = 'if (rc != TREXIO_SUCCESS) return rc;\n' - indentlevel = len(line) - len(line.lstrip()) - for dim in detailed_source[item]['dims']: - if not dim.isdigit() and not dim in num_written: - num_written.append(dim) - templine = line.replace('$group_dset_dim$', dim) - if '_read' in templine and (not 'fortran' in fname): - line_toadd = indentlevel*" " + rc_line - templine += line_toadd + # special case to add error handling for read/write of dimensioning variables + if '$group_dset_dim$' in line: + rc_line = 'if (rc != TREXIO_SUCCESS) return rc;\n' + indentlevel = len(line) - len(line.lstrip()) + for dim in detailed_source[item]['dims']: + if not dim.isdigit() and not dim in num_written: + num_written.append(dim) + templine = line.replace('$group_dset_dim$', dim) + if '_read' in templine and (not 'fortran' in fname): + line_toadd = indentlevel*" " + rc_line + templine += line_toadd - f_out.write(templine) - num_written = [] - continue - # general case of recursive replacement of inline triggers - else: - populated_line = recursive_replace_line(line, triggers, detailed_source[item]) - f_out.write(populated_line) + f_out.write(templine) + num_written = [] + continue + # general case of recursive replacement of inline triggers + else: + populated_line = recursive_replace_line(line, triggers, detailed_source[item]) + f_out.write(populated_line) + + f_out.write("\n") def recursive_replace_line (input_line: str, triggers: list, source: dict) -> str: @@ -214,6 +216,8 @@ def iterative_populate_file (filename: str, paths: dict, groups: dict, datasets: f_out.write(populated_line) else: f_out.write(line) + + f_out.write("\n") def iterative_replace_line (input_line: str, case: str, source: dict, add_line: str) -> str: @@ -403,6 +407,8 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai else: loop_body += line + f_out.write("\n") + def get_template_path (filename: str, path_dict: dict) -> str: """ From cf51d3971f054327b40a6bea9809d30619bd7614 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 30 Aug 2021 13:46:11 +0300 Subject: [PATCH 071/107] read module version from _version.py file --- python/pytrexio/__init__.py | 1 + python/pytrexio/_version.py | 1 + python/setup.py | 13 ++++++++++++- 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 python/pytrexio/_version.py diff --git a/python/pytrexio/__init__.py b/python/pytrexio/__init__.py index e69de29..8dee4bf 100644 --- a/python/pytrexio/__init__.py +++ b/python/pytrexio/__init__.py @@ -0,0 +1 @@ +from ._version import __version__ diff --git a/python/pytrexio/_version.py b/python/pytrexio/_version.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/python/pytrexio/_version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/python/setup.py b/python/setup.py index a18ff79..53dccf6 100644 --- a/python/setup.py +++ b/python/setup.py @@ -14,6 +14,17 @@ c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] with open("README.md", "r") as fh: long_description = fh.read() +VERSIONFILE = "pytrexio/_version.py" +try: + exec(open(VERSIONFILE).read()) +except: + raise IOError(f"Could not open the version file {VERSIONFILE}.") + +version_r = __version__ +if not version_r: + raise RuntimeError(f"Unable to find a version string in {VERSIONFILE}.") + + # =========================== Start of the HDF5 block =========================== # # The block below is needed to derive additional flags related to the HDF5 library, # which is required to build pytrexio extension module during the setup.py execution @@ -55,7 +66,7 @@ pytrexio_module = Extension('pytrexio._pytrexio', setup(name = 'trexio', - version = '0.1', + version = version_r, author = "TREX-CoE", author_email = "posenitskiy@irsamc.ups-tlse.fr", description = """Python API of the TREXIO library""", From ad44e29e95145e97ff03f9b535e223a7ab7c1a7c Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 30 Aug 2021 15:08:26 +0300 Subject: [PATCH 072/107] fix bug with Python version import --- python/install_pytrexio.sh | 2 +- python/pytrexio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index fd3dd09..eb9fb2e 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -27,7 +27,7 @@ python3 setup.py sdist bdist_wheel # Install pytrexio in the current environment from the aforementioned wheel # --force-reinstall is needed here because build-system pre-installs pytrexio in the environment # but does not install things in the corresponding site-packages directory -python3 -m pip install dist/trexio-0.1-*.whl --force-reinstall +python3 -m pip install dist/trexio-*.whl --force-reinstall # Run the command below in the root directory to install the package in 'editable' (-e) mode without dependencies (--no-deps) #python -m pip install -e . --no-deps diff --git a/python/pytrexio/__init__.py b/python/pytrexio/__init__.py index 8dee4bf..415bcc4 100644 --- a/python/pytrexio/__init__.py +++ b/python/pytrexio/__init__.py @@ -1 +1 @@ -from ._version import __version__ +#from ._version import __version__ From 74c69bb2935e7e466d524952236a911165aa7d20 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 30 Aug 2021 18:04:35 +0300 Subject: [PATCH 073/107] better Exception handling with custom exception class --- src/templates_front/templator_front.org | 77 +++++++++++++++---------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 028ed12..645c1eb 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -450,6 +450,21 @@ end interface **** Python interface #+begin_src python :tangle prefix_python.py :noexport +class Error(Exception): + """Base class for TREXIO errors. + + Attributes: + error: int -- exit code returned by the call to TREXIO library; + message: str -- decoded string corresponding to trexio_exit_code. + + """ + + def __init__(self, trexio_return_code): + self.error = trexio_return_code + self.message = string_of_error(trexio_return_code) + super().__init__(self.message) + + def string_of_error(return_code: int) -> str: """Decode the TREXIO exit code. @@ -458,9 +473,8 @@ def string_of_error(return_code: int) -> str: Returns string that contains description of TREXIO ~return_code~. """ - try: - error_str = trexio_string_of_error(trexio_return_code) + error_str = pytr.trexio_string_of_error(return_code) except: raise @@ -1002,9 +1016,10 @@ def close(trexio_file): try: rc = pytr.trexio_close(trexio_file) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + if rc != TREXIO_SUCCESS: + raise Error(rc) + except: + raise #+end_src * Templates for front end @@ -1372,9 +1387,8 @@ def write_$group_num$(trexio_file, num_w: int) -> None: try: rc = pytr.trexio_write_$group_num$(trexio_file.pytrexio_s, num_w) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + if rc != TREXIO_SUCCESS: + raise Error(rc) except: raise @@ -1397,9 +1411,8 @@ def read_$group_num$(trexio_file) -> int: try: rc, num_r = pytr.trexio_read_$group_num$(trexio_file.pytrexio_s) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + if rc != TREXIO_SUCCESS: + raise Error(rc) except: raise @@ -1984,9 +1997,8 @@ def write_$group_dset$(trexio_file, dset_w) -> None: else: rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_w) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + if rc != TREXIO_SUCCESS: + raise Error(rc) except: raise @@ -2046,9 +2058,9 @@ def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): try: rc, dset_64 = pytr.trexio_read_safe_$group_dset$_64(trexio_file.pytrexio_s, dim) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + + if rc != TREXIO_SUCCESS: + raise Error(rc) except: raise @@ -2568,9 +2580,9 @@ def write_$group_dset$(trexio_file, dset_w: list) -> None: try: rc = pytr.trexio_write_$group_dset$(trexio_file.pytrexio_s, dset_w, max_str_length) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + + if rc != TREXIO_SUCCESS: + raise Error(rc) except: raise @@ -2610,9 +2622,9 @@ def read_$group_dset$(trexio_file, dim = None) -> list: try: rc, dset_1d_r = pytr.trexio_read_$group_dset$_low(trexio_file.pytrexio_s, PYTREXIO_MAX_STR_LENGTH) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + + if rc != TREXIO_SUCCESS: + raise Error(rc) except: raise @@ -2620,9 +2632,10 @@ def read_$group_dset$(trexio_file, dim = None) -> list: try: dset_full = dset_1d_r.split(pytr.TREXIO_DELIM) dset_2d_r = [dset_full[i] for i in range(dim) if dset_full[i]] - assert dset_2d_r - except AssertionError: - raise TypeError(f"Output of {read_$group_dset$.__name__} function cannot be an empty list.") + if not dset_2d_r: + raise ValueError(f"Output of {read_$group_dset$.__name__} function cannot be an empty list.") + except: + raise return dset_2d_r #+end_src @@ -2827,9 +2840,9 @@ def write_$group_str$(trexio_file, str_w: str) -> None: try: rc = pytr.trexio_write_$group_str$(trexio_file.pytrexio_s, str_w, max_str_length) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + + if rc != TREXIO_SUCCESS: + raise Error(rc) except: raise #+end_src @@ -2851,9 +2864,9 @@ def read_$group_str$(trexio_file) -> str: try: rc, str_r = pytr.trexio_read_$group_str$(trexio_file.pytrexio_s, PYTREXIO_MAX_STR_LENGTH) - assert rc==TREXIO_SUCCESS - except AssertionError: - raise Exception(pytr.trexio_string_of_error(rc)) + + if rc != TREXIO_SUCCESS: + raise Error(rc) except: raise From ced8210ff00063c70ac1976a2937df26a4dbff19 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 30 Aug 2021 18:23:24 +0300 Subject: [PATCH 074/107] BUG: trexio_open fails when test script is executed from outside the root test directory --- Makefile.am | 2 +- python/test/test_api.py | 79 +++++++++++++++++++++--------------- python/test/test_pytrexio.py | 2 +- 3 files changed, 49 insertions(+), 34 deletions(-) diff --git a/Makefile.am b/Makefile.am index e833fab..50c8681 100644 --- a/Makefile.am +++ b/Makefile.am @@ -191,7 +191,7 @@ setup_py = $(srcdir)/python/setup.py setup_cfg = $(srcdir)/python/setup.cfg pytrexio_py = $(srcdir)/python/pytrexio/pytrexio.py trexio_py = $(srcdir)/python/trexio.py -TEST_PY = $(srcdir)/python/test/test_api.py +TEST_PY = python/test/test_api.py pytrexio_c = $(srcdir)/src/pytrexio_wrap.c pytrexio_i = $(srcdir)/src/pytrexio.i numpy_i = $(srcdir)/src/numpy.i diff --git a/python/test/test_api.py b/python/test/test_api.py index a97b87e..5f23a91 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python3 + import os import shutil import numpy as np -import trexio as tr +import trexio #=========================================================# #======== SETUP THE BACK END AND OUTPUT FILE NAME ========# @@ -15,9 +17,9 @@ OUTPUT_FILENAME_HDF5 = 'test_py_swig.h5' # define TREXIO file name -if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: +if TEST_TREXIO_BACKEND == trexio.TREXIO_HDF5: output_filename = OUTPUT_FILENAME_HDF5 -elif TEST_TREXIO_BACKEND == tr.TREXIO_TEXT: +elif TEST_TREXIO_BACKEND == trexio.TREXIO_TEXT: output_filename = OUTPUT_FILENAME_TEXT else: raise ValueError ('Specify one of the supported back ends as TEST_TREXIO_BACKEND') @@ -25,9 +27,9 @@ else: # remove TREXIO file if exists in the current directory try: - if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: + if TEST_TREXIO_BACKEND == trexio.TREXIO_HDF5: os.remove(output_filename) - elif TEST_TREXIO_BACKEND == tr.TREXIO_TEXT: + elif TEST_TREXIO_BACKEND == trexio.TREXIO_TEXT: shutil.rmtree(output_filename) except: print ('Nothing to remove.') @@ -39,16 +41,29 @@ except: # create TREXIO file and open it for writing -#test_file = tr.open(output_filename, 'w', TEST_TREXIO_BACKEND) -test_file = tr.File(output_filename, mode='w', back_end=TEST_TREXIO_BACKEND) +#test_file = trexio.open(output_filename, 'w', TEST_TREXIO_BACKEND) +test_file = trexio.File(output_filename, mode='w', back_end=TEST_TREXIO_BACKEND) -# Print docstring of the tr.open function -#print(tr.open.__doc__) +# Print docstring of the trexio.open function +#print(trexio.open.__doc__) nucleus_num = 12 +try: + trexio.write_nucleus_num(test_file, -100) +except trexio.Error: + print("Writing negative nucleus_num: checked.") + # write nucleus_num in the file -tr.write_nucleus_num(test_file, nucleus_num) +try: + trexio.write_nucleus_num(test_file, nucleus_num) +except: + raise + +try: + trexio.write_nucleus_num(test_file, nucleus_num*2) +except trexio.Error: + print("Attempt to overwrite nucleus_num: checked.") # initialize charge arrays as a list and convert it to numpy array charges = [6., 6., 6., 6., 6., 6., 1., 1., 1., 1., 1., 1.] @@ -57,7 +72,7 @@ charges_np = np.array(charges, dtype=np.int32) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived # from the size of the list/array by SWIG using typemaps from numpy.i -tr.write_nucleus_charge(test_file, charges_np) +trexio.write_nucleus_charge(test_file, charges_np) # initialize arrays of nuclear indices as a list and convert it to numpy array indices = [i for i in range(nucleus_num)] @@ -66,7 +81,7 @@ indices_np = np.array(indices, dtype=np.int64) # function call below works with both lists and numpy arrays, dimension needed for memory-safety is derived # from the size of the list/array by SWIG using typemacs from numpy.i -tr.write_basis_nucleus_index(test_file, indices_np) +trexio.write_basis_nucleus_index(test_file, indices_np) # initialize a list of nuclear coordinates coords = [ @@ -85,12 +100,12 @@ coords = [ ] # write coordinates in the file -tr.write_nucleus_coord(test_file, coords) +trexio.write_nucleus_coord(test_file, coords) point_group = 'B3U' # write nucleus_point_group in the file -tr.write_nucleus_point_group(test_file, point_group) +trexio.write_nucleus_point_group(test_file, point_group) labels = [ 'C', @@ -107,13 +122,13 @@ labels = [ 'H'] # write nucleus_label in the file -tr.write_nucleus_label(test_file,labels) +trexio.write_nucleus_label(test_file,labels) # close TREXIO file # [TODO:] this functional call is no longer needed as we introduced TREXIO_File class which has a desctructor that closes the file -#tr.close(test_file) +#trexio.close(test_file) # [TODO:] without calling destructor on test_file the TREXIO_FILE is not getting created and the data is not written when using TEXT back end. This, the user still has to explicitly call destructor on test_file object instead -# tr.close function. This is only an issue when the data is getting written and read in the same session (e.g. in Jupyter notebook) +# trexio.close function. This is only an issue when the data is getting written and read in the same session (e.g. in Jupyter notebook) del test_file @@ -123,68 +138,68 @@ del test_file #==========================================================# # open previously created TREXIO file, now in 'read' mode -#test_file2 = tr.open(output_filename, 'r', TEST_TREXIO_BACKEND) -test_file2 = tr.File(output_filename, 'r', TEST_TREXIO_BACKEND) +#test_file2 = trexio.open(output_filename, 'r', TEST_TREXIO_BACKEND) +test_file2 = trexio.File(output_filename, 'r', TEST_TREXIO_BACKEND) # read nucleus_num from file -rnum = tr.read_nucleus_num(test_file2) +rnum = trexio.read_nucleus_num(test_file2) assert rnum==nucleus_num # safe call to read_nucleus_charge array of float values -rcharges_np = tr.read_nucleus_charge(test_file2, dim=nucleus_num) +rcharges_np = trexio.read_nucleus_charge(test_file2, dim=nucleus_num) assert rcharges_np.dtype is np.dtype(np.float64) np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) # unsafe call to read_safe should fail with error message corresponding to TREXIO_UNSAFE_ARRAY_DIM try: - rcharges_fail = tr.read_nucleus_charge(test_file2, dim=nucleus_num*5) + rcharges_fail = trexio.read_nucleus_charge(test_file2, dim=nucleus_num*5) except Exception: print("Unsafe call to safe API: checked") # safe call to read array of int values (nuclear indices) -rindices_np_16 = tr.read_basis_nucleus_index(test_file2, dim=nucleus_num, dtype=np.int16) +rindices_np_16 = trexio.read_basis_nucleus_index(test_file2, dim=nucleus_num, dtype=np.int16) assert rindices_np_16.dtype is np.dtype(np.int16) for i in range(nucleus_num): assert rindices_np_16[i]==indices_np[i] -rindices_np_32 = tr.read_basis_nucleus_index(test_file2, dim=nucleus_num, dtype=np.int32) +rindices_np_32 = trexio.read_basis_nucleus_index(test_file2, dim=nucleus_num, dtype=np.int32) assert rindices_np_32.dtype is np.dtype(np.int32) for i in range(nucleus_num): assert rindices_np_32[i]==indices_np[i] -rindices_np_64 = tr.read_basis_nucleus_index(test_file2) +rindices_np_64 = trexio.read_basis_nucleus_index(test_file2) assert rindices_np_64.dtype is np.dtype(np.int64) assert rindices_np_64.size==nucleus_num for i in range(nucleus_num): assert rindices_np_64[i]==indices_np[i] # read nuclear coordinates without providing optional argument dim -rcoords_np = tr.read_nucleus_coord(test_file2) +rcoords_np = trexio.read_nucleus_coord(test_file2) assert rcoords_np.size==nucleus_num*3 np.testing.assert_array_almost_equal(rcoords_np, np.array(coords).reshape(nucleus_num,3), decimal=8) # set doReshape to False to get a flat 1D array (e.g. when reading matrices like nuclear coordinates) -#rcoords_reshaped_2 = tr.read_nucleus_coord(test_file2, doReshape=False) +#rcoords_reshaped_2 = trexio.read_nucleus_coord(test_file2, doReshape=False) # read array of nuclear labels -rlabels_2d = tr.read_nucleus_label(test_file2, dim=nucleus_num) +rlabels_2d = trexio.read_nucleus_label(test_file2, dim=nucleus_num) print(rlabels_2d) for i in range(nucleus_num): assert rlabels_2d[i]==labels[i] # read a string corresponding to nuclear point group -rpoint_group = tr.read_nucleus_point_group(test_file2) +rpoint_group = trexio.read_nucleus_point_group(test_file2) assert rpoint_group==point_group # close TREXIO file -#tr.close(test_file2) +#trexio.close(test_file2) # cleaning (remove the TREXIO file) try: - if TEST_TREXIO_BACKEND == tr.TREXIO_HDF5: + if TEST_TREXIO_BACKEND == trexio.TREXIO_HDF5: os.remove(output_filename) - elif TEST_TREXIO_BACKEND == tr.TREXIO_TEXT: + elif TEST_TREXIO_BACKEND == trexio.TREXIO_TEXT: shutil.rmtree(output_filename) except: print (f'No output file {output_filename} has been produced') diff --git a/python/test/test_pytrexio.py b/python/test/test_pytrexio.py index b048bfe..04b1266 100644 --- a/python/test/test_pytrexio.py +++ b/python/test/test_pytrexio.py @@ -113,7 +113,7 @@ np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) # unsafe call to read_safe should not only have return code = TREXIO_UNSAFE_ARRAY_DIM # TODO: it should not return numpy array filled with garbage rc, rcharges_fail = trexio_read_safe_nucleus_charge(test_file2, nucleus_num*5) -assert rc==TREXIO_UNSAFE_ARRAY_DIM +assert rc==23 # less Python-ic way to read/write arrays using Array classes (probably more portable to other languages) #charges2 = doubleArray(nucleus_num) From 3e82fd9ae8eb6c6112ef5c29899d8973409f8b1c Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 31 Aug 2021 12:11:22 +0300 Subject: [PATCH 075/107] FIX: a bug that was leading to Python and valgrind errors --- src/templates_front/templator_front.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 645c1eb..f00e9e8 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -727,7 +727,7 @@ trexio_open(const char* file_name, const char mode, /* Data for the parent type */ - strncpy(result->file_name, file_name, TREXIO_MAX_FILENAME_LENGTH-1); + strncpy(result->file_name, file_name, TREXIO_MAX_FILENAME_LENGTH); if (result->file_name[TREXIO_MAX_FILENAME_LENGTH-1] != '\0') { free(result); return NULL; From 96678fea2eb5a3d4d31aa2af27844279b4fa8758 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 7 Sep 2021 17:14:23 +0200 Subject: [PATCH 076/107] adapt write functions to receive multidimensional arrays or lists --- python/test/test_api.py | 24 ++++----- src/templates_front/templator_front.org | 65 +++++++++++++++---------- 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index 5f23a91..104a7f6 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -85,18 +85,18 @@ trexio.write_basis_nucleus_index(test_file, indices_np) # initialize a list of nuclear coordinates coords = [ - 0.00000000 , 1.39250319 , 0.00000000 , - -1.20594314 , 0.69625160 , 0.00000000 , - -1.20594314 , -0.69625160 , 0.00000000 , - 0.00000000 , -1.39250319 , 0.00000000 , - 1.20594314 , -0.69625160 , 0.00000000 , - 1.20594314 , 0.69625160 , 0.00000000 , - -2.14171677 , 1.23652075 , 0.00000000 , - -2.14171677 , -1.23652075 , 0.00000000 , - 0.00000000 , -2.47304151 , 0.00000000 , - 2.14171677 , -1.23652075 , 0.00000000 , - 2.14171677 , 1.23652075 , 0.00000000 , - 0.00000000 , 2.47304151 , 0.00000000 , + [ 0.00000000 , 1.39250319 , 0.00000000 ], + [-1.20594314 , 0.69625160 , 0.00000000 ], + [-1.20594314 , -0.69625160 , 0.00000000 ], + [ 0.00000000 , -1.39250319 , 0.00000000 ], + [ 1.20594314 , -0.69625160 , 0.00000000 ], + [ 1.20594314 , 0.69625160 , 0.00000000 ], + [-2.14171677 , 1.23652075 , 0.00000000 ], + [-2.14171677 , -1.23652075 , 0.00000000 ], + [ 0.00000000 , -2.47304151 , 0.00000000 ], + [ 2.14171677 , -1.23652075 , 0.00000000 ], + [ 2.14171677 , 1.23652075 , 0.00000000 ], + [ 0.00000000 , 2.47304151 , 0.00000000 ], ] # write coordinates in the file diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index f00e9e8..7af827d 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -1973,35 +1973,48 @@ def write_$group_dset$(trexio_file, dset_w) -> None: - Exception from some other error (e.g. RuntimeError). """ - cutPrecision = False - if not isinstance(dset_w, (list, tuple)): - try: - import numpy as np - except ImportError: - raise Exception("NumPy cannot be imported.") - - if isinstance(dset_w, np.ndarray) and (not dset_w.dtype==np.int64 or not dset_w.dtype==np.float64): - cutPrecision = True - - - if cutPrecision: - try: - dset_64 = np.$group_dset_py_dtype$64(dset_w) - except: - raise - - try: - if cutPrecision: - rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_64) - else: - rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_w) + import numpy as np + except ImportError: + raise Exception("NumPy cannot be imported.") - if rc != TREXIO_SUCCESS: - raise Error(rc) - except: - raise + doConversion = False + doFlatten = False + if not isinstance(dset_w, (list, tuple)): + # if input array is not a list or tuple then it is probably a numpy array + if isinstance(dset_w, np.ndarray) and (not dset_w.dtype==np.int64 or not dset_w.dtype==np.float64): + doConversion = True + if len(dset_w.shape) > 1: + doFlatten = True + if doConversion: + dset_64 = np.$group_dset_py_dtype$64(dset_w).flatten() + else: + dset_flat = np.array(dset_w, dtype=np.$group_dset_py_dtype$64).flatten() + else: + if doConversion: + dset_64 = np.$group_dset_py_dtype$64(dset_w) + + else: + # if input array is a multidimensional list or tuple, we have to convert it + try: + doFlatten = True + ncol = len(dset_w[0]) + dset_flat = np.array(dset_w, dtype=np.$group_dset_py_dtype$64).flatten() + except TypeError: + doFlatten = False + pass + + + if doConversion: + rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_64) + elif doFlatten: + rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_flat) + else: + rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_w) + + if rc != TREXIO_SUCCESS: + raise Error(rc) #+end_src #+begin_src python :tangle read_dset_data_front.py From e72fbb4645d7d094b27ef478ce7d074f43f2f701 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 7 Sep 2021 17:31:25 +0200 Subject: [PATCH 077/107] minor fixed in setup script to be compatible with Docker containers + replaced dependency on h5py with numpy --- python/setup.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/python/setup.py b/python/setup.py index 53dccf6..6084bb9 100644 --- a/python/setup.py +++ b/python/setup.py @@ -18,11 +18,11 @@ VERSIONFILE = "pytrexio/_version.py" try: exec(open(VERSIONFILE).read()) except: - raise IOError(f"Could not open the version file {VERSIONFILE}.") + raise IOError("Could not open the version file %s." % (VERSIONFILE, )) version_r = __version__ if not version_r: - raise RuntimeError(f"Unable to find a version string in {VERSIONFILE}.") + raise RuntimeError("Unable to find a version string in %s." % (VERSIONFILE, )) # =========================== Start of the HDF5 block =========================== # @@ -55,10 +55,16 @@ h5_ldflags = h5_ldflags_withl.split(" ")[0] # ============================ End of the HDF5 block ============================ # +# we need to explicitly provide an include path to the numpy headers in some environments (e.g. in Docker containers) +try: + from numpy import get_include as np_get_include +except ImportError: + raise Exception("numpy Python package has not been found") + # Define pytrexio extension module based on TREXIO source codes + SWIG-generated wrapper pytrexio_module = Extension('pytrexio._pytrexio', sources = [os.path.join(srcpath, code) for code in c_files], - include_dirs = [h5_cflags, srcpath], + include_dirs = [h5_cflags, srcpath, np_get_include()], libraries = ['hdf5', 'hdf5_hl'], extra_compile_args = ['-Wno-discarded-qualifiers'], extra_link_args = [h5_ldflags] @@ -72,6 +78,7 @@ setup(name = 'trexio', description = """Python API of the TREXIO library""", long_description = long_description, long_description_content_type = "text/markdown", + install_requires = ['numpy'], ext_modules = [pytrexio_module], py_modules = ['trexio'], packages = ['pytrexio'], @@ -82,7 +89,6 @@ setup(name = 'trexio', "Programming Language :: C", "License :: OSI Approved :: BSD License", "Operating System :: POSIX :: Linux" - ], - install_requires = ['h5py'] + ] ) From f24a274c4daf76c099b8ef9339c8fcd36fe0fb93 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 7 Sep 2021 17:32:23 +0200 Subject: [PATCH 078/107] add requirements file with packages needed for a proper setup --- python/install_pytrexio.sh | 1 + python/requirements.txt | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 python/requirements.txt diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index eb9fb2e..ff414bf 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -11,6 +11,7 @@ H5_LDFLAGS_LOCAL=$2 # Install/upgrade packages required for the installation python3 -m pip install --upgrade setuptools wheel twine +python3 -m pip install -r requirements.txt # Create build directory and compile extension files (*.c) # --no-user-cfg disables custom .cfg files of the user machine, so that only setup.cfg is used diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000..67128a1 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,2 @@ +pkgconfig +numpy From 04a713590a8b20259ec90674fc9dd0a725f39db4 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 8 Sep 2021 19:26:30 +0200 Subject: [PATCH 079/107] FIX bug due to the direct import of numpy in the setup script + add more classifiers --- python/setup.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/python/setup.py b/python/setup.py index 6084bb9..323ef5b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,9 +1,7 @@ -#!/usr/bin/env python - """ setup.py file for TREXIO Python package """ - +from distutils.sysconfig import get_python_inc from setuptools import setup, Extension import os @@ -11,6 +9,8 @@ rootpath = os.path.dirname(os.path.abspath(__file__)) srcpath = os.path.join(rootpath, 'src') c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] +numpy_includedir = os.path.join(get_python_inc(plat_specific=1), 'numpy') + with open("README.md", "r") as fh: long_description = fh.read() @@ -55,16 +55,10 @@ h5_ldflags = h5_ldflags_withl.split(" ")[0] # ============================ End of the HDF5 block ============================ # -# we need to explicitly provide an include path to the numpy headers in some environments (e.g. in Docker containers) -try: - from numpy import get_include as np_get_include -except ImportError: - raise Exception("numpy Python package has not been found") - # Define pytrexio extension module based on TREXIO source codes + SWIG-generated wrapper pytrexio_module = Extension('pytrexio._pytrexio', sources = [os.path.join(srcpath, code) for code in c_files], - include_dirs = [h5_cflags, srcpath, np_get_include()], + include_dirs = [h5_cflags, srcpath, numpy_includedir], libraries = ['hdf5', 'hdf5_hl'], extra_compile_args = ['-Wno-discarded-qualifiers'], extra_link_args = [h5_ldflags] @@ -78,17 +72,25 @@ setup(name = 'trexio', description = """Python API of the TREXIO library""", long_description = long_description, long_description_content_type = "text/markdown", - install_requires = ['numpy'], ext_modules = [pytrexio_module], py_modules = ['trexio'], packages = ['pytrexio'], url = 'https://github.com/TREX-CoE/trexio', license = 'BSD', classifiers=[ - "Programming Language :: Python :: 3", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Topic :: Scientific/Engineering", "Programming Language :: C", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", "License :: OSI Approved :: BSD License", - "Operating System :: POSIX :: Linux" - ] + "Operating System :: POSIX", + "Operating System :: Unix", + "Operating System :: MacOS" + ], + install_requires = ['numpy'] ) From 67e59f156c67508762f8f1ffb0143642b8b151a9 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 8 Sep 2021 19:28:18 +0200 Subject: [PATCH 080/107] explicitly provide __version__ attribute of the trexio module --- src/templates_front/templator_front.org | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 7af827d..c974fa1 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -85,9 +85,7 @@ module trexio #+begin_src python :tangle prefix_python.py """The Python API of the TREXIO library. -This module contains wrappers of the pytrexio.py module produced by SWIG. -Top-level wrapper like this allows to extend functionality of SWIG-generated codes. -Moreover, exception handling becomes trivial thanks to the use of TREXIO exit codes. +This package is a top-level wrapper of the SWIG-generated pytrexio module. """ @@ -98,6 +96,13 @@ except ImportError: # define max length of a string to be read, required for the low-level C routines PYTREXIO_MAX_STR_LENGTH = 2048 + +# setuptools do not assign __version__ variable to the trexio package, so we set it manually +from os import path +__trexio_path__ = path.dirname(path.abspath(__file__)) +__pytrexio_path__ = path.join(__trexio_path__, 'pytrexio') +with open(path.join(__pytrexio_path__, '_version.py')) as version_file: + __version__ = version_file.read().split('"')[1] #+end_src * Coding conventions From 04529823f609b6a7584f0433022b1095c078c8a6 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 8 Sep 2021 19:34:16 +0200 Subject: [PATCH 081/107] add README file for the Python API --- README.md | 7 ++++- python/.gitignore | 1 - python/README.md | 65 +++++++++++++++++++++++++++++++++++++++++ tools/prepare_python.sh | 2 +- 4 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 python/README.md diff --git a/README.md b/README.md index 4c5aaf2..74fda47 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ TREX library for efficient I/O. ## Minimal requirements (for users): -- Autotools (autoconf, automake, libtool) +- Autotools (autoconf >= 2.69, automake >= 1.11, libtool >= 2.2) - C compiler (gcc/icc/clang) - Fortran compiler (gfortran/ifort) - HDF5 library (>= 1.8) @@ -64,6 +64,11 @@ There is no naming conflict when, for example, `num` variable exists both in the These quantities can be accessed using the corresponding `trexio_[has|read|write]_nucleus_num` and `trexio_[has|read|write]_mo_num`, respectively. +## Python API + +For more details regarding the installation and usage of the TREXIO Python API, see [this page](python/README.md). + + ## Tutorial **TODO** diff --git a/python/.gitignore b/python/.gitignore index a4b23d2..f1c8046 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,6 +1,5 @@ AUTHORS LICENSE -README.md src/ build/ diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..64fa49c --- /dev/null +++ b/python/README.md @@ -0,0 +1,65 @@ +## TREXIO Python API + +TREXIO provides a Python API for interactive calls to the library. +It allows to simplify interfacing between different codes and can +be used to convert between different input/output file formats. + + +### Requirements + +- python3 (>= 3.6) +- numpy +- C compiler (gcc/icc) + +- HDF5 library (when compiling from source) +- pkgconfig (when compiling from source ----------> TODO: CHECK THIS by installing wheels) + + +### Installation from PyPI + +Run `pip3 install trexio` + +**Note:** we highly recommend to use virtual environments to avoid compatibility issues. + + +### Installation from source + +1. Download the latest source code distribution (in `.tar.gz` format) of the TREXIO Python API +2. Unpack and `cd` in the output directory +3. Run `pip3 install -r requirements.txt` (this installs all python dependencies) +4. Run `pip3 install .` (this install `trexio` in your environment) +5. Run `cd test && python3 test_api.py` (this executes several tests that check the installation) + +You are ready to go! + + +### Examples + +An interactive `Jupyter` notebook called `tutorial_benzene.ipynb` can be found in the `examples` directory or on Binder (TODO: link). +It's goal is to demonstrate some basic use cases of the `trexio` Python API. + + +#### Additional requirements to run Jupyter notebooks with TREXIO + +`Jupyter` can be installed using `pip install jupyter`. + +If you have installed `trexio` in the virtual environemnt called, e.g. `myvenv`, make sure to also install it as a kernel for `ipython` (requires `ipykernel` to be installed) by executing the following: + +`python3 -m ipykernel install --user --name=myvenv` + + +#### Running the notebook + +The example notebook can be launched using the following command + +`jupyter-notebook tutorial_benzene.ipynb` + +Once the notebook is open, make sure that your virtual environment is selected as the current kernel. +If this is not the case: + +1. Press the `Kernel` button in the navigation panel +2. In the output list of options select `Change kernel` +3. Find the name of your virtual environment (e.g. `myvenv`) in the list and select it + +That's it, you have activated the virtual environment and can now run the cells of the `tutorial_benzene.ipynb` notebook. + diff --git a/tools/prepare_python.sh b/tools/prepare_python.sh index 4808d15..be35d36 100755 --- a/tools/prepare_python.sh +++ b/tools/prepare_python.sh @@ -30,5 +30,5 @@ cp ${SRC}/*.h ${PYDIR}/src cp ${INCLUDIR}/trexio.h ${PYDIR}/src # Copy additional info -cp ${TREXIO_ROOT}/AUTHORS ${TREXIO_ROOT}/LICENSE ${TREXIO_ROOT}/README.md ${PYDIR} +cp ${TREXIO_ROOT}/AUTHORS ${TREXIO_ROOT}/LICENSE ${PYDIR} From a8d14f4c791448c8493e43ab17ef1dbe5457ca60 Mon Sep 17 00:00:00 2001 From: q-posev Date: Wed, 8 Sep 2021 19:53:45 +0200 Subject: [PATCH 082/107] better README for python --- python/README.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/python/README.md b/python/README.md index 64fa49c..c43d260 100644 --- a/python/README.md +++ b/python/README.md @@ -11,15 +11,18 @@ be used to convert between different input/output file formats. - numpy - C compiler (gcc/icc) -- HDF5 library (when compiling from source) -- pkgconfig (when compiling from source ----------> TODO: CHECK THIS by installing wheels) - ### Installation from PyPI Run `pip3 install trexio` -**Note:** we highly recommend to use virtual environments to avoid compatibility issues. +**Note:** we highly recommend to use virtual environments to avoid compatibility issues. + + +### Additional requirements (for installation from source) + +- HDF5 library (>= 1.8) +- pkgconfig (TODO: CHECK THIS by installing wheels) ### Installation from source @@ -35,15 +38,15 @@ You are ready to go! ### Examples -An interactive `Jupyter` notebook called `tutorial_benzene.ipynb` can be found in the `examples` directory or on Binder (TODO: link). -It's goal is to demonstrate some basic use cases of the `trexio` Python API. +An interactive Jupyter notebook called `tutorial_benzene.ipynb` can be found in the `examples` directory or on Binder (TODO: link). +It is provided to demonstrate some basic use cases of the TREXIO library in general and the Python API in particular. #### Additional requirements to run Jupyter notebooks with TREXIO -`Jupyter` can be installed using `pip install jupyter`. +Jupyter can be installed using `pip install jupyter`. -If you have installed `trexio` in the virtual environemnt called, e.g. `myvenv`, make sure to also install it as a kernel for `ipython` (requires `ipykernel` to be installed) by executing the following: +If you have installed `trexio` in the virtual environemnt called, e.g. `myvenv`, make sure to also install it as a kernel for (this requires `ipykernel` python package to be installed) by executing the following: `python3 -m ipykernel install --user --name=myvenv` @@ -55,7 +58,7 @@ The example notebook can be launched using the following command `jupyter-notebook tutorial_benzene.ipynb` Once the notebook is open, make sure that your virtual environment is selected as the current kernel. -If this is not the case: +If this is not the case, try the following: 1. Press the `Kernel` button in the navigation panel 2. In the output list of options select `Change kernel` @@ -63,3 +66,7 @@ If this is not the case: That's it, you have activated the virtual environment and can now run the cells of the `tutorial_benzene.ipynb` notebook. +To uninstall the kernel named `myvenv` from Jupyter, execute the following: + +`jupyter kernelspec uninstall myvenv + From b9bbf178c55ae6d31198ff6835c6801813b462ce Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 9 Sep 2021 15:06:16 +0200 Subject: [PATCH 083/107] add make python-sdist rule --- Makefile.am | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 50c8681..3f81463 100644 --- a/Makefile.am +++ b/Makefile.am @@ -205,6 +205,10 @@ python-install: $(pytrexio_py) $(setup_py) $(setup_cfg) cd python && \ ./install_pytrexio.sh $(HDF5_CFLAGS) $(HDF5_LDFLAGS) +python-sdist: $(pytrexio_py) $(setup_py) $(setup_cfg) + cd python && \ + python setup.py sdist + $(pytrexio_py): $(pytrexio_c) cd tools && ./prepare_python.sh @@ -228,7 +232,7 @@ CLEANFILES += $(pytrexio_c) \ python/src/*.c \ python/src/*.h -.PHONY: cppcheck python-test python-install check-numpy +.PHONY: cppcheck python-test python-install python-sdist check-numpy endif From 09937c58c59523a55c8e725f94f0f4dc5c97e300 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 9 Sep 2021 15:06:26 +0200 Subject: [PATCH 084/107] better README --- python/README.md | 49 +++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/python/README.md b/python/README.md index c43d260..c7c7b43 100644 --- a/python/README.md +++ b/python/README.md @@ -1,8 +1,8 @@ ## TREXIO Python API -TREXIO provides a Python API for interactive calls to the library. -It allows to simplify interfacing between different codes and can -be used to convert between different input/output file formats. +TREXIO provides a Python API, which enables interactive calls to the library. +It facilitates the development of interfaces between different codes and +can be used to convert data from one input/output file format into another. ### Requirements @@ -16,7 +16,8 @@ be used to convert between different input/output file formats. Run `pip3 install trexio` -**Note:** we highly recommend to use virtual environments to avoid compatibility issues. +**Note: we highly recommend to use virtual environments to avoid compatibility issues.** +For more details, see the corresponding part of the [Python documentation](https://docs.python.org/3/library/venv.html#creating-virtual-environments). ### Additional requirements (for installation from source) @@ -30,43 +31,45 @@ Run `pip3 install trexio` 1. Download the latest source code distribution (in `.tar.gz` format) of the TREXIO Python API 2. Unpack and `cd` in the output directory 3. Run `pip3 install -r requirements.txt` (this installs all python dependencies) -4. Run `pip3 install .` (this install `trexio` in your environment) -5. Run `cd test && python3 test_api.py` (this executes several tests that check the installation) +4. Run `pip3 install .` (this installs `trexio` in your environment) +5. Run `cd test && python3 test_api.py` (this executes several tests that verify the installation) You are ready to go! ### Examples -An interactive Jupyter notebook called `tutorial_benzene.ipynb` can be found in the `examples` directory or on Binder (TODO: link). -It is provided to demonstrate some basic use cases of the TREXIO library in general and the Python API in particular. +An interactive Jupyter notebook called `tutorial_benzene.ipynb` is provided in the `examples` directory. +It demonstrates some basic use cases of the TREXIO library in general and of the Python API in particular. - -#### Additional requirements to run Jupyter notebooks with TREXIO - -Jupyter can be installed using `pip install jupyter`. - -If you have installed `trexio` in the virtual environemnt called, e.g. `myvenv`, make sure to also install it as a kernel for (this requires `ipykernel` python package to be installed) by executing the following: - -`python3 -m ipykernel install --user --name=myvenv` +Jupyter can be installed using `pip install jupyter`. If you are not familiar with it, feel free to consult the [Jupyter documentation](https://jupyter-notebook.readthedocs.io/en/stable/notebook.html). #### Running the notebook -The example notebook can be launched using the following command +The example notebook can be launched using the following command: -`jupyter-notebook tutorial_benzene.ipynb` +`jupyter notebook tutorial_benzene.ipynb` -Once the notebook is open, make sure that your virtual environment is selected as the current kernel. -If this is not the case, try the following: + +#### Additional steps needed to run a custom virtual environment in Jupyter notebooks + +If you have installed `trexio` in a virtual environemnt called, e.g. `myvenv`, but would like to use your system-wide Jupyter installation, this is also possible. +This requires `ipykernel` python package to be installed, which usually comes together with the Jupyter installation. If this is not the case, run `pip install ipykernel`. +You can install `myvenv` as a kernel by executing the following command: + +`python3 -m ipykernel install --user --name=myvenv` + +Now you can launch a Jupyter notebook. Once it is open, make sure that your virtual environment is selected as the current kernel. +If this is not the case, try this: 1. Press the `Kernel` button in the navigation panel 2. In the output list of options select `Change kernel` 3. Find the name of your virtual environment (e.g. `myvenv`) in the list and select it -That's it, you have activated the virtual environment and can now run the cells of the `tutorial_benzene.ipynb` notebook. +That's it, you have activated the custom virtual environment called `myvenv` in your notebook. -To uninstall the kernel named `myvenv` from Jupyter, execute the following: +To uninstall the kernel named `myvenv`, execute the following command: -`jupyter kernelspec uninstall myvenv +`jupyter kernelspec uninstall myvenv` From 58b611e21bf9f7e7219de8e8a744794c62e37c13 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 9 Sep 2021 15:08:01 +0200 Subject: [PATCH 085/107] add setuptools and jupyter to requirements.txt --- python/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/requirements.txt b/python/requirements.txt index 67128a1..769a89d 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,2 +1,4 @@ +setuptools>=42 pkgconfig numpy +jupyter From b9c3ddb30e7858e57cab50cd802e939778d6c502 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 9 Sep 2021 15:09:26 +0200 Subject: [PATCH 086/107] FIX: bug related to import numpy in setup.py script use the custom NUMPY_INCLUDEDIR environment variable derived from a call to numpy.get_include --- python/MANIFEST.in | 2 +- python/set_NUMPY_INCLUDEDIR.sh | 4 ++++ python/setup.py | 31 +++++++++++++++++++++++++++---- 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100755 python/set_NUMPY_INCLUDEDIR.sh diff --git a/python/MANIFEST.in b/python/MANIFEST.in index eab38bc..babee5d 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1 +1 @@ -include src/*.c src/trexio*.h +include src/*.c src/trexio*.h examples/*.ipynb requirements.txt set_NUMPY_INCLUDEDIR.sh diff --git a/python/set_NUMPY_INCLUDEDIR.sh b/python/set_NUMPY_INCLUDEDIR.sh new file mode 100755 index 0000000..1c7ba4f --- /dev/null +++ b/python/set_NUMPY_INCLUDEDIR.sh @@ -0,0 +1,4 @@ + +INCLUDEDIR=`python -c 'import numpy; print(numpy.get_include())'` +export NUMPY_INCLUDEDIR=${INCLUDEDIR} + diff --git a/python/setup.py b/python/setup.py index 323ef5b..839b957 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,15 +1,36 @@ +#!/usr/bin/env python3 """ setup.py file for TREXIO Python package """ -from distutils.sysconfig import get_python_inc -from setuptools import setup, Extension + + import os +from setuptools import setup, Extension + +# this was recommended to solve the problem of the missing numpy header files +# bit it causes `pip install .` to fail with numpy module not found error +#try: +# import numpy +#except ImportError: +# raise Exception("numpy Python package cannot be imported.") +#numpy_includedir = numpy.get_include() + +# this does not cause aforementioned issue but the includedir points to system-wide numpy and not to venv-wide +#from distutils.sysconfig import get_python_inc +#numpy_includedir = os.path.join(get_python_inc(plat_specific=1), 'numpy') + +# dirty workaround: get numpy includedir from the environment variable that can be pre-set using set_NUMPY_INCLUDEDIR.sh +numpy_includedir = os.environ.get("NUMPY_INCLUDEDIR", None) +numpy_isUndefined = numpy_includedir is None or numpy_includedir=="" + +if numpy_isUndefined: + raise Exception("NUMPY_INCLUDEDIR environment variable is not specified. Please do it manually or execute set_NUMPY_INCLUDEDIR.sh script.") + rootpath = os.path.dirname(os.path.abspath(__file__)) srcpath = os.path.join(rootpath, 'src') c_files = ['trexio.c', 'trexio_hdf5.c', 'trexio_text.c', 'pytrexio_wrap.c'] -numpy_includedir = os.path.join(get_python_inc(plat_specific=1), 'numpy') with open("README.md", "r") as fh: long_description = fh.read() @@ -40,7 +61,7 @@ if h5_ldflags_isUndefined or h5_cflags_isUndefined: try: import pkgconfig as pk except ImportError: - raise Exception("pkgconfig Python package has not been found") + raise Exception("pkgconfig Python package cannot be imported.") try: assert pk.exists('hdf5') @@ -91,6 +112,8 @@ setup(name = 'trexio', "Operating System :: Unix", "Operating System :: MacOS" ], + python_requires = ">=3.6", + setup_requires = ['numpy', 'pkgconfig'], install_requires = ['numpy'] ) From 569fac5578e97861ef41366e454a89d219cbf857 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 9 Sep 2021 17:28:37 +0200 Subject: [PATCH 087/107] move set_NUMPY_INCLUDEDIR into tools directory --- python/MANIFEST.in | 2 +- python/{ => tools}/set_NUMPY_INCLUDEDIR.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename python/{ => tools}/set_NUMPY_INCLUDEDIR.sh (100%) diff --git a/python/MANIFEST.in b/python/MANIFEST.in index babee5d..fb39dac 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1 +1 @@ -include src/*.c src/trexio*.h examples/*.ipynb requirements.txt set_NUMPY_INCLUDEDIR.sh +include src/*.c src/trexio*.h examples/*.ipynb requirements.txt tools/set_NUMPY_INCLUDEDIR.sh diff --git a/python/set_NUMPY_INCLUDEDIR.sh b/python/tools/set_NUMPY_INCLUDEDIR.sh similarity index 100% rename from python/set_NUMPY_INCLUDEDIR.sh rename to python/tools/set_NUMPY_INCLUDEDIR.sh From dcfe1da87075b01b370d6cde9b1d6a5d38127ed9 Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 9 Sep 2021 17:29:17 +0200 Subject: [PATCH 088/107] better documentation --- python/README.md | 13 +++++++++---- python/README_SETUP.txt | 24 ------------------------ python/install_pytrexio.sh | 18 +++++++----------- 3 files changed, 16 insertions(+), 39 deletions(-) delete mode 100644 python/README_SETUP.txt diff --git a/python/README.md b/python/README.md index c7c7b43..8decbf9 100644 --- a/python/README.md +++ b/python/README.md @@ -31,8 +31,12 @@ For more details, see the corresponding part of the [Python documentation](https 1. Download the latest source code distribution (in `.tar.gz` format) of the TREXIO Python API 2. Unpack and `cd` in the output directory 3. Run `pip3 install -r requirements.txt` (this installs all python dependencies) -4. Run `pip3 install .` (this installs `trexio` in your environment) -5. Run `cd test && python3 test_api.py` (this executes several tests that verify the installation) +4. Export custom environment variables needed for the installation. Steps 1 and 2 can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options). + 1. `export H5_CFLAGS=-I/path/to/hdf5/include` + 2. `export H5_LDFLAGS=-L/path/to/hdf5/lib` + 3. `source tools/set_NUMPY_INCLUDEDIR.sh` +5. Run `pip3 install .` (this installs `trexio` in your environment) +6. Run `cd test && python3 test_api.py` (this executes several tests that verify the installation) You are ready to go! @@ -54,8 +58,9 @@ The example notebook can be launched using the following command: #### Additional steps needed to run a custom virtual environment in Jupyter notebooks -If you have installed `trexio` in a virtual environemnt called, e.g. `myvenv`, but would like to use your system-wide Jupyter installation, this is also possible. -This requires `ipykernel` python package to be installed, which usually comes together with the Jupyter installation. If this is not the case, run `pip install ipykernel`. +In some cases, it may happen that the Jupyter kernels in the activated virtual environment (e.g. `myvenv`) still point to the system-wide python binaries and not to the environment ones. +This will result in `ImportError` when importing `trexio` in the notebook cell. In order to avoid this, the `myvenv` has to be installed as an additional kernel. +This requires `ipykernel` python package, which usually comes together with the Jupyter installation. If this is not the case, run `pip install ipykernel`. You can install `myvenv` as a kernel by executing the following command: `python3 -m ipykernel install --user --name=myvenv` diff --git a/python/README_SETUP.txt b/python/README_SETUP.txt deleted file mode 100644 index b989b27..0000000 --- a/python/README_SETUP.txt +++ /dev/null @@ -1,24 +0,0 @@ -# Additional information related to the installation of the TREXIO Python API - -Removing MANIFEST.in leads to issues in the installation. In particular, the .c and .h source files do not get copied -from the src/ directory into the tar.gz which is produced by setup sdist command. -These source files are required to build the pytrexio.so extension module, which is needed for the Python API. - -Then the following commands do the job of installing trexio in virtual environment: - -$ python3 -m pip install --upgrade setuptools wheel twine -$ python3 -s setup.py --no-user-cfg build -$ python3 setup.py sdist bdist_wheel -$ python3 -m pip install dist/trexio-0.1-*.whl --force-reinstall - -BUG: when trying to install trexio from tar.gz, trexio appears to be installed -after user runs [python3 -s setup.py --no-user-cfg build] -but before [python3 -m pip install dist/{WHEELNAME}.whl] -which leads to pip install saying that trexio is already installed but the test fails saying that _trexio was not found, -indicating that probably first installation did not go well. -FIX: Use --force-reinstall option with pip install - -The installed in virtual environment trexio can be uninstalled using [pip uninstall trexio] -but only from the root directory where the initially used (to install) setup.py script is located. -For example, it is most probably the python/ directory of the TREXIO_ROOT folder. - diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index ff414bf..81cde9f 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -6,8 +6,6 @@ set -e H5_CFLAGS_LOCAL=$1 H5_LDFLAGS_LOCAL=$2 -# This script should update the version of the Python package -#source version.py # Install/upgrade packages required for the installation python3 -m pip install --upgrade setuptools wheel twine @@ -33,23 +31,21 @@ python3 -m pip install dist/trexio-*.whl --force-reinstall # Run the command below in the root directory to install the package in 'editable' (-e) mode without dependencies (--no-deps) #python -m pip install -e . --no-deps -# Uninstall pytrexio: this only works from the pytrexio root directory (more likely python/ folder) and only after --force-reinstall +# Uninstall pytrexio: this only works from the pytrexio root directory #python3 -m pip uninstall trexio # Test the current release by uploading to TestPyPI sandbox #python3 -m twine upload --repository testpypi dist/trexio-*.tar.gz -# NOTE:I get an error: -# Binary wheel 'trexio-0.1-cp38-cp38-linux_x86_64.whl' has an unsupported platform tag 'linux_x86_64'. -# when uploading the wheel instead of the tar.gz file to testpypi -# This is a well-known issue, see https://stackoverflow.com/questions/59451069/binary-wheel-cant-be-uploaded-on-pypi-using-twine -# Once fixed both .tar.gz and .whl distribution can be uploaded, meaning that dist/trexio-*.tar.gz can be replaced by dist/* -# -# Compatibility tags for Linux have been discussed in PEP-513 https://www.python.org/dev/peps/pep-0513/ - # Upload updated version of PyTREXIO to PyPI #python3 -m twine upload dist/trexio-*.tar.gz # Cleaning rm -rf build dist trexio.egg-info +# Additional information related to the installation of the TREXIO Python API + +#Removing MANIFEST.in leads to issues in the installation. In particular, the .c and .h source files do not get copied +#from the src/ directory into the tar.gz which is produced by setup sdist command. +#These source files are required to build the pytrexio.so extension module, which is needed for the Python API. + From c63d420c7f35afe1551a8960da9c7dc235513bab Mon Sep 17 00:00:00 2001 From: q-posev Date: Thu, 9 Sep 2021 17:35:24 +0200 Subject: [PATCH 089/107] WIP: Docker containers and bash scripts to produce manylinux wheels for different platforms --- docker/README | 45 +++++ .../build_manylinux_wheels_py_36_37_38_39.sh | 165 ++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 docker/README create mode 100755 docker/build_manylinux_wheels_py_36_37_38_39.sh diff --git a/docker/README b/docker/README new file mode 100644 index 0000000..31a6300 --- /dev/null +++ b/docker/README @@ -0,0 +1,45 @@ +First, make sure to place the source code distribution (suffixed with .tar.gz) of the TREXIO Python API in this directory. + +TODO: the scripts have to be adapted for an arbitrary version of TREXIO and Python ! + +Outside Docker image: + +# Build containers with hdf5 inside: + +# for manylinux2014_x86_64 +sudo docker build -t hdf5_1_12_on_2014_x86_64 . -f Dockerfile_2014_x86_64 + +# for manylinux_2_24_x86_64 +sudo docker build -t hdf5_1_12_on_2_24_x86_64 . -f Dockerfile_2_24_x86_64 + +# (create an image using HDF5 containers, see https://github.com/h5py/hdf5-manylinux) +# -t hdf5_1_12_on_${PLATFORM} builds an image with a custom name (hdf5_1_12_1) + +# Run one of the produced containers in the interactive mode + +# for manylinux2014_x86_64 +sudo docker run --rm -it -e PLAT=manylinux2014_x86_64 -v `pwd`:/tmp hdf5_1_12_on_2014_x86_64 /bin/bash + +# for manylinux_2_24_x86_64 +sudo docker run --rm -it -e PLAT=manylinux_2_24_x86_64 -v `pwd`:/tmp hdf5_1_12_on_2_24_x86_64 /bin/bash + +#(PLAT specifies the base platform tag for manilinux) +#-i (run docker container interactively) +#-t f8e4232fa208 (run an image using ID if the image is not named) +#/bin/bash (a binary to run inside a container environment) + +In the Docker image: + +cd tmp && ./build_manylinux_wheels_py_36_37_38.sh +# (creates virtual environments, installs things with pip, produces conventional wheels and repairs them with auditwheel) + +# auditwheel repair trexio-0.1.0/dist/trexio-0.1.0-cp37-cp37m-linux_x86_64.whl +# (repairs wheel and produces manylinux one) + +After Docker container execution: + +# the produced manylinux wheels are in trexio-0.1.0/wheelhouse directory + +# ALTERNATIVELY: one can copy the produced manylinux wheel from the container back to the host system +sudo docker cp $CONTAINER_ID:/tmp/trexio-0.1.0/wheelhouse/MANYLINUX_NAME.whl directory-on-the-user-machine/ + diff --git a/docker/build_manylinux_wheels_py_36_37_38_39.sh b/docker/build_manylinux_wheels_py_36_37_38_39.sh new file mode 100755 index 0000000..d65872e --- /dev/null +++ b/docker/build_manylinux_wheels_py_36_37_38_39.sh @@ -0,0 +1,165 @@ +set -x +set -e + +export H5_LDFLAGS=-L/usr/local/lib +export H5_CFLAGS=-I/usr/local/include + +# install emacs for Debian + +#apt-get update +#apt install software-properties-common -y +#apt-get install wget -y + +#wget -q http://emacs.ganneff.de/apt.key -O- | apt-key add + +#add-apt-repository "deb http://emacs.ganneff.de/ stretch main" +#apt-get update +#apt-get install emacs-snapshot -y +#update-alternatives --config emacsclient + +# =============================== + +# install TREXIO in the container from the GitHub repo clone + +#apt-get install git -y + +#git clone https://github.com/TREX-CoE/trexio.git +#cd trexio +#git checkout swig-python + +#./autogen.sh +#TREXIO_DEVEL=1 ./configure --enable-silent-rules +#make +#make check + +# =============================== + +# alternatively: build wheel directly from developer-provided .tar.gz of TREXIO (generated with `python setup.py sdist`) +# note: trexio-VERSION.tar.gz has to be in the root directory of the host machine + +# unzip and enter the folder with TREXIO Python API +gzip -cd /tmp/trexio-0.1.0.tar.gz | tar xvf - +cd trexio-0.1.0 + +# create and activate a virtual environment based on CPython version 3.6 +/opt/python/cp36-cp36m/bin/python3 -m venv --clear trexio-manylinux-py36 +source trexio-manylinux-py36/bin/activate +python3 --version + +# upgrade pip, otherwise it complains that manylinux wheel is "...not supported wheel on this platform" +pip install --upgrade pip +# install dependencies needed to build manylinux wheel +pip install --upgrade setuptools wheel auditwheel numpy + +# set an environment variable needed to locate numpy header files +source tools/set_NUMPY_INCLUDEDIR.sh + +# produce conventional (non-manylinux) wheel +python3 setup.py bdist_wheel + +# use auditwheel from PyPA to repair all wheels and make them manylinux-compatible +auditwheel repair dist/trexio-0.1.0-cp36-cp36m-*.whl + +# install the produced manylinux wheel in the virtual environment +python3 -m pip install wheelhouse/trexio-0.1.0-cp36-cp36m-manylinux*.whl + +# run test script +cd test && python3 test_api.py && cd .. + +# cleaning +rm -rf -- dist/ build/ trexio.egg-info/ + +# deactivate the current environment +deactivate + +# create and activate a virtual environment based on CPython version 3.7 +/opt/python/cp37-cp37m/bin/python3 -m venv --clear trexio-manylinux-py37 +source trexio-manylinux-py37/bin/activate +python3 --version + +# upgrade pip, otherwise it complains that manylinux wheel is "...not supported wheel on this platform" +pip install --upgrade pip +# install dependencies needed to build manylinux wheel +pip install --upgrade setuptools wheel auditwheel numpy + +# set an environment variable needed to locate numpy header files +source tools/set_NUMPY_INCLUDEDIR.sh + +# produce conventional (non-manylinux) wheel +python3 setup.py bdist_wheel + +# use auditwheel from PyPA to repair all wheels and make them manylinux-compatible +auditwheel repair dist/trexio-0.1.0-cp37-cp37m-*.whl + +# install the produced manylinux wheel in the virtual environment +python3 -m pip install wheelhouse/trexio-0.1.0-cp37-cp37m-manylinux*.whl + +# run test script +cd test && python3 test_api.py && cd .. + +# cleaning +rm -rf -- dist/ build/ trexio.egg-info/ + +# deactivate the current environment +deactivate + +# create and activate a virtual environment based on CPython version 3.8 +# NOTE: starting from CPython 3.8 there is no need to add m in the abi-tag, e.g. use cp38-cp38 instead of cp38-cp38m +/opt/python/cp38-cp38/bin/python3 -m venv --clear trexio-manylinux-py38 +source trexio-manylinux-py38/bin/activate +python3 --version + +# upgrade pip, otherwise it complains that manylinux wheel is "...not supported wheel on this platform" +pip install --upgrade pip +# install dependencies needed to build manylinux wheel +pip3 install --upgrade setuptools wheel auditwheel numpy + +# set an environment variable needed to locate numpy header files +source tools/set_NUMPY_INCLUDEDIR.sh + +# produce conventional (non-manylinux) wheel +python3 setup.py bdist_wheel + +# use auditwheel from PyPA to repair all wheels and make them manylinux-compatible +auditwheel repair dist/trexio-0.1.0-cp38-cp38-*.whl + +# install the produced manylinux wheel in the virtual environment +python3 -m pip install wheelhouse/trexio-0.1.0-cp38-cp38-manylinux*.whl + +# run test script +cd test && python3 test_api.py && cd .. + +# cleaning +rm -rf -- dist/ build/ trexio.egg-info/ + +# deactivate the current environment +deactivate + +# create and activate a virtual environment based on CPython version 3.8 +/opt/python/cp39-cp39/bin/python3 -m venv --clear trexio-manylinux-py39 +source trexio-manylinux-py39/bin/activate +python3 --version + +# upgrade pip, otherwise it complains that manylinux wheel is "...not supported wheel on this platform" +pip install --upgrade pip +# install dependencies needed to build manylinux wheel +pip3 install --upgrade setuptools wheel auditwheel numpy + +# produce conventional (non-manylinux) wheel +python3 setup.py bdist_wheel + +# use auditwheel from PyPA to repair all wheels and make them manylinux-compatible +auditwheel repair dist/trexio-0.1.0-cp39-cp39-*.whl + +# install the produced manylinux wheel in the virtual environment +python3 -m pip install wheelhouse/trexio-0.1.0-cp39-cp39-manylinux*.whl + +# run test script +cd test && python3 test_api.py && cd .. + +# cleaning +rm -rf -- dist/ build/ trexio.egg-info/ + +# deactivate the current environment +deactivate + From 40d5fc31fcaa9cb684ca9770f85ac27abcb825ea Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 10 Sep 2021 09:38:26 +0200 Subject: [PATCH 090/107] check that the data object exists before reading it from the file --- src/templates_front/templator_front.org | 73 +++++++++++++++---------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index c974fa1..371e819 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -148,34 +148,36 @@ with open(path.join(__pytrexio_path__, '_version.py')) as version_file: ** Error handling #+NAME: table-exit-codes - | Macro | Code | Description | - |------------------------------+------+--------------------------------------| - | ~TREXIO_FAILURE~ | -1 | 'Unknown failure' | - | ~TREXIO_SUCCESS~ | 0 | 'Success' | - | ~TREXIO_INVALID_ARG_1~ | 1 | 'Invalid argument 1' | - | ~TREXIO_INVALID_ARG_2~ | 2 | 'Invalid argument 2' | - | ~TREXIO_INVALID_ARG_3~ | 3 | 'Invalid argument 3' | - | ~TREXIO_INVALID_ARG_4~ | 4 | 'Invalid argument 4' | - | ~TREXIO_INVALID_ARG_5~ | 5 | 'Invalid argument 5' | - | ~TREXIO_END~ | 6 | 'End of file' | - | ~TREXIO_READONLY~ | 7 | 'Read-only file' | - | ~TREXIO_ERRNO~ | 8 | strerror(errno) | - | ~TREXIO_INVALID_ID~ | 9 | 'Invalid ID' | - | ~TREXIO_ALLOCATION_FAILED~ | 10 | 'Allocation failed' | - | ~TREXIO_HAS_NOT~ | 11 | 'Element absent' | - | ~TREXIO_INVALID_NUM~ | 12 | 'Invalid dimensions' | - | ~TREXIO_ATTR_ALREADY_EXISTS~ | 13 | 'Attribute (num/str) already exists' | - | ~TREXIO_DSET_ALREADY_EXISTS~ | 14 | 'Dataset already exists' | - | ~TREXIO_OPEN_ERROR~ | 15 | 'Error opening file' | - | ~TREXIO_LOCK_ERROR~ | 16 | 'Error locking file' | - | ~TREXIO_UNLOCK_ERROR~ | 17 | 'Error unlocking file' | - | ~TREXIO_FILE_ERROR~ | 18 | 'Invalid file handle' | - | ~TREXIO_GROUP_READ_ERROR~ | 19 | 'Error reading group' | - | ~TREXIO_GROUP_WRITE_ERROR~ | 20 | 'Error writing group' | - | ~TREXIO_ELEM_READ_ERROR~ | 21 | 'Error reading element' | - | ~TREXIO_ELEM_WRITE_ERROR~ | 22 | 'Error writing element' | - | ~TREXIO_UNSAFE_ARRAY_DIM~ | 23 | 'Access to memory beyond allocated' | - | ~TREXIO_INVALID_STR_LEN~ | 30 | 'Invalid max_str_len' | + | Macro | Code | Description | + |------------------------------+------+----------------------------------------| + | ~TREXIO_FAILURE~ | -1 | 'Unknown failure' | + | ~TREXIO_SUCCESS~ | 0 | 'Success' | + | ~TREXIO_INVALID_ARG_1~ | 1 | 'Invalid argument 1' | + | ~TREXIO_INVALID_ARG_2~ | 2 | 'Invalid argument 2' | + | ~TREXIO_INVALID_ARG_3~ | 3 | 'Invalid argument 3' | + | ~TREXIO_INVALID_ARG_4~ | 4 | 'Invalid argument 4' | + | ~TREXIO_INVALID_ARG_5~ | 5 | 'Invalid argument 5' | + | ~TREXIO_END~ | 6 | 'End of file' | + | ~TREXIO_READONLY~ | 7 | 'Read-only file' | + | ~TREXIO_ERRNO~ | 8 | strerror(errno) | + | ~TREXIO_INVALID_ID~ | 9 | 'Invalid ID' | + | ~TREXIO_ALLOCATION_FAILED~ | 10 | 'Allocation failed' | + | ~TREXIO_HAS_NOT~ | 11 | 'Element absent' | + | ~TREXIO_INVALID_NUM~ | 12 | 'Invalid dimensions' | + | ~TREXIO_ATTR_ALREADY_EXISTS~ | 13 | 'Attribute already exists' | + | ~TREXIO_DSET_ALREADY_EXISTS~ | 14 | 'Dataset already exists' | + | ~TREXIO_OPEN_ERROR~ | 15 | 'Error opening file' | + | ~TREXIO_LOCK_ERROR~ | 16 | 'Error locking file' | + | ~TREXIO_UNLOCK_ERROR~ | 17 | 'Error unlocking file' | + | ~TREXIO_FILE_ERROR~ | 18 | 'Invalid file handle' | + | ~TREXIO_GROUP_READ_ERROR~ | 19 | 'Error reading group' | + | ~TREXIO_GROUP_WRITE_ERROR~ | 20 | 'Error writing group' | + | ~TREXIO_ELEM_READ_ERROR~ | 21 | 'Error reading element' | + | ~TREXIO_ELEM_WRITE_ERROR~ | 22 | 'Error writing element' | + | ~TREXIO_UNSAFE_ARRAY_DIM~ | 23 | 'Access to memory beyond allocated' | + | ~TREXIO_ATTR_MISSING~ | 24 | 'Attribute does not exist in the file' | + | ~TREXIO_DSET_MISSING~ | 25 | 'Dataset does not exist in the file' | + | ~TREXIO_INVALID_STR_LEN~ | 30 | 'Invalid max_str_len' | # We need to force Emacs not to indent the Python code: # -*- org-src-preserve-indentation: t @@ -241,6 +243,8 @@ return '\n'.join(result) #define TREXIO_ELEM_READ_ERROR ((trexio_exit_code) 21) #define TREXIO_ELEM_WRITE_ERROR ((trexio_exit_code) 22) #define TREXIO_UNSAFE_ARRAY_DIM ((trexio_exit_code) 23) + #define TREXIO_ATTR_MISSING ((trexio_exit_code) 24) + #define TREXIO_DSET_MISSING ((trexio_exit_code) 25) #define TREXIO_INVALID_STR_LEN ((trexio_exit_code) 30) #+end_src @@ -270,6 +274,8 @@ return '\n'.join(result) integer(trexio_exit_code), parameter :: TREXIO_ELEM_READ_ERROR = 21 integer(trexio_exit_code), parameter :: TREXIO_ELEM_WRITE_ERROR = 22 integer(trexio_exit_code), parameter :: TREXIO_UNSAFE_ARRAY_DIM = 23 + integer(trexio_exit_code), parameter :: TREXIO_ATTR_MISSING = 24 + integer(trexio_exit_code), parameter :: TREXIO_DSET_MISSING = 25 integer(trexio_exit_code), parameter :: TREXIO_INVALID_STR_LEN = 30 #+end_src @@ -300,6 +306,8 @@ return '\n'.join(result) TREXIO_ELEM_READ_ERROR = 21 TREXIO_ELEM_WRITE_ERROR = 22 TREXIO_UNSAFE_ARRAY_DIM = 23 + TREXIO_ATTR_MISSING = 24 + TREXIO_DSET_MISSING = 25 TREXIO_INVALID_STR_LEN = 30 #+end_src :END: @@ -1136,6 +1144,7 @@ trexio_exit_code trexio_read_$group_num$_64 (trexio_t* const file, int64_t* const num) { if (file == NULL) return TREXIO_INVALID_ARG_1; + if (trexio_has_$group_num$(file) != TREXIO_SUCCESS) return TREXIO_ATTR_MISSING; uint64_t u_num = 0; trexio_exit_code rc = TREXIO_GROUP_READ_ERROR; @@ -1196,6 +1205,7 @@ trexio_exit_code trexio_read_$group_num$_32 (trexio_t* const file, int32_t* const num) { if (file == NULL) return TREXIO_INVALID_ARG_1; + if (trexio_has_$group_num$(file) != TREXIO_SUCCESS) return TREXIO_ATTR_MISSING; uint64_t u_num = 0; trexio_exit_code rc = TREXIO_GROUP_READ_ERROR; @@ -1477,6 +1487,7 @@ trexio_read_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* co if (file == NULL) return TREXIO_INVALID_ARG_1; if ($group_dset$ == NULL) return TREXIO_INVALID_ARG_2; + if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; trexio_exit_code rc; int64_t $group_dset_dim$ = 0; @@ -1601,6 +1612,7 @@ trexio_read_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_single$* co if (file == NULL) return TREXIO_INVALID_ARG_1; if ($group_dset$ == NULL) return TREXIO_INVALID_ARG_2; + if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; trexio_exit_code rc; int64_t $group_dset_dim$ = 0; @@ -1758,6 +1770,7 @@ trexio_read_safe_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_single if (file == NULL) return TREXIO_INVALID_ARG_1; if (dset_out == NULL) return TREXIO_INVALID_ARG_2; + if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; <> @@ -1792,6 +1805,7 @@ trexio_read_safe_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double if (file == NULL) return TREXIO_INVALID_ARG_1; if (dset_out == NULL) return TREXIO_INVALID_ARG_2; + if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; <> @@ -2266,6 +2280,7 @@ trexio_read_$group_dset$_low (trexio_t* const file, char* dset_out, const uint32 if (file == NULL) return TREXIO_INVALID_ARG_1; if (dset_out == NULL) return TREXIO_INVALID_ARG_2; if (max_str_len <= 0) return TREXIO_INVALID_ARG_3; + if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; trexio_exit_code rc; int64_t $group_dset_dim$ = 0; @@ -2305,6 +2320,7 @@ trexio_read_$group_dset$ (trexio_t* const file, char** dset_out, const uint32_t if (file == NULL) return TREXIO_INVALID_ARG_1; if (dset_out == NULL) return TREXIO_INVALID_ARG_2; if (max_str_len <= 0) return TREXIO_INVALID_ARG_3; + if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; assert(file->back_end < TREXIO_INVALID_BACK_END); @@ -2684,6 +2700,7 @@ trexio_read_$group_str$ (trexio_t* const file, char* const str_out, const uint32 if (file == NULL) return TREXIO_INVALID_ARG_1; if (str_out == NULL) return TREXIO_INVALID_ARG_2; if (max_str_len <= 0) return TREXIO_INVALID_ARG_3; + if (trexio_has_$group_str$(file) != TREXIO_SUCCESS) return TREXIO_ATTR_MISSING; switch (file->back_end) { From 39c7587b5ece750c8793a411ef7a360881aa112e Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 10 Sep 2021 13:36:51 +0200 Subject: [PATCH 091/107] more portable setup of environment variables for install_pytrexio.sh --- Makefile.am | 3 ++- python/install_pytrexio.sh | 29 ++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/Makefile.am b/Makefile.am index 3f81463..8ae0413 100644 --- a/Makefile.am +++ b/Makefile.am @@ -160,6 +160,7 @@ $(HTML_FILES): docs/index.html SWIG = @SWIG@ HDF5_LDFLAGS = @HDF5_LDFLAGS@ HDF5_CFLAGS = @HDF5_CFLAGS@ +HDF5_CPPFLAGS = @HDF5_CPPFLAGS@ if TREXIO_DEVEL @@ -203,7 +204,7 @@ python-test: $(TEST_PY) python-install: $(pytrexio_py) $(setup_py) $(setup_cfg) cd python && \ - ./install_pytrexio.sh $(HDF5_CFLAGS) $(HDF5_LDFLAGS) + ./install_pytrexio.sh $(HDF5_CFLAGS) $(HDF5_CPPFLAGS) $(HDF5_LDFLAGS) python-sdist: $(pytrexio_py) $(setup_py) $(setup_cfg) cd python && \ diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index 81cde9f..bdcad50 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -3,17 +3,40 @@ set -x set -e -H5_CFLAGS_LOCAL=$1 -H5_LDFLAGS_LOCAL=$2 +# the parser below is needed when ./configure outputs several HDF5-related flags, which are then provided as input arguments to this script +for arg in "$@" +do + if [[ $arg == "-L"* ]] && [[ $arg == *"hdf5"* ]]; then + H5_LDFLAGS_LOCAL=$arg + elif [[ $arg == "-I"* ]] && [[ $arg == *"hdf5"* ]]; then + H5_CFLAGS_LOCAL=$arg + fi +done +# check that both variables are set +if [[ -z ${H5_LDFLAGS_LOCAL} ]] || [[ -z ${H5_CFLAGS_LOCAL} ]]; then + echo "Paths to the HDF5 installation are not found. pkgconfig Python package will try to detect them." +else + # additional explicit export was needed on MacOS + export H5_LDFLAGS=${H5_LDFLAGS_LOCAL} + export H5_CFLAGS=${H5_CFLAGS_LOCAL} +fi # Install/upgrade packages required for the installation python3 -m pip install --upgrade setuptools wheel twine python3 -m pip install -r requirements.txt +# export NUMPY_INCLUDEDIR environment variable needed for the proper setup +source tools/set_NUMPY_INCLUDEDIR.sh + +if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then + echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)." + exit 1 +fi + # Create build directory and compile extension files (*.c) # --no-user-cfg disables custom .cfg files of the user machine, so that only setup.cfg is used -H5_LDFLAGS=${H5_LDFLAGS_LOCAL} H5_CFLAGS=${H5_CFLAGS_LOCAL} python3 -s setup.py --no-user-cfg build +python3 -s setup.py --no-user-cfg build # Local inplace build of the .so module with SWIG-produced pytrexio_wrap.c (from the SWIG documentation) #python3 setup.py build_ext --inplace --swig-opts="-modern" From ab3f56c02350a97a9e994bd3b9ca428781ae3899 Mon Sep 17 00:00:00 2001 From: q-posev Date: Fri, 10 Sep 2021 14:02:10 +0200 Subject: [PATCH 092/107] remove jupyter from requirements.txt --- python/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index 769a89d..e7bbedf 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,4 +1,3 @@ setuptools>=42 pkgconfig numpy -jupyter From 97f1d3b723934788d1bfdd1910ea874672112950 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sun, 12 Sep 2021 12:27:08 +0200 Subject: [PATCH 093/107] better documentation --- python/README.md | 29 ++++++++++++++++++----------- python/install_pytrexio.sh | 7 +++++-- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/python/README.md b/python/README.md index 8decbf9..01d73a3 100644 --- a/python/README.md +++ b/python/README.md @@ -1,18 +1,18 @@ -## TREXIO Python API +# TREXIO Python API TREXIO provides a Python API, which enables interactive calls to the library. It facilitates the development of interfaces between different codes and can be used to convert data from one input/output file format into another. -### Requirements +## Requirements - python3 (>= 3.6) - numpy - C compiler (gcc/icc) -### Installation from PyPI +## Installation from PyPI Run `pip3 install trexio` @@ -20,18 +20,19 @@ Run `pip3 install trexio` For more details, see the corresponding part of the [Python documentation](https://docs.python.org/3/library/venv.html#creating-virtual-environments). -### Additional requirements (for installation from source) +## Additional requirements (for installation from source) - HDF5 library (>= 1.8) -- pkgconfig (TODO: CHECK THIS by installing wheels) +- pkgconfig (Python package) -### Installation from source +## Installation from source 1. Download the latest source code distribution (in `.tar.gz` format) of the TREXIO Python API 2. Unpack and `cd` in the output directory -3. Run `pip3 install -r requirements.txt` (this installs all python dependencies) -4. Export custom environment variables needed for the installation. Steps 1 and 2 can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options). +3. Run `pip3 install -r requirements.txt` (this installs all required python dependencies) +4. Export custom environment variables needed for the installation following the procedure below and replacing `/path/to/hdf5/` with your paths. +Steps (i) and (ii) can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options). 1. `export H5_CFLAGS=-I/path/to/hdf5/include` 2. `export H5_LDFLAGS=-L/path/to/hdf5/lib` 3. `source tools/set_NUMPY_INCLUDEDIR.sh` @@ -40,8 +41,14 @@ For more details, see the corresponding part of the [Python documentation](https You are ready to go! +**Note:** +installation based on `pip` compiles its own C extension (shared library) called `pytrexio`. +This extension is built from the TREXIO source files coupled to the wrapper code generated by [SWIG](http://www.swig.org/). +The compiler options during such installation may differ from the ones used to compile the primary TREXIO API in C. +Furthermore, custom compiler flags provided to `./configure` or `make` are not applicable to the Python API. -### Examples + +## Examples An interactive Jupyter notebook called `tutorial_benzene.ipynb` is provided in the `examples` directory. It demonstrates some basic use cases of the TREXIO library in general and of the Python API in particular. @@ -49,14 +56,14 @@ It demonstrates some basic use cases of the TREXIO library in general and of the Jupyter can be installed using `pip install jupyter`. If you are not familiar with it, feel free to consult the [Jupyter documentation](https://jupyter-notebook.readthedocs.io/en/stable/notebook.html). -#### Running the notebook +### Running the notebook The example notebook can be launched using the following command: `jupyter notebook tutorial_benzene.ipynb` -#### Additional steps needed to run a custom virtual environment in Jupyter notebooks +### Additional steps needed to run a custom virtual environment in Jupyter notebooks In some cases, it may happen that the Jupyter kernels in the activated virtual environment (e.g. `myvenv`) still point to the system-wide python binaries and not to the environment ones. This will result in `ImportError` when importing `trexio` in the notebook cell. In order to avoid this, the `myvenv` has to be installed as an additional kernel. diff --git a/python/install_pytrexio.sh b/python/install_pytrexio.sh index bdcad50..1b7c314 100755 --- a/python/install_pytrexio.sh +++ b/python/install_pytrexio.sh @@ -15,9 +15,12 @@ done # check that both variables are set if [[ -z ${H5_LDFLAGS_LOCAL} ]] || [[ -z ${H5_CFLAGS_LOCAL} ]]; then - echo "Paths to the HDF5 installation are not found. pkgconfig Python package will try to detect them." + if [[ -z ${H5_LDFLAGS} ]] || [[ -z ${H5_CFLAGS} ]]; then + echo "Paths to the HDF5 installation are not provided. pkgconfig will try to detect them." + else + echo "Using exported H5_LDFLAGS and H5_CFLAGS environment variables." + fi else - # additional explicit export was needed on MacOS export H5_LDFLAGS=${H5_LDFLAGS_LOCAL} export H5_CFLAGS=${H5_CFLAGS_LOCAL} fi From 12dd1fd8dce270021bc6bcd2116622db89ae170c Mon Sep 17 00:00:00 2001 From: q-posev Date: Sun, 12 Sep 2021 13:25:03 +0200 Subject: [PATCH 094/107] apply some suggestions from cppcheck --- src/templates_front/templator_front.org | 8 +++++--- src/templates_hdf5/templator_hdf5.org | 7 +++++++ src/templates_text/templator_text.org | 6 ++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 371e819..0a98fba 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -2387,10 +2387,12 @@ trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const char* tmp_str = CALLOC(dims[0]*(max_str_len+1), char); if (tmp_str == NULL) return TREXIO_ALLOCATION_FAILED; char** dset_str = CALLOC(dims[0], char*); - if (dset_str == NULL) return TREXIO_ALLOCATION_FAILED; + if (dset_str == NULL) { + FREE(tmp_str); + return TREXIO_ALLOCATION_FAILED; + } char* pch; - size_t pch_len; /* parse the string using strtok */ for(uint64_t i=0; i max_str_len) { FREE(dset_str[0]); diff --git a/src/templates_hdf5/templator_hdf5.org b/src/templates_hdf5/templator_hdf5.org index 6d16cf9..90a9881 100644 --- a/src/templates_hdf5/templator_hdf5.org +++ b/src/templates_hdf5/templator_hdf5.org @@ -536,15 +536,22 @@ trexio_hdf5_write_$group_dset$ (trexio_t* const file, const char** $group_dset$, /* we are going to write variable-length strings */ hid_t memtype = H5Tcopy (H5T_C_S1); + if (memtype <= 0) return TREXIO_INVALID_ID; + status = H5Tset_size (memtype, H5T_VARIABLE); + if (status < 0) return TREXIO_FAILURE; if ( H5LTfind_dataset(f->$group$_group, $GROUP_DSET$_NAME) != 1 ) { /* code to create dataset */ hid_t filetype = H5Tcopy (H5T_FORTRAN_S1); + if (filetype <= 0) return TREXIO_INVALID_ID; + status = H5Tset_size (filetype, H5T_VARIABLE); + if (status < 0) return TREXIO_FAILURE; hid_t dspace = H5Screate_simple( (int) rank, (const hsize_t*) dims, NULL); + if (dspace <= 0) return TREXIO_INVALID_ID; dset_id = H5Dcreate (f->$group$_group, $GROUP_DSET$_NAME, filetype, dspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); diff --git a/src/templates_text/templator_text.org b/src/templates_text/templator_text.org index 27e0af7..4fcc892 100644 --- a/src/templates_text/templator_text.org +++ b/src/templates_text/templator_text.org @@ -484,7 +484,6 @@ trexio_text_read_$group$ (trexio_text_t* const file) char* tmp_$group_dset$; if(size_$group_dset$ != 0) tmp_$group_dset$ = CALLOC(size_$group_dset$*32, char); - size_t tmp_$group_dset$_len = 0; for (uint64_t i=0 ; i$group_dset$[i] = tmp_$group_dset$; /* conventional fcanf with "%s" only return the string before the first space character @@ -500,7 +499,7 @@ trexio_text_read_$group$ (trexio_text_t* const file) return NULL; } - tmp_$group_dset$_len = strlen($group$->$group_dset$[i]); + size_t tmp_$group_dset$_len = strlen($group$->$group_dset$[i]); tmp_$group_dset$ += tmp_$group_dset$_len + 1; } // END REPEAT GROUP_DSET_STR @@ -851,9 +850,8 @@ trexio_text_write_$group_dset$ (trexio_t* const file, const char** dset, const u char* tmp_str = CALLOC(dims[0]*32 + 1, char); if (tmp_str == NULL) return TREXIO_ALLOCATION_FAILED; - size_t tmp_len = 0; for (uint64_t i=0 ; i$group_dset$[i] = tmp_str; strncpy(tmp_str, dset[i], tmp_len); tmp_str += tmp_len + 1; From d0608084674f55110788282d4f810b150bf43afe Mon Sep 17 00:00:00 2001 From: q-posev Date: Sun, 12 Sep 2021 13:36:24 +0200 Subject: [PATCH 095/107] do not reveal paths to trexio (security concerns) --- src/templates_front/templator_front.org | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 0a98fba..89e7880 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -100,9 +100,10 @@ PYTREXIO_MAX_STR_LENGTH = 2048 # setuptools do not assign __version__ variable to the trexio package, so we set it manually from os import path __trexio_path__ = path.dirname(path.abspath(__file__)) -__pytrexio_path__ = path.join(__trexio_path__, 'pytrexio') -with open(path.join(__pytrexio_path__, '_version.py')) as version_file: +with open(path.join(path.join(__trexio_path__, 'pytrexio'), '_version.py')) as version_file: __version__ = version_file.read().split('"')[1] + +__trexio_path__ = None #+end_src * Coding conventions From 77e3f1ac11f47abbcf87e389eb251834dedca839 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sun, 12 Sep 2021 14:24:50 +0200 Subject: [PATCH 096/107] print UserWarning instead of raising Error for an attempt to overwrite data following suggestion of Vijay G.C. --- python/test/test_api.py | 6 +++-- src/templates_front/templator_front.org | 33 ++++++++++++++++++++----- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index 104a7f6..94dc722 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -60,10 +60,12 @@ try: except: raise +import warnings +warnings.filterwarnings("error") try: trexio.write_nucleus_num(test_file, nucleus_num*2) -except trexio.Error: - print("Attempt to overwrite nucleus_num: checked.") +except UserWarning: + print("Attemp to overwrite nucleus_num: checked.") # initialize charge arrays as a list and convert it to numpy array charges = [6., 6., 6., 6., 6., 6., 1., 1., 1., 1., 1., 1.] diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 89e7880..4fa5497 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -392,7 +392,7 @@ return '\n'.join(result) return "Invalid dimensions"; break; case TREXIO_ATTR_ALREADY_EXISTS: - return "Attribute (num/str) already exists"; + return "Attribute already exists"; break; case TREXIO_DSET_ALREADY_EXISTS: return "Dataset already exists"; @@ -424,6 +424,12 @@ return '\n'.join(result) case TREXIO_UNSAFE_ARRAY_DIM: return "Access to memory beyond allocated"; break; + case TREXIO_ATTR_MISSING: + return "Attribute does not exist in the file"; + break; + case TREXIO_DSET_MISSING: + return "Dataset does not exist in the file"; + break; case TREXIO_INVALID_STR_LEN: return "Invalid max_str_len"; break; @@ -1404,10 +1410,13 @@ def write_$group_num$(trexio_file, num_w: int) -> None: try: rc = pytr.trexio_write_$group_num$(trexio_file.pytrexio_s, num_w) if rc != TREXIO_SUCCESS: - raise Error(rc) + if rc == TREXIO_ATTR_ALREADY_EXISTS: + import warnings + warnings.warn(string_of_error(rc)) + else: + raise Error(rc) except: raise - #+end_src #+begin_src python :tangle read_num_front.py @@ -2034,7 +2043,11 @@ def write_$group_dset$(trexio_file, dset_w) -> None: rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_w) if rc != TREXIO_SUCCESS: - raise Error(rc) + if rc == TREXIO_DSET_ALREADY_EXISTS: + import warnings + warnings.warn(string_of_error(rc)) + else: + raise Error(rc) #+end_src #+begin_src python :tangle read_dset_data_front.py @@ -2619,7 +2632,11 @@ def write_$group_dset$(trexio_file, dset_w: list) -> None: rc = pytr.trexio_write_$group_dset$(trexio_file.pytrexio_s, dset_w, max_str_length) if rc != TREXIO_SUCCESS: - raise Error(rc) + if rc == TREXIO_DSET_ALREADY_EXISTS: + import warnings + warnings.warn(string_of_error(rc)) + else: + raise Error(rc) except: raise @@ -2880,7 +2897,11 @@ def write_$group_str$(trexio_file, str_w: str) -> None: rc = pytr.trexio_write_$group_str$(trexio_file.pytrexio_s, str_w, max_str_length) if rc != TREXIO_SUCCESS: - raise Error(rc) + if rc == TREXIO_ATTR_ALREADY_EXISTS: + import warnings + warnings.warn(string_of_error(rc)) + else: + raise Error(rc) except: raise #+end_src From 2bd4ef9bfd506c65596792f45b532be7870a0d26 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 13 Sep 2021 11:29:00 +0200 Subject: [PATCH 097/107] add has_ functions to the Python API --- python/test/test_api.py | 25 ++++-- src/templates_front/templator_front.org | 110 ++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 7 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index 94dc722..756277d 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -41,7 +41,6 @@ except: # create TREXIO file and open it for writing -#test_file = trexio.open(output_filename, 'w', TEST_TREXIO_BACKEND) test_file = trexio.File(output_filename, mode='w', back_end=TEST_TREXIO_BACKEND) # Print docstring of the trexio.open function @@ -65,7 +64,7 @@ warnings.filterwarnings("error") try: trexio.write_nucleus_num(test_file, nucleus_num*2) except UserWarning: - print("Attemp to overwrite nucleus_num: checked.") + print("Attempt to overwrite nucleus_num: checked.") # initialize charge arrays as a list and convert it to numpy array charges = [6., 6., 6., 6., 6., 6., 1., 1., 1., 1., 1., 1.] @@ -127,22 +126,28 @@ labels = [ trexio.write_nucleus_label(test_file,labels) # close TREXIO file -# [TODO:] this functional call is no longer needed as we introduced TREXIO_File class which has a desctructor that closes the file +# this call is no longer needed as we introduced TREXIO_File class which has a desctructor that closes the file #trexio.close(test_file) -# [TODO:] without calling destructor on test_file the TREXIO_FILE is not getting created and the data is not written when using TEXT back end. This, the user still has to explicitly call destructor on test_file object instead -# trexio.close function. This is only an issue when the data is getting written and read in the same session (e.g. in Jupyter notebook) +# without calling destructor on test_file the TREXIO_FILE is not getting created and the data is not written when using TEXT back end. +# This, the user still has to explicitly call destructor on test_file object instead of the trexio.close function. +# This is only an issue when the data is getting written and read in the same session (e.g. in Jupyter notebook) del test_file - #==========================================================# #============ READ THE DATA FROM THE TEST FILE ============# #==========================================================# # open previously created TREXIO file, now in 'read' mode -#test_file2 = trexio.open(output_filename, 'r', TEST_TREXIO_BACKEND) test_file2 = trexio.File(output_filename, 'r', TEST_TREXIO_BACKEND) +# check for existence of some of the previously written variables +assert trexio.has_nucleus_num +assert trexio.has_nucleus_charge +assert trexio.has_nucleus_coord +assert trexio.has_nucleus_label +assert trexio.has_nucleus_point_group + # read nucleus_num from file rnum = trexio.read_nucleus_num(test_file2) assert rnum==nucleus_num @@ -194,6 +199,12 @@ for i in range(nucleus_num): rpoint_group = trexio.read_nucleus_point_group(test_file2) assert rpoint_group==point_group +# another way to read only if the variable exists +if trexio.has_mo_num(test_file2): + rmo_num = trexio.read_mo_num(test_file2) +else: + print("Not reading the non-existing variable mo_num.") + # close TREXIO file #trexio.close(test_file2) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 4fa5497..7deba75 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -1444,6 +1444,33 @@ def read_$group_num$(trexio_file) -> int: return num_r #+end_src + #+begin_src python :tangle has_num_front.py +def has_$group_num$(trexio_file) -> bool: + """Check that $group_num$ variable exists in the TREXIO file. + + Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. + + Returns: + True if the variable exists, False otherwise + + Raises: + - Exception from trexio.Error class if TREXIO return code ~rc~ is TREXIO_FAILURE and prints the error message using string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + + try: + rc = pytr.trexio_has_$group_num$(trexio_file.pytrexio_s) + if rc == TREXIO_FAILURE: + raise Error(rc) + except: + raise + + if rc == TREXIO_SUCCESS: + return True + else: + return False + #+end_src + ** Templates for front end has/read/write a dataset of numerical data This section concerns API calls related to datasets. @@ -2148,6 +2175,34 @@ def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): else: return dset_64 #+end_src + + #+begin_src python :tangle has_dset_data_front.py +def has_$group_dset$(trexio_file) -> bool: + """Check that $group_dset$ variable exists in the TREXIO file. + + Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. + + Returns: + True if the variable exists, False otherwise + + Raises: + - Exception from trexio.Error class if TREXIO return code ~rc~ is TREXIO_FAILURE and prints the error message using string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + + try: + rc = pytr.trexio_has_$group_dset$(trexio_file.pytrexio_s) + if rc == TREXIO_FAILURE: + raise Error(rc) + except: + raise + + if rc == TREXIO_SUCCESS: + return True + else: + return False + #+end_src + ** Sparse data structures Sparse data structures are used typically for large tensors such as @@ -2693,6 +2748,34 @@ def read_$group_dset$(trexio_file, dim = None) -> list: return dset_2d_r #+end_src + + #+begin_src python :tangle has_dset_str_front.py +def has_$group_dset$(trexio_file) -> bool: + """Check that $group_dset$ variable exists in the TREXIO file. + + Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. + + Returns: + True if the variable exists, False otherwise + + Raises: + - Exception from trexio.Error class if TREXIO return code ~rc~ is TREXIO_FAILURE and prints the error message using string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + + try: + rc = pytr.trexio_has_$group_dset$(trexio_file.pytrexio_s) + if rc == TREXIO_FAILURE: + raise Error(rc) + except: + raise + + if rc == TREXIO_SUCCESS: + return True + else: + return False + #+end_src + ** Templates for front end has/read/write a single string attribute *** Introduction @@ -2931,6 +3014,33 @@ def read_$group_str$(trexio_file) -> str: return str_r #+end_src + + #+begin_src python :tangle has_attr_str_front.py +def has_$group_str$(trexio_file) -> bool: + """Check that $group_str$ variable exists in the TREXIO file. + + Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. + + Returns: + True if the variable exists, False otherwise + + Raises: + - Exception from trexio.Error class if TREXIO return code ~rc~ is TREXIO_FAILURE and prints the error message using string_of_error. + - Exception from some other error (e.g. RuntimeError). + """ + + try: + rc = pytr.trexio_has_$group_str$(trexio_file.pytrexio_s) + if rc == TREXIO_FAILURE: + raise Error(rc) + except: + raise + + if rc == TREXIO_SUCCESS: + return True + else: + return False + #+end_src * Fortran helper/wrapper functions From 1589995775400c9e498391eebe45ad0c490d78f9 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 13 Sep 2021 11:46:31 +0200 Subject: [PATCH 098/107] do not use warnings for an attempt to overwrite in the Python API --- python/test/test_api.py | 8 ++------ src/templates_front/templator_front.org | 24 ++++-------------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index 756277d..ae1a1ab 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -38,8 +38,6 @@ except: #============ WRITE THE DATA IN THE TEST FILE ============# #=========================================================# - - # create TREXIO file and open it for writing test_file = trexio.File(output_filename, mode='w', back_end=TEST_TREXIO_BACKEND) @@ -59,11 +57,9 @@ try: except: raise -import warnings -warnings.filterwarnings("error") try: trexio.write_nucleus_num(test_file, nucleus_num*2) -except UserWarning: +except trexio.Error: print("Attempt to overwrite nucleus_num: checked.") # initialize charge arrays as a list and convert it to numpy array @@ -160,7 +156,7 @@ np.testing.assert_array_almost_equal(rcharges_np, charges_np, decimal=8) # unsafe call to read_safe should fail with error message corresponding to TREXIO_UNSAFE_ARRAY_DIM try: rcharges_fail = trexio.read_nucleus_charge(test_file2, dim=nucleus_num*5) -except Exception: +except trexio.Error: print("Unsafe call to safe API: checked") # safe call to read array of int values (nuclear indices) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 7deba75..37fefb2 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -1410,11 +1410,7 @@ def write_$group_num$(trexio_file, num_w: int) -> None: try: rc = pytr.trexio_write_$group_num$(trexio_file.pytrexio_s, num_w) if rc != TREXIO_SUCCESS: - if rc == TREXIO_ATTR_ALREADY_EXISTS: - import warnings - warnings.warn(string_of_error(rc)) - else: - raise Error(rc) + raise Error(rc) except: raise #+end_src @@ -2070,11 +2066,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_w) if rc != TREXIO_SUCCESS: - if rc == TREXIO_DSET_ALREADY_EXISTS: - import warnings - warnings.warn(string_of_error(rc)) - else: - raise Error(rc) + raise Error(rc) #+end_src #+begin_src python :tangle read_dset_data_front.py @@ -2687,11 +2679,7 @@ def write_$group_dset$(trexio_file, dset_w: list) -> None: rc = pytr.trexio_write_$group_dset$(trexio_file.pytrexio_s, dset_w, max_str_length) if rc != TREXIO_SUCCESS: - if rc == TREXIO_DSET_ALREADY_EXISTS: - import warnings - warnings.warn(string_of_error(rc)) - else: - raise Error(rc) + raise Error(rc) except: raise @@ -2980,11 +2968,7 @@ def write_$group_str$(trexio_file, str_w: str) -> None: rc = pytr.trexio_write_$group_str$(trexio_file.pytrexio_s, str_w, max_str_length) if rc != TREXIO_SUCCESS: - if rc == TREXIO_ATTR_ALREADY_EXISTS: - import warnings - warnings.warn(string_of_error(rc)) - else: - raise Error(rc) + raise Error(rc) except: raise #+end_src From d04106f72d8d8753b9c0a96a1d672fc99ea94dff Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 13 Sep 2021 13:35:45 +0200 Subject: [PATCH 099/107] max_str_len should be int32_t to properly check for negative values --- src/pytrexio.i | 2 +- src/templates_front/templator_front.org | 40 ++++++++++++------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/pytrexio.i b/src/pytrexio.i index f7fe29e..8d3c174 100644 --- a/src/pytrexio.i +++ b/src/pytrexio.i @@ -46,7 +46,7 @@ /* This enables read of single string attributes with pre-defined max_str_len for Python we pre-define max_str_len = PYTREXIO_MAX_STR_LENGTH everywhere for simplicity */ -%cstring_output_maxsize(char* const str_out, const uint32_t max_str_len); +%cstring_output_maxsize(char* const str_out, const int32_t max_str_len); /* [WIP] TREXIO back ends and exit codes can be redefined in the SWIG target language using %ignore and further #define statements (instead of disabling the type cast in the trexio.h file) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 37fefb2..312e494 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -2327,15 +2327,15 @@ trexio_read_chunk_ao_2e_int_eri_value_64(trexio_t* const file, #+begin_src c :tangle hrw_dset_str_front.h :exports none trexio_exit_code trexio_has_$group_dset$(trexio_t* const file); -trexio_exit_code trexio_read_$group_dset$_low(trexio_t* const file, char* dset_out, const uint32_t max_str_len); -trexio_exit_code trexio_write_$group_dset$_low(trexio_t* const file, const char* dset_in, const uint32_t max_str_len); -trexio_exit_code trexio_read_$group_dset$(trexio_t* const file, char** dset_out, const uint32_t max_str_len); -trexio_exit_code trexio_write_$group_dset$(trexio_t* const file, const char** dset_in, const uint32_t max_str_len); +trexio_exit_code trexio_read_$group_dset$_low(trexio_t* const file, char* dset_out, const int32_t max_str_len); +trexio_exit_code trexio_write_$group_dset$_low(trexio_t* const file, const char* dset_in, const int32_t max_str_len); +trexio_exit_code trexio_read_$group_dset$(trexio_t* const file, char** dset_out, const int32_t max_str_len); +trexio_exit_code trexio_write_$group_dset$(trexio_t* const file, const char** dset_in, const int32_t max_str_len); #+end_src #+begin_src c :tangle read_dset_str_front.c trexio_exit_code -trexio_read_$group_dset$_low (trexio_t* const file, char* dset_out, const uint32_t max_str_len) +trexio_read_$group_dset$_low (trexio_t* const file, char* dset_out, const int32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; @@ -2358,11 +2358,11 @@ trexio_read_$group_dset$_low (trexio_t* const file, char* dset_out, const uint32 switch (file->back_end) { case TREXIO_TEXT: - return trexio_text_read_$group_dset$(file, dset_out, rank, dims, max_str_len); + return trexio_text_read_$group_dset$(file, dset_out, rank, dims, (uint32_t) max_str_len); break; case TREXIO_HDF5: - return trexio_hdf5_read_$group_dset$(file, dset_out, rank, dims, max_str_len); + return trexio_hdf5_read_$group_dset$(file, dset_out, rank, dims, (uint32_t) max_str_len); break; /* case TREXIO_JSON: @@ -2375,7 +2375,7 @@ trexio_read_$group_dset$_low (trexio_t* const file, char* dset_out, const uint32 } trexio_exit_code -trexio_read_$group_dset$ (trexio_t* const file, char** dset_out, const uint32_t max_str_len) +trexio_read_$group_dset$ (trexio_t* const file, char** dset_out, const int32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; @@ -2402,9 +2402,9 @@ trexio_read_$group_dset$ (trexio_t* const file, char** dset_out, const uint32_t return rc; } - char * pch; for (uint64_t i=0; i < (uint64_t) dset_dim; i++) { + char * pch; pch = i == 0 ? strtok(str_compiled, TREXIO_DELIM) : strtok(NULL, TREXIO_DELIM) ; if (pch == NULL) { FREE(str_compiled); @@ -2424,7 +2424,7 @@ trexio_read_$group_dset$ (trexio_t* const file, char** dset_out, const uint32_t #+begin_src c :tangle write_dset_str_front.c trexio_exit_code -trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const uint32_t max_str_len) +trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const int32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; @@ -2453,10 +2453,10 @@ trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const return TREXIO_ALLOCATION_FAILED; } - char* pch; /* parse the string using strtok */ for(uint64_t i=0; i max_str_len) { + if (pch_len > (size_t) max_str_len) { FREE(dset_str[0]); FREE(dset_str); return TREXIO_INVALID_STR_LEN; @@ -2503,7 +2503,7 @@ trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const } trexio_exit_code -trexio_write_$group_dset$ (trexio_t* const file, const char** dset_in, const uint32_t max_str_len) +trexio_write_$group_dset$ (trexio_t* const file, const char** dset_in, const int32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; @@ -2779,13 +2779,13 @@ def has_$group_dset$(trexio_file) -> bool: #+begin_src c :tangle hrw_attr_str_front.h :exports none trexio_exit_code trexio_has_$group_str$(trexio_t* const file); -trexio_exit_code trexio_read_$group_str$(trexio_t* const file, char* const str_out, const uint32_t max_str_len); -trexio_exit_code trexio_write_$group_str$(trexio_t* const file, const char* str, const uint32_t max_str_len); +trexio_exit_code trexio_read_$group_str$(trexio_t* const file, char* const str_out, const int32_t max_str_len); +trexio_exit_code trexio_write_$group_str$(trexio_t* const file, const char* str, const int32_t max_str_len); #+end_src #+begin_src c :tangle read_attr_str_front.c trexio_exit_code -trexio_read_$group_str$ (trexio_t* const file, char* const str_out, const uint32_t max_str_len) +trexio_read_$group_str$ (trexio_t* const file, char* const str_out, const int32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; @@ -2796,11 +2796,11 @@ trexio_read_$group_str$ (trexio_t* const file, char* const str_out, const uint32 switch (file->back_end) { case TREXIO_TEXT: - return trexio_text_read_$group_str$(file, str_out, max_str_len); + return trexio_text_read_$group_str$(file, str_out, (uint32_t) max_str_len); break; case TREXIO_HDF5: - return trexio_hdf5_read_$group_str$(file, str_out, max_str_len); + return trexio_hdf5_read_$group_str$(file, str_out, (uint32_t) max_str_len); break; /* case TREXIO_JSON: @@ -2815,7 +2815,7 @@ trexio_read_$group_str$ (trexio_t* const file, char* const str_out, const uint32 #+begin_src c :tangle write_attr_str_front.c trexio_exit_code -trexio_write_$group_str$ (trexio_t* const file, const char* str, const uint32_t max_str_len) +trexio_write_$group_str$ (trexio_t* const file, const char* str, const int32_t max_str_len) { if (file == NULL) return TREXIO_INVALID_ARG_1; @@ -2824,7 +2824,7 @@ trexio_write_$group_str$ (trexio_t* const file, const char* str, const uint32_t if (trexio_has_$group_str$(file) == TREXIO_SUCCESS) return TREXIO_ATTR_ALREADY_EXISTS; size_t len_write = strlen(str); - if (max_str_len < len_write) return TREXIO_INVALID_STR_LEN; + if ((size_t) max_str_len < len_write) return TREXIO_INVALID_STR_LEN; switch (file->back_end) { From 3b0c1a637d7c9b914ca7548fefa18b3604d79e2e Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 13 Sep 2021 14:54:00 +0200 Subject: [PATCH 100/107] more flexible build_manylinux_ bash script --- .../build_manylinux_wheels_py_36_37_38_39.sh | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/docker/build_manylinux_wheels_py_36_37_38_39.sh b/docker/build_manylinux_wheels_py_36_37_38_39.sh index d65872e..9c6390b 100755 --- a/docker/build_manylinux_wheels_py_36_37_38_39.sh +++ b/docker/build_manylinux_wheels_py_36_37_38_39.sh @@ -1,3 +1,5 @@ +#!/bin/bash + set -x set -e @@ -37,9 +39,24 @@ export H5_CFLAGS=-I/usr/local/include # alternatively: build wheel directly from developer-provided .tar.gz of TREXIO (generated with `python setup.py sdist`) # note: trexio-VERSION.tar.gz has to be in the root directory of the host machine +# process input: first argument is the name of the .tar.gz with the source code of the Python API +if [[ -z "$1" ]]; then + echo "Please specify the name of the TREXIO source code distribution (with .tar.gz suffix)" + exit 1 +fi + +TREXIO_SOURCE=${1} + +# remove prefix that ends with "-" +tmp=${TREXIO_SOURCE#*-} +# remove suffix that ends with ".tar.gz" +TR_VERSION=${tmp%.tar.gz*} + +echo "TREXIO VERSION:" ${TR_VERSION} + # unzip and enter the folder with TREXIO Python API -gzip -cd /tmp/trexio-0.1.0.tar.gz | tar xvf - -cd trexio-0.1.0 +gzip -cd /tmp/trexio-${TR_VERSION}.tar.gz | tar xvf - +cd trexio-${TR_VERSION} # create and activate a virtual environment based on CPython version 3.6 /opt/python/cp36-cp36m/bin/python3 -m venv --clear trexio-manylinux-py36 @@ -58,10 +75,10 @@ source tools/set_NUMPY_INCLUDEDIR.sh python3 setup.py bdist_wheel # use auditwheel from PyPA to repair all wheels and make them manylinux-compatible -auditwheel repair dist/trexio-0.1.0-cp36-cp36m-*.whl +auditwheel repair dist/trexio-${TR_VERSION}-cp36-cp36m-*.whl # install the produced manylinux wheel in the virtual environment -python3 -m pip install wheelhouse/trexio-0.1.0-cp36-cp36m-manylinux*.whl +python3 -m pip install wheelhouse/trexio-${TR_VERSION}-cp36-cp36m-manylinux*.whl # run test script cd test && python3 test_api.py && cd .. @@ -89,10 +106,10 @@ source tools/set_NUMPY_INCLUDEDIR.sh python3 setup.py bdist_wheel # use auditwheel from PyPA to repair all wheels and make them manylinux-compatible -auditwheel repair dist/trexio-0.1.0-cp37-cp37m-*.whl +auditwheel repair dist/trexio-${TR_VERSION}-cp37-cp37m-*.whl # install the produced manylinux wheel in the virtual environment -python3 -m pip install wheelhouse/trexio-0.1.0-cp37-cp37m-manylinux*.whl +python3 -m pip install wheelhouse/trexio-${TR_VERSION}-cp37-cp37m-manylinux*.whl # run test script cd test && python3 test_api.py && cd .. @@ -121,10 +138,10 @@ source tools/set_NUMPY_INCLUDEDIR.sh python3 setup.py bdist_wheel # use auditwheel from PyPA to repair all wheels and make them manylinux-compatible -auditwheel repair dist/trexio-0.1.0-cp38-cp38-*.whl +auditwheel repair dist/trexio-${TR_VERSION}-cp38-cp38-*.whl # install the produced manylinux wheel in the virtual environment -python3 -m pip install wheelhouse/trexio-0.1.0-cp38-cp38-manylinux*.whl +python3 -m pip install wheelhouse/trexio-${TR_VERSION}-cp38-cp38-manylinux*.whl # run test script cd test && python3 test_api.py && cd .. @@ -149,10 +166,10 @@ pip3 install --upgrade setuptools wheel auditwheel numpy python3 setup.py bdist_wheel # use auditwheel from PyPA to repair all wheels and make them manylinux-compatible -auditwheel repair dist/trexio-0.1.0-cp39-cp39-*.whl +auditwheel repair dist/trexio-${TR_VERSION}-cp39-cp39-*.whl # install the produced manylinux wheel in the virtual environment -python3 -m pip install wheelhouse/trexio-0.1.0-cp39-cp39-manylinux*.whl +python3 -m pip install wheelhouse/trexio-${TR_VERSION}-cp39-cp39-manylinux*.whl # run test script cd test && python3 test_api.py && cd .. @@ -163,3 +180,9 @@ rm -rf -- dist/ build/ trexio.egg-info/ # deactivate the current environment deactivate +# remove all virtual environments used to produce the wheels +rm -rf -- trexio-manylinux-py39 \ + trexio-manylinux-py38 \ + trexio-manylinux-py37 \ + trexio-manylinux-py36 + From bff44a8f649d7cfc57f2b5dad8fbc1af00722b16 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 13 Sep 2021 14:54:25 +0200 Subject: [PATCH 101/107] add gitignore --- docker/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docker/.gitignore diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 0000000..4aac024 --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +tmp* +trexio-* From 7c0b7a36ba34de196611d99548fd6a2abc80b908 Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 13 Sep 2021 17:20:51 +0200 Subject: [PATCH 102/107] update Tutorial section and add Binder badge --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 74fda47..e907abe 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![build](https://github.com/TREX-CoE/trexio/actions/workflows/actions.yml/badge.svg)](https://github.com/TREX-CoE/trexio/actions/workflows/actions.yml) ![GitHub release (latest by date)](https://img.shields.io/github/v/release/TREX-CoE/trexio) +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD) TREX library for efficient I/O. @@ -71,7 +72,9 @@ For more details regarding the installation and usage of the TREXIO Python API, ## Tutorial -**TODO** +TREXIO tutorials in Jupyter notebook format can be found in the +[corresponding GitHub repository](https://github.com/TREX-CoE/trexio-tutorials) +or on [Binder](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD). ## Technical documentation From 71f40069871cd121be4fce8581322f344f01f7fb Mon Sep 17 00:00:00 2001 From: q-posev Date: Mon, 13 Sep 2021 17:21:34 +0200 Subject: [PATCH 103/107] better README + Binder badge --- python/README.md | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/python/README.md b/python/README.md index 01d73a3..23bcc09 100644 --- a/python/README.md +++ b/python/README.md @@ -1,3 +1,6 @@ + +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD) + # TREXIO Python API TREXIO provides a Python API, which enables interactive calls to the library. @@ -14,9 +17,15 @@ can be used to convert data from one input/output file format into another. ## Installation from PyPI -Run `pip3 install trexio` +In short, you can run the following command: -**Note: we highly recommend to use virtual environments to avoid compatibility issues.** +`pip install trexio` + +However, it is good practice to first check for updates of the build-system packages. This can be achieved by running + +`python3 -m pip install --upgrade pip setuptools wheel` + +**Note: we highly recommend to use virtual environments to avoid compatibility issues and to improve reproducibility.** For more details, see the corresponding part of the [Python documentation](https://docs.python.org/3/library/venv.html#creating-virtual-environments). @@ -28,30 +37,31 @@ For more details, see the corresponding part of the [Python documentation](https ## Installation from source -1. Download the latest source code distribution (in `.tar.gz` format) of the TREXIO Python API -2. Unpack and `cd` in the output directory -3. Run `pip3 install -r requirements.txt` (this installs all required python dependencies) -4. Export custom environment variables needed for the installation following the procedure below and replacing `/path/to/hdf5/` with your paths. +1. Download the `trexio-.tar.gz` file with the latest Python API +2. `gzip -cd trexio-.tar.gz | tar xvf -` +3. `cd trexio-` +4. `pip3 install -r requirements.txt` (this installs all required python dependencies) +5. Export custom environment variables needed for the installation following the procedure below and replacing `/path/to/hdf5/` with your paths. Steps (i) and (ii) can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options). 1. `export H5_CFLAGS=-I/path/to/hdf5/include` 2. `export H5_LDFLAGS=-L/path/to/hdf5/lib` 3. `source tools/set_NUMPY_INCLUDEDIR.sh` -5. Run `pip3 install .` (this installs `trexio` in your environment) -6. Run `cd test && python3 test_api.py` (this executes several tests that verify the installation) +6. `pip3 install .` (this installs `trexio` in your environment) +7. `cd test && python3 test_api.py` (this executes several tests that verify the installation) You are ready to go! **Note:** installation based on `pip` compiles its own C extension (shared library) called `pytrexio`. This extension is built from the TREXIO source files coupled to the wrapper code generated by [SWIG](http://www.swig.org/). -The compiler options during such installation may differ from the ones used to compile the primary TREXIO API in C. -Furthermore, custom compiler flags provided to `./configure` or `make` are not applicable to the Python API. +The compiler options during this installation may differ from the ones used to compile the primary TREXIO API in C. +Furthermore, custom compiler flags provided to `./configure` or `make` are not applied to the Python API. ## Examples An interactive Jupyter notebook called `tutorial_benzene.ipynb` is provided in the `examples` directory. -It demonstrates some basic use cases of the TREXIO library in general and of the Python API in particular. +The notebook can be lauched either locally (see [next section](#Running-the-notebook) for details) or using [pre-built environment on Binder](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD?filepath=notebooks%2Ftutorial_benzene.ipynb). Jupyter can be installed using `pip install jupyter`. If you are not familiar with it, feel free to consult the [Jupyter documentation](https://jupyter-notebook.readthedocs.io/en/stable/notebook.html). From 8917cb747b2b18db1e99c2e0896cd1038bba1dab Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 14 Sep 2021 09:48:55 +0200 Subject: [PATCH 104/107] better README --- README.md | 15 ++++++++++----- python/README.md | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e907abe..9171859 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ [![build](https://github.com/TREX-CoE/trexio/actions/workflows/actions.yml/badge.svg)](https://github.com/TREX-CoE/trexio/actions/workflows/actions.yml) ![GitHub release (latest by date)](https://img.shields.io/github/v/release/TREX-CoE/trexio) -[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD) TREX library for efficient I/O. @@ -57,7 +56,8 @@ The primary TREXIO API is composed of the following functions: - `trexio_has_[group]_[variable]` - `trexio_close` -where `[group]` and `[variable]` substitutions correspond to the contents of the `trex.json` configuration file (for more details, see the corresponding [documentation](https://trex-coe.github.io/trexio/trex.html) page). +where `[group]` and `[variable]` substitutions correspond to the contents of the `trex.json` configuration file +(for more details, see the corresponding [documentation](https://trex-coe.github.io/trexio/trex.html) page). For example, consider the `coord` variable (array), which belongs to the `nucleus` group. The TREXIO user can write or read it using `trexio_write_nucleus_coord` or `trexio_read_nucleus_coord` functions, respectively. Note: the `[variable]` names have to be unique only within the corresponding parent `[group]`. @@ -67,7 +67,8 @@ These quantities can be accessed using the corresponding `trexio_[has|read|write ## Python API -For more details regarding the installation and usage of the TREXIO Python API, see [this page](python/README.md). +For more details regarding the installation and usage of the TREXIO Python API, +see [this page](python/README.md). ## Tutorial @@ -76,6 +77,9 @@ TREXIO tutorials in Jupyter notebook format can be found in the [corresponding GitHub repository](https://github.com/TREX-CoE/trexio-tutorials) or on [Binder](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD). +For example, the tutorial covering TREXIO basics using benzene molecule as an example can be viewed and executed online by clicking on this badge: +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD?filepath=notebooks%2Ftutorial_benzene.ipynb) + ## Technical documentation @@ -84,8 +88,9 @@ or on [Binder](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD). ### Miscellaneous -Note: The code should be compliant with the C99 [CERT C coding -standard](https://resources.sei.cmu.edu/downloads/secure-coding/assets/sei-cert-c-coding-standard-2016-v01.pdf). This can be checked with the `cppcheck` tool. +Note: The code should be compliant with the C99 +[CERT C coding standard](https://resources.sei.cmu.edu/downloads/secure-coding/assets/sei-cert-c-coding-standard-2016-v01.pdf). +This can be checked with the `cppcheck` tool. diff --git a/python/README.md b/python/README.md index 23bcc09..b3e3b06 100644 --- a/python/README.md +++ b/python/README.md @@ -1,8 +1,8 @@ -[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD) - # TREXIO Python API +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TREX-CoE/trexio-tutorials/HEAD) + TREXIO provides a Python API, which enables interactive calls to the library. It facilitates the development of interfaces between different codes and can be used to convert data from one input/output file format into another. From e9c376c83b964cdf829a99d341db87ec2eb2b234 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 14 Sep 2021 10:08:25 +0200 Subject: [PATCH 105/107] add trexio-tutorials repo as git submodule --- .gitmodules | 3 +++ python/examples | 1 + 2 files changed, 4 insertions(+) create mode 160000 python/examples diff --git a/.gitmodules b/.gitmodules index 4f78255..1e764cd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "docs/org-html-themes"] path = docs/org-html-themes url = https://github.com:fniessen/org-html-themes.git +[submodule "python/examples"] + path = python/examples + url = git@github.com:TREX-CoE/trexio-tutorials.git diff --git a/python/examples b/python/examples new file mode 160000 index 0000000..b631619 --- /dev/null +++ b/python/examples @@ -0,0 +1 @@ +Subproject commit b631619698a4b608a8778c75a9a359731bfa682a From 39b127270f0c55ab1a95676c24e31db05cab9929 Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 14 Sep 2021 10:17:20 +0200 Subject: [PATCH 106/107] add workflow to check Python API on Ubuntu --- .github/workflows/actions.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index a6ddbf2..3d69bb1 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -30,7 +30,19 @@ jobs: - name: check TREXIO run: make check - + + - name: create virtual environment + run: | + python3 -m venv --clear pytrexio-venv + source pytrexio-venv/bin/activate + + - name: install Python API + run: make python-install + # alternatively we can also run pip install trexio to check PyPI installation + + - name: check Python API + run: make python-test + - name: clean run: make clean From 264faf9c743c22ddf0227e4144c890eb7414af6c Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 14 Sep 2021 10:51:36 +0200 Subject: [PATCH 107/107] adapt MANIFEST.in to the structure of examples directory --- python/MANIFEST.in | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/MANIFEST.in b/python/MANIFEST.in index fb39dac..7c16f42 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1 +1,9 @@ -include src/*.c src/trexio*.h examples/*.ipynb requirements.txt tools/set_NUMPY_INCLUDEDIR.sh +include src/*.c +include src/trexio*.h +include examples/notebooks/* +include examples/README.md +include requirements.txt tools/set_NUMPY_INCLUDEDIR.sh + +exclude examples/LICENSE +exclude examples/requirements.txt +exclude examples/runtime.txt