diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index d6932bf..41faa44 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -17,7 +17,7 @@ */ #+end_src - + ** C #+begin_src c :tangle prefix_front.h :noweb yes @@ -48,7 +48,7 @@ typedef int32_t trexio_exit_code; #include "trexio_private.h" #include "trexio_s.h" #include "trexio_text.h" -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 #include "trexio_hdf5.h" #endif /* @@ -91,7 +91,7 @@ module trexio ** Python #+begin_src python :tangle prefix_python.py -"""The Python API of the TREXIO library. +"""The Python API of the TREXIO library. This package is a top-level wrapper of the SWIG-generated pytrexio module. """ @@ -150,7 +150,7 @@ __trexio_path__ = None #+end_src * Front end - + All calls to TREXIO are thread-safe. TREXIO front end is modular, which simplifies implementation of new back ends. @@ -324,7 +324,7 @@ return '\n'.join(result) TREXIO_INVALID_STR_LEN = 30 #+end_src :END: - + *** Decoding errors The ~trexio_string_of_error~ converts an exit code into a string. The @@ -508,7 +508,7 @@ def string_of_error(return_code: int) -> str: try: error_str = pytr.trexio_string_of_error(return_code) except: - raise + raise return error_str #+end_src @@ -526,7 +526,7 @@ def string_of_error(return_code: int) -> str: Then the corresponding back-end ~has/read/write~ functions has to be implemented. For example, see the commented lines that correspond to the ~TREXIO_JSON~ back end (not implemented yet). - _Note_: It is important to increment the value of TREXIO_INVALID_BACK_END when a new back end is added. Otherwise, it will not be available. + _Note_: It is important to increment the value of TREXIO_INVALID_BACK_END when a new back end is added. Otherwise, it will not be available. *** C @@ -561,13 +561,13 @@ bool trexio_has_backend(back_end_t back_end) { #endif } return false; -} +} #+end_src - + *** Fortran #+begin_src f90 :tangle prefix_fortran.f90 - integer(trexio_backend), parameter :: TREXIO_HDF5 = 0 + integer(trexio_backend), parameter :: TREXIO_HDF5 = 0 integer(trexio_backend), parameter :: TREXIO_TEXT = 1 ! integer(trexio_backend), parameter :: TREXIO_JSON = 2 integer(trexio_backend), parameter :: TREXIO_INVALID_BACK_END = 2 @@ -588,12 +588,12 @@ end interface #+begin_src python :tangle prefix_python.py # define TREXIO back ends -TREXIO_HDF5 = 0 +TREXIO_HDF5 = 0 TREXIO_TEXT = 1 #TREXIO_JSON = 2 TREXIO_INVALID_BACK_END = 2 #+end_src - + ** Read/write behavior Every time a reading function is called, the data is read from the @@ -662,7 +662,7 @@ class File: pytrexio_s: A PyObject corresponding to SWIG proxy of the trexio_s struct in C. This argument is in fact a TREXIO file handle, which is required for - communicating with the C back end. + communicating with the C back end. info: dict Dictionary of key-value pairs with additional information about the file. """ @@ -675,8 +675,8 @@ class File: self.filename = filename self.mode = mode self.back_end = back_end - - self.isOpen = False + + self.isOpen = False if pytrexio_s is None: self.pytrexio_s = open(filename, mode, back_end) self.isOpen = True @@ -705,7 +705,7 @@ class File: elif self.isOpen is None: raise Exception("[WIP]: TREXIO file handle provided but what if the file is already closed?") else: - pass + pass #+end_src ** Polymorphism of the file handle @@ -749,7 +749,7 @@ struct trexio_back_end_s { *** C #+begin_src c :tangle prefix_front.h :exports none -trexio_t* trexio_open(const char* file_name, const char mode, +trexio_t* trexio_open(const char* file_name, const char mode, const back_end_t back_end, trexio_exit_code* const rc_open); #+end_src @@ -786,7 +786,7 @@ trexio_open(const char* file_name, const char mode, break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 result_tmp = malloc(sizeof(trexio_hdf5_t)); break; #else @@ -842,7 +842,7 @@ trexio_open(const char* file_name, const char mode, break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_init(result); break; #else @@ -868,7 +868,7 @@ trexio_open(const char* file_name, const char mode, free(result); return NULL; } - + if (rc == TREXIO_HAS_NOT) { switch (back_end) { @@ -877,13 +877,13 @@ trexio_open(const char* file_name, const char mode, break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_write_metadata_package_version(result, TREXIO_PACKAGE_VERSION); break; #else if (rc_open != NULL) *rc_open = TREXIO_BACK_END_MISSING; return NULL; -#endif +#endif /* case TREXIO_JSON: rc = trexio_json_write_metadata_package_version(result, TREXIO_PACKAGE_VERSION); @@ -910,7 +910,7 @@ trexio_open(const char* file_name, const char mode, break; /* HDF5 v.>=1.10 has file locking activated by default */ case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = TREXIO_SUCCESS; break; #else @@ -965,9 +965,9 @@ def open(file_name: str, mode: str, back_end: int): One of the currently supported ~open~ modes (e.g. 'w', 'r') back_end: int - One of the currently supported TREXIO back ends (e.g. TREXIO_HDF5, TREXIO_TEXT) + One of the currently supported TREXIO back ends (e.g. TREXIO_HDF5, TREXIO_TEXT) - Return: + Return: SWIG object of type trexio_s. Examples: @@ -1059,7 +1059,7 @@ trexio_close (trexio_t* file) break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_deinit(file); break; #else @@ -1088,7 +1088,7 @@ trexio_close (trexio_t* file) break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = TREXIO_SUCCESS; break; #else @@ -1194,9 +1194,9 @@ def close(trexio_file): | ~$group_dset_py_dtype$~ | Standard datatype of the dataset [Python] | ~float/int~ | | ~$default_prec$~ | Default precision for read/write without suffix [C] | ~64/32~ | | ~$is_index$~ | Expands to ~true~ if dataset has a type ~index~ [C] | ~true/false~ | - - Some of the aforementioned template variables with ~group_dset~ prefix are duplicated with ~group_num~ prefix, + + Some of the aforementioned template variables with ~group_dset~ prefix are duplicated with ~group_num~ prefix, e.g. you might find $group_num_dtype_double$ in the templates corresponding to numerical attributes. The expanding values are the same as for ~group_dset~ and thus are not listed in the table above. @@ -1276,7 +1276,7 @@ trexio_read_$group_num$_64 (trexio_t* const file, $group_num_dtype_double$* cons break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_read_$group_num$(file, num); break; #else @@ -1305,17 +1305,17 @@ trexio_write_$group_num$_64 (trexio_t* const file, const $group_num_dtype_double case TREXIO_TEXT: return trexio_text_write_$group_num$(file, num); - break; - + break; + case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_write_$group_num$(file, num); break; #else return TREXIO_BACK_END_MISSING; #endif -/* - case TREXIO_JSON: +/* + case TREXIO_JSON: return trexio_json_write_$group_num$(file, num); break; ,*/ @@ -1344,7 +1344,7 @@ trexio_read_$group_num$_32 (trexio_t* const file, $group_num_dtype_single$* cons break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_read_$group_num$(file, &num_64); break; #else @@ -1377,17 +1377,17 @@ trexio_write_$group_num$_32 (trexio_t* const file, const $group_num_dtype_single case TREXIO_TEXT: return trexio_text_write_$group_num$(file, ($group_num_dtype_double$) num); - break; - + break; + case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_write_$group_num$(file, ($group_num_dtype_double$) num); break; #else return TREXIO_BACK_END_MISSING; -#endif -/* - case TREXIO_JSON: +#endif +/* + case TREXIO_JSON: return trexio_json_write_$group_num$(file, ($group_num_dtype_double$) num); break; ,*/ @@ -1431,7 +1431,7 @@ trexio_has_$group_num$ (trexio_t* const file) break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_has_$group_num$(file); break; #else @@ -1525,7 +1525,7 @@ end interface *** Python templates for front end #+begin_src python :tangle write_attr_num_front.py -def write_$group_num$(trexio_file, num_w: $group_num_py_dtype$) -> None: +def write_$group_num$(trexio_file, num_w: $group_num_py_dtype$) -> None: """Write the $group_num$ variable in the TREXIO file. Parameters: @@ -1535,7 +1535,7 @@ def write_$group_num$(trexio_file, num_w: $group_num_py_dtype$) -> None: num_w: int Value of the $group_num$ variable to be written. - + Raises: - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. - Exception from some other error (e.g. RuntimeError). @@ -1546,11 +1546,11 @@ def write_$group_num$(trexio_file, num_w: $group_num_py_dtype$) -> None: if rc != TREXIO_SUCCESS: raise Error(rc) except: - raise + raise #+end_src #+begin_src python :tangle read_attr_num_front.py -def read_$group_num$(trexio_file) -> $group_num_py_dtype$: +def read_$group_num$(trexio_file) -> $group_num_py_dtype$: """Read the $group_num$ variable from the TREXIO file. Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. @@ -1569,13 +1569,13 @@ def read_$group_num$(trexio_file) -> $group_num_py_dtype$: if rc != TREXIO_SUCCESS: raise Error(rc) except: - raise + raise return num_r #+end_src #+begin_src python :tangle has_attr_num_front.py -def has_$group_num$(trexio_file) -> bool: +def has_$group_num$(trexio_file) -> bool: """Check that $group_num$ variable exists in the TREXIO file. Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. @@ -1593,7 +1593,7 @@ def has_$group_num$(trexio_file) -> bool: if rc == TREXIO_FAILURE: raise Error(rc) except: - raise + raise if rc == TREXIO_SUCCESS: return True @@ -1670,13 +1670,13 @@ trexio_read_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* co rc = TREXIO_FAILURE; switch (file->back_end) { - + case TREXIO_TEXT: rc = trexio_text_read_$group_dset$(file, $group_dset$, rank, dims); break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_read_$group_dset$(file, $group_dset$, rank, dims); break; #else @@ -1689,7 +1689,7 @@ trexio_read_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* co ,*/ } if (rc != TREXIO_SUCCESS) return rc; - + /* Handle index type */ if ($is_index$) { uint64_t dim_size = 1; @@ -1700,7 +1700,7 @@ trexio_read_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* co $group_dset$[i] += ($group_dset_dtype_double$) 1; } } - + return TREXIO_SUCCESS; } #+end_src @@ -1754,7 +1754,7 @@ trexio_write_$group_dset$_64 (trexio_t* const file, const $group_dset_dtype_doub break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_write_$group_dset$(file, $group_dset$_p, rank, dims); break; #else @@ -1818,7 +1818,7 @@ trexio_read_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_single$* co break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_read_$group_dset$(file, $group_dset$_64, rank, dims); break; #else @@ -1901,7 +1901,7 @@ trexio_write_$group_dset$_32 (trexio_t* const file, const $group_dset_dtype_sing break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_write_$group_dset$(file, $group_dset$_64, rank, dims); break; #else @@ -1955,7 +1955,7 @@ trexio_read_safe_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_single if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; <> - + if (dim_out > (int64_t) dim_size) return TREXIO_UNSAFE_ARRAY_DIM; return trexio_read_$group_dset$_32(file, dset_out); @@ -1990,7 +1990,7 @@ trexio_read_safe_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; <> - + if (dim_out > (int64_t) dim_size) return TREXIO_UNSAFE_ARRAY_DIM; return trexio_read_$group_dset$_64(file, dset_out); @@ -2069,7 +2069,7 @@ trexio_has_$group_dset$ (trexio_t* const file) break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_has_$group_dset$(file); break; #else @@ -2162,7 +2162,7 @@ end interface *** Python templates for front end #+begin_src python :tangle write_dset_data_front.py -def write_$group_dset$(trexio_file, dset_w) -> None: +def write_$group_dset$(trexio_file, dset_w) -> None: """Write the $group_dset$ array of numbers in the TREXIO file. Parameters: @@ -2172,7 +2172,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: dset_w: list OR numpy.ndarray Array of $group_dset$ values to be written. If array data type does not correspond to int64 or float64, the conversion is performed. - + Raises: - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. - Exception from some other error (e.g. RuntimeError). @@ -2199,7 +2199,7 @@ def write_$group_dset$(trexio_file, dset_w) -> None: else: if doConversion: dset_64 = np.$group_dset_py_dtype$64(dset_w) - + else: # if input array is a multidimensional list or tuple, we have to convert it try: @@ -2217,13 +2217,13 @@ def write_$group_dset$(trexio_file, dset_w) -> None: rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_flat) else: rc = pytr.trexio_write_safe_$group_dset$_64(trexio_file.pytrexio_s, dset_w) - + if rc != TREXIO_SUCCESS: raise Error(rc) #+end_src #+begin_src python :tangle read_dset_data_front.py -def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): +def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): """Read the $group_dset$ array of numbers from the TREXIO file. Parameters: @@ -2237,13 +2237,13 @@ def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): dtype (Optional): type NumPy data type of the output (e.g. np.int32|int16 or np.float32|float16). If specified, the output array will be converted from the default double precision. - + doReshape (Optional): bool Flag to determine whether the output NumPy array has be reshaped or not. Be default, reshaping is performed based on the dimensions from the ~trex.json~ file. Otherwise, ~shape~ array (list or tuple) is used if provided by the user. Returns: - ~dset_64~ if dtype is None or ~dset_converted~ otherwise: numpy.ndarray + ~dset_64~ if dtype is None or ~dset_converted~ otherwise: numpy.ndarray 1D NumPy array with ~dim~ elements corresponding to $group_dset$ values read from the TREXIO file. Raises: @@ -2258,18 +2258,18 @@ def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): if doReshape is None: doReshape = True - + # if dim is not specified, read dimensions from the TREXIO file dims_list = None if dim is None or doReshape: - $group_dset_dim$ = read_$group_dset_dim$(trexio_file) - + $group_dset_dim$ = read_$group_dset_dim$(trexio_file) + dims_list = [$group_dset_dim_list$] dim = 1 for i in range($group_dset_rank$): dim *= dims_list[i] - + shape = tuple(dims_list) if shape is None and doReshape: raise ValueError("Reshaping failure: shape is None.") @@ -2286,7 +2286,7 @@ def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): isConverted = False dset_converted = None if dtype is not None: - + try: assert isinstance(dtype, type) except AssertionError: @@ -2298,7 +2298,7 @@ def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): dset_converted = np.array(dset_64, dtype=dtype) except: raise - + isConverted = True # additional assert can be added here to check that read_safe functions returns numpy array of proper dimension @@ -2306,14 +2306,14 @@ def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): if doReshape: try: # in-place reshaping did not work so I have to make a copy - if isConverted: + if isConverted: dset_reshaped = np.reshape(dset_converted, shape, order='C') else: dset_reshaped = np.reshape(dset_64, shape, order='C') except: raise - if isConverted: + if isConverted: return dset_converted elif doReshape: return dset_reshaped @@ -2322,7 +2322,7 @@ def read_$group_dset$(trexio_file, dim = None, doReshape = None, dtype = None): #+end_src #+begin_src python :tangle has_dset_data_front.py -def has_$group_dset$(trexio_file) -> bool: +def has_$group_dset$(trexio_file) -> bool: """Check that $group_dset$ variable exists in the TREXIO file. Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. @@ -2340,7 +2340,7 @@ def has_$group_dset$(trexio_file) -> bool: if rc == TREXIO_FAILURE: raise Error(rc) except: - raise + raise if rc == TREXIO_SUCCESS: return True @@ -2360,8 +2360,8 @@ def has_$group_dset$(trexio_file) -> bool: "eri" : [ "float sparse", [ "ao.num", "ao.num", "ao.num", "ao.num" ] ] } #+end_src - - The electron repulsion integral (eri) $\langle ij | kl \rangle$ is + + The electron repulsion integral (eri) $\langle ij | kl \rangle$ is represented as a quartet of integers $(i,j,k,l)$ and a floating point value. @@ -2374,16 +2374,16 @@ def has_$group_dset$(trexio_file) -> bool: Knowing the maximum dimensions allows to check that the integers are in a valid range, and also lets the library choose the smallest integer representation to compress the storage. - + Fortran uses 1-based array indexing, while C uses 0-based indexing. Internally, we use a 0-based representation but the Fortran binding does the appropriate conversion when reading or writing. - As the number of integrals to store can be prohibitively large, we + As the number of integrals to store can be prohibitively large, we provide the possibility to read/write the integrals in chunks. So the functions take two extra parameters: - - ~offset~ : how many integrals in the file should be skipped when reading. + - ~offset~ : how many integrals in the file should be skipped when reading. An offset of zero implies to read the first integral. - ~size~ : the number of integrals to read. @@ -2394,21 +2394,21 @@ def has_$group_dset$(trexio_file) -> bool: *** C templates for front end **** Function declarations - #+begin_src c :tangle hrw_sparse_front.h :exports none + #+begin_src c :tangle hrw_dset_sparse_front.h :exports none trexio_exit_code trexio_has_$group_sparse_dset$(trexio_t* const file); -trexio_exit_code trexio_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int64_t size, int32_t* const index_sparse, double* const value_sparse); -trexio_exit_code trexio_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int64_t size, const int32_t* index_sparse, const double* value_sparse); +trexio_exit_code trexio_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_data, const int64_t size, int32_t* const index_sparse, double* const value_sparse); +trexio_exit_code trexio_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_data, const int64_t size, const int32_t* index_sparse, const double* value_sparse); //trexio_exit_code trexio_read_$group_sparse_dset$_value(trexio_t* const file, const uint64_t offset, const uint_64_t size, int32_t* const value_sparse); //trexio_exit_code trexio_write_$group_sparse_dset$_value(trexio_t* const file, const uint64_t offset, const uint_64_t size, double* const value_sparse); #+end_src **** Source code for default functions - #+begin_src c :tangle read_sparse_front.c + #+begin_src c :tangle read_dset_sparse_front.c trexio_exit_code trexio_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, - const int64_t offset_dset, + const int64_t offset_data, const int64_t size, int32_t* const index_sparse, double* const value_sparse @@ -2416,28 +2416,35 @@ trexio_read_$group_sparse_dset$(trexio_t* const file, { if (file == NULL) return TREXIO_INVALID_ARG_1; if (offset_file < 0L) return TREXIO_INVALID_ARG_2; - if (offset_dset < 0L) return TREXIO_INVALID_ARG_3; - if (size <= 0L) return TREXIO_INVALID_ARG_4; - if (index_sparse == NULL) return TREXIO_INVALID_ARG_5; - if (value_sparse == NULL) return TREXIO_INVALID_ARG_6; + if (size <= 0L) return TREXIO_INVALID_ARG_3; + if (index_sparse == NULL) return TREXIO_INVALID_ARG_4; + if (value_sparse == NULL) return TREXIO_INVALID_ARG_5; - const uint32_t rank = $group_sparse_dset_rank$; // To be set by generator : number of indices + const uint32_t rank = $group_dset_rank$; // To be set by generator : number of indices int64_t size_max; // Max number of integrals (already in the file) trexio_exit_code rc; + // temporary + size_max = size; + /* TODO rc = trexio_read_$group_sparse_dset$_num(file, &size_max); if (rc != TREXIO_SUCCESS) return rc; + */ switch (file->back_end) { case TREXIO_TEXT: - return trexio_text_read_$group_sparse_dset$(file, offset_file, offset_dset, size, size_max, rank, index_sparse, value_sparse); + return trexio_text_read_$group_sparse_dset$(file, offset_file, offset_data, size, size_max, index_sparse, value_sparse); break; case TREXIO_HDF5: - return trexio_hdf5_read_$group_sparse_dset$(file, offset_file, offset_dset, size, size_max, rank, index_sparse, value_sparse); +#ifdef HAVE_HDF5 + return trexio_hdf5_read_$group_sparse_dset$(file, offset_file, offset_data, size, size_max, index_sparse, value_sparse); break; +#else + return TREXIO_BACK_END_MISSING; +#endif /* case TREXIO_JSON: return trexio_json_read_$group_sparse_dset$(...); @@ -2449,11 +2456,11 @@ trexio_read_$group_sparse_dset$(trexio_t* const file, } #+end_src - #+begin_src c :tangle write_sparse_front.c + #+begin_src c :tangle write_dset_sparse_front.c trexio_exit_code trexio_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, - const int64_t offset_dset, + const int64_t offset_data, const int64_t size, const int32_t* index_sparse, const double* value_sparse @@ -2461,28 +2468,35 @@ trexio_write_$group_sparse_dset$(trexio_t* const file, { if (file == NULL) return TREXIO_INVALID_ARG_1; if (offset_file < 0L) return TREXIO_INVALID_ARG_2; - if (offset_dset < 0L) return TREXIO_INVALID_ARG_3; - if (size <= 0L) return TREXIO_INVALID_ARG_4; - if (index_sparse == NULL) return TREXIO_INVALID_ARG_5; - if (value_sparse == NULL) return TREXIO_INVALID_ARG_6; + if (size <= 0L) return TREXIO_INVALID_ARG_3; + if (index_sparse == NULL) return TREXIO_INVALID_ARG_4; + if (value_sparse == NULL) return TREXIO_INVALID_ARG_5; - const uint32_t rank = $group_sparse_dset_rank$; // To be set by generator : number of indices + const uint32_t rank = $group_dset_rank$; // To be set by generator : number of indices int64_t size_max; // Max number of integrals (already in the file) trexio_exit_code rc; + // temporary + size_max = size; + /* TODO rc = trexio_read_$group_sparse_dset$_num(file, &size_max); if (rc != TREXIO_SUCCESS) return rc; + */ switch (file->back_end) { case TREXIO_TEXT: - return trexio_text_write_$group_sparse_dset$(file, offset_file, offset_dset, size, size_max, rank, index_sparse, value_sparse); + return trexio_text_write_$group_sparse_dset$(file, offset_file, offset_data, size, size_max, index_sparse, value_sparse); break; case TREXIO_HDF5: - return trexio_hdf5_write_$group_sparse_dset$(file, offset_file, offset_dset, size, size_max, rank, index_sparse, value_sparse); +#ifdef HAVE_HDF5 + return trexio_hdf5_write_$group_sparse_dset$(file, offset_file, offset_data, size, size_max, index_sparse, value_sparse); break; +#else + return TREXIO_BACK_END_MISSING; +#endif /* case TREXIO_JSON: return trexio_json_write_$group_sparse_dset$(...); @@ -2495,6 +2509,39 @@ trexio_write_$group_sparse_dset$(trexio_t* const file, #+end_src + #+begin_src c :tangle has_dset_sparse_front.c +trexio_exit_code +trexio_has_$group_sparse_dset$ (trexio_t* const file) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + + assert(file->back_end < TREXIO_INVALID_BACK_END); + + switch (file->back_end) { + + case TREXIO_TEXT: + return trexio_text_has_$group_sparse_dset$(file); + break; + + case TREXIO_HDF5: +#ifdef HAVE_HDF5 + return trexio_hdf5_has_$group_sparse_dset$(file); + break; +#else + return TREXIO_BACK_END_MISSING; +#endif +/* + case TREXIO_JSON: + return trexio_json_has_$group_sparse_dset$(file); + break; +,*/ + } + + return TREXIO_FAILURE; +} + #+end_src + ** Templates for front end has/read/write a dataset of strings *** Introduction This section concerns API calls related to datasets of strings. @@ -2551,7 +2598,7 @@ trexio_read_$group_dset$_low (trexio_t* const file, char* dset_out, const int32_ break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_read_$group_dset$(file, dset_out, rank, dims, (uint32_t) max_str_len); break; #else @@ -2671,7 +2718,7 @@ trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const tmp_str += pch_len + 1; } - rc = TREXIO_FAILURE; + rc = TREXIO_FAILURE; switch (file->back_end) { case TREXIO_TEXT: @@ -2679,7 +2726,7 @@ trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 rc = trexio_hdf5_write_$group_dset$(file, (const char**) dset_str, rank, dims); break; #else @@ -2694,7 +2741,7 @@ trexio_write_$group_dset$_low (trexio_t* const file, const char* dset_in, const FREE(dset_str[0]); FREE(dset_str); - + return rc; } @@ -2751,7 +2798,7 @@ trexio_has_$group_dset$ (trexio_t* const file) break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_has_$group_dset$(file); break; #else @@ -2814,14 +2861,14 @@ end interface character, allocatable :: str_compiled(:) integer(8) :: $group_dset_dim$ integer :: rc - + rc = trexio_read_$group_dset_dim$_64(trex_file, $group_dset_dim$) if (rc /= TREXIO_SUCCESS) trexio_read_$group_dset$ = rc allocate(str_compiled($group_dset_dim$*(max_str_len+1)+1)) - + rc = trexio_read_$group_dset$_low(trex_file, str_compiled, max_str_len) - if (rc /= TREXIO_SUCCESS) then + if (rc /= TREXIO_SUCCESS) then deallocate(str_compiled) trexio_read_$group_dset$ = rc else @@ -2843,7 +2890,7 @@ end interface character(len=:), allocatable :: str_compiled integer(8) :: $group_dset_dim$ integer :: rc - + rc = trexio_read_$group_dset_dim$_64(trex_file, $group_dset_dim$) if (rc /= TREXIO_SUCCESS) then trexio_write_$group_dset$ = rc @@ -2858,7 +2905,7 @@ end interface *** Python templates for front end #+begin_src python :tangle write_dset_str_front.py -def write_$group_dset$(trexio_file, dset_w: list) -> None: +def write_$group_dset$(trexio_file, dset_w: list) -> None: """Write the $group_dset$ array of strings in the TREXIO file. Parameters: @@ -2868,7 +2915,7 @@ def write_$group_dset$(trexio_file, dset_w: list) -> None: dset_w: list Array of $group_dset$ strings to be written. - + Raises: - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. - Exception from some other error (e.g. RuntimeError). @@ -2882,12 +2929,12 @@ def write_$group_dset$(trexio_file, dset_w: list) -> None: if rc != TREXIO_SUCCESS: raise Error(rc) except: - raise + raise #+end_src #+begin_src python :tangle read_dset_str_front.py -def read_$group_dset$(trexio_file, dim = None) -> list: +def read_$group_dset$(trexio_file, dim = None) -> list: """Read the $group_dset$ array of strings from the TREXIO file. Parameters: @@ -2910,8 +2957,8 @@ def read_$group_dset$(trexio_file, dim = None) -> list: # if dim is not specified, read dimensions from the TREXIO file if dim is None: - $group_dset_dim$ = read_$group_dset_dim$(trexio_file) - + $group_dset_dim$ = read_$group_dset_dim$(trexio_file) + dims_list = [$group_dset_dim_list$] dim = 1 for i in range($group_dset_rank$): @@ -2924,7 +2971,7 @@ def read_$group_dset$(trexio_file, dim = None) -> list: if rc != TREXIO_SUCCESS: raise Error(rc) except: - raise + raise try: @@ -2939,7 +2986,7 @@ def read_$group_dset$(trexio_file, dim = None) -> list: #+end_src #+begin_src python :tangle has_dset_str_front.py -def has_$group_dset$(trexio_file) -> bool: +def has_$group_dset$(trexio_file) -> bool: """Check that $group_dset$ variable exists in the TREXIO file. Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. @@ -2957,7 +3004,7 @@ def has_$group_dset$(trexio_file) -> bool: if rc == TREXIO_FAILURE: raise Error(rc) except: - raise + raise if rc == TREXIO_SUCCESS: return True @@ -3004,7 +3051,7 @@ trexio_read_$group_str$ (trexio_t* const file, char* const str_out, const int32_ break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_read_$group_str$(file, str_out, (uint32_t) max_str_len); break; #else @@ -3032,7 +3079,7 @@ trexio_write_$group_str$ (trexio_t* const file, const char* str, const int32_t m if (trexio_has_$group_str$(file) == TREXIO_SUCCESS) return TREXIO_ATTR_ALREADY_EXISTS; size_t len_write = strlen(str); - if ((size_t) max_str_len < len_write) return TREXIO_INVALID_STR_LEN; + if ((size_t) max_str_len < len_write) return TREXIO_INVALID_STR_LEN; switch (file->back_end) { @@ -3041,7 +3088,7 @@ trexio_write_$group_str$ (trexio_t* const file, const char* str, const int32_t m break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_write_$group_str$(file, str); break; #else @@ -3057,7 +3104,7 @@ trexio_write_$group_str$ (trexio_t* const file, const char* str, const int32_t m return TREXIO_FAILURE; } #+end_src - + #+begin_src c :tangle has_attr_str_front.c trexio_exit_code trexio_has_$group_str$ (trexio_t* const file) @@ -3074,7 +3121,7 @@ trexio_has_$group_str$ (trexio_t* const file) break; case TREXIO_HDF5: -#ifdef HAVE_HDF5 +#ifdef HAVE_HDF5 return trexio_hdf5_has_$group_str$(file); break; #else @@ -3096,7 +3143,7 @@ trexio_has_$group_str$ (trexio_t* const file) The ~Fortran~ templates that provide an access to the ~C~ API calls from Fortran. These templates are based on the use of ~iso_c_binding~. Pointers have to be passed by value. - + #+begin_src f90 :tangle write_attr_str_front_fortran.f90 interface integer function trexio_write_$group_str$_c (trex_file, str, max_str_len) & @@ -3136,7 +3183,7 @@ end interface integer(8), intent(in), value :: trex_file integer(4), intent(in), value :: max_str_len character, intent(out) :: str(*) - + trexio_read_$group_str$ = trexio_read_$group_str$_c(trex_file, str, max_str_len) end function trexio_read_$group_str$ @@ -3162,7 +3209,7 @@ end interface *** Python templates for front end #+begin_src python :tangle write_attr_str_front.py -def write_$group_str$(trexio_file, str_w: str) -> None: +def write_$group_str$(trexio_file, str_w: str) -> None: """Write the $group_str$ variable in the TREXIO file. Parameters: @@ -3172,7 +3219,7 @@ def write_$group_str$(trexio_file, str_w: str) -> None: str_w: str String corresponding to the $group_str$ variable to be written. - + Raises: - Exception from AssertionError if TREXIO return code ~rc~ is different from TREXIO_SUCCESS and prints the error message using trexio_string_of_error. - Exception from some other error (e.g. RuntimeError). @@ -3186,11 +3233,11 @@ def write_$group_str$(trexio_file, str_w: str) -> None: if rc != TREXIO_SUCCESS: raise Error(rc) except: - raise + raise #+end_src #+begin_src python :tangle read_attr_str_front.py -def read_$group_str$(trexio_file) -> str: +def read_$group_str$(trexio_file) -> str: """Read the $group_str$ variable from the TREXIO file. Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. @@ -3210,13 +3257,13 @@ def read_$group_str$(trexio_file) -> str: if rc != TREXIO_SUCCESS: raise Error(rc) except: - raise + raise return str_r #+end_src - + #+begin_src python :tangle has_attr_str_front.py -def has_$group_str$(trexio_file) -> bool: +def has_$group_str$(trexio_file) -> bool: """Check that $group_str$ variable exists in the TREXIO file. Parameter is a ~TREXIO File~ object that has been created by a call to ~open~ function. @@ -3234,7 +3281,7 @@ def has_$group_str$(trexio_file) -> bool: if rc == TREXIO_FAILURE: raise Error(rc) except: - raise + raise if rc == TREXIO_SUCCESS: return True @@ -3274,8 +3321,8 @@ contains end function trexio_open #+end_src - The subroutine below transforms an array of Fortran strings into one big string using ~TREXIO_DELIM~ symbol - as a delimeter and adds ~NULL~ character in the end in order to properly pass the desired string to + The subroutine below transforms an array of Fortran strings into one big string using ~TREXIO_DELIM~ symbol + as a delimeter and adds ~NULL~ character in the end in order to properly pass the desired string to C API. This is needed due to the fact that strings in C are terminated by ~NULL~ character ~\0~. #+begin_src f90 :tangle helper_fortran.f90 @@ -3308,13 +3355,13 @@ contains integer, intent(in), value :: max_len_str ! maximum length of a string in an array character, intent(in) :: str_flat(*) character(len=*), intent(inout) :: str_array(*) - + character(len=max_len_str) :: tmp_str integer :: i, j, k, ind, offset integer(8) :: len_flat len_flat = (max_len_str+1)*max_num_str + 1 - + ind=1 offset=1 do i=1,max_num_str diff --git a/src/templates_text/build.sh b/src/templates_text/build.sh index 4cf6c64..d219ae5 100644 --- a/src/templates_text/build.sh +++ b/src/templates_text/build.sh @@ -19,23 +19,26 @@ cat populated/pop_flush_group_text.h >> trexio_text.h cat populated/pop_has_dset_data_text.c >> trexio_text.c cat populated/pop_has_dset_str_text.c >> trexio_text.c +cat populated/pop_has_dset_sparse_text.c >> trexio_text.c cat populated/pop_has_attr_num_text.c >> trexio_text.c cat populated/pop_has_attr_str_text.c >> trexio_text.c + cat populated/pop_read_dset_data_text.c >> trexio_text.c cat populated/pop_read_dset_str_text.c >> trexio_text.c +cat populated/pop_read_dset_sparse_text.c >> trexio_text.c cat populated/pop_read_attr_str_text.c >> trexio_text.c cat populated/pop_read_attr_num_text.c >> trexio_text.c + cat populated/pop_write_dset_data_text.c >> trexio_text.c cat populated/pop_write_dset_str_text.c >> trexio_text.c +cat populated/pop_write_dset_sparse_text.c >> trexio_text.c cat populated/pop_write_attr_str_text.c >> trexio_text.c cat populated/pop_write_attr_num_text.c >> trexio_text.c + cat populated/pop_hrw_dset_data_text.h >> trexio_text.h cat populated/pop_hrw_dset_str_text.h >> trexio_text.h +cat populated/pop_hrw_dset_sparse_text.h >> trexio_text.h cat populated/pop_hrw_attr_num_text.h >> trexio_text.h cat populated/pop_hrw_attr_str_text.h >> trexio_text.h -cat rdm_text.c >> trexio_text.c -cat rdm_text.h >> trexio_text.h - cat suffix_text.h >> trexio_text.h - diff --git a/src/templates_text/templator_text.org b/src/templates_text/templator_text.org index c9d8204..a1ec0d1 100644 --- a/src/templates_text/templator_text.org +++ b/src/templates_text/templator_text.org @@ -1001,28 +1001,28 @@ trexio_text_has_$group_str$ (trexio_t* const file) } #+end_src -** Template for has/read/write the dataset of sparse data +** Template for has/read/write the dataset of sparse data - #+begin_src c :tangle hrw_sparse_text.h :exports none + #+begin_src c :tangle hrw_dset_sparse_text.h :exports none trexio_exit_code trexio_text_has_$group_sparse_dset$(trexio_t* const file); -trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int_64_t size, const int64_t size_max, int32_t* const index_sparse, double* const value_sparse); -trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int_64_t size, const int64_t size_max, const int32_t* index_sparse, const double* value_sparse); +trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int64_t size, const int64_t size_max, int32_t* const index_sparse, double* const value_sparse); +trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int64_t size, const int64_t size_max, const int32_t* index_sparse, const double* value_sparse); #+end_src - #+begin_src c :tangle write_sparse_text.c -trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, - const int64_t offset_file, - const int64_t offset_dset, - const int64_t size, - const int64_t size_max, - const int32_t* index_sparse, + #+begin_src c :tangle write_dset_sparse_text.c +trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, + const int64_t offset_file, + const int64_t offset_dset, + const int64_t size, + const int64_t size_max, + const int32_t* index_sparse, const double* value_sparse) { if (file == NULL) return TREXIO_FILE_ERROR; /* Build the name of the file with sparse data*/ const char* $group_sparse_dset$_file_name = "/$group_sparse_dset$.txt"; - const char file_abs_path[TREXIO_MAX_FILENAME_LENGTH]; + char file_abs_path[TREXIO_MAX_FILENAME_LENGTH]; strncpy (file_abs_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH); strncat (file_abs_path, $group_sparse_dset$_file_name, @@ -1032,17 +1032,18 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, FILE* f = fopen(file_abs_path, "a"); //TODO ERROR HANDLING assert(f != NULL); - + // read the currently written number of elements // line_length is 69 because // 10 per index (40 in total) + 4 spaces + 24 for floating point value + 1 for newline char // in general: 10*n_indices + n_indices + 24 + 1 const uint64_t line_length = $group_sparse_dset_line_length$L; - - fseek(f, (long) offset_file * line_length, SEEK_SET); - - for (uint64_t i=0L+offset_data ; i 0); @@ -1055,16 +1056,18 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, index[4*i+3], value[i]); */ - - fclose(f); + + rc = fclose(f); + assert(rc == 0); + return TREXIO_SUCCESS; } #+end_src - - #+begin_src c :tangle read_sparse_text.c -trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, - const int64_t offset_file, + + #+begin_src c :tangle read_dset_sparse_text.c +trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, + const int64_t offset_file, const int64_t offset_dset, const int64_t size, const int64_t size_max, @@ -1072,81 +1075,67 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, double* const value_sparse) { if (file == NULL) return TREXIO_FILE_ERROR; - + /* Build the name of the file with sparse data*/ const char* $group_sparse_dset$_file_name = "/$group_sparse_dset$.txt"; - const char file_abs_path[TREXIO_MAX_FILENAME_LENGTH]; + char file_abs_path[TREXIO_MAX_FILENAME_LENGTH]; strncpy (file_abs_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH); strncat (file_abs_path, $group_sparse_dset$_file_name, TREXIO_MAX_FILENAME_LENGTH-strlen($group_sparse_dset$_file_name)); - FILE* f = fopen(file_abs_path, "a"); + FILE* f = fopen(file_abs_path, "r"); //TODO ERROR HANDLING assert(f != NULL); - + // read the currently written number of elements // line_length is 69 because // 10 per index (40 in total) + 4 spaces + 24 for floating point value + 1 for newline char // in general: 10*n_indices + n_indices + 24 + 1 const uint64_t line_length = $group_sparse_dset_line_length$L; - + fseek(f, (long) offset_file * line_length, SEEK_SET); - - for (uint64_t i=0L+offset_data ; i 0); - } + } } -/* - int rc = fscanf(f, "%d %d %d %d %lf", - &index[4*i], - &index[4*i+1], - &index[4*i+2],N OPEN - &index[4*i+3], - &value[i]); -,*/ - - fclose(f); + + rc = fclose(f); + assert(rc==0); + return TREXIO_SUCCESS; } #+end_src - #+begin_src c :tangle has_sparse_text.c + #+begin_src c :tangle has_dset_sparse_text.c trexio_exit_code trexio_text_has_$group_sparse_dset$(trexio_t* const file) { if (file == NULL) return TREXIO_FILE_ERROR; - + /* Build the name of the file with sparse data*/ const char* $group_sparse_dset$_file_name = "/$group_sparse_dset$.txt"; - const char file_abs_path[TREXIO_MAX_FILENAME_LENGTH]; + char file_abs_path[TREXIO_MAX_FILENAME_LENGTH]; strncpy (file_abs_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH); strncat (file_abs_path, $group_sparse_dset$_file_name, TREXIO_MAX_FILENAME_LENGTH-strlen($group_sparse_dset$_file_name)); - /*struct stat buffer; - int rc = stat(file_abs_path, &buffer) - if (rc == 0) { + if (access(file_abs_path, F_OK) == 0){ return TREXIO_SUCCESS; } else { return TREXIO_HAS_NOT; - }*/ - - int fd = open(file_abs_path, O_CREAT | O_EXCL | O_RDONLY); - if (fd < 0) { - if(errno == EEXIST) return TREXIO_SUCCESS; - } else { - return TREXIO_HAS_NOT; } } #+end_src @@ -1156,4 +1145,3 @@ trexio_exit_code trexio_text_has_$group_sparse_dset$(trexio_t* const file) #+begin_src c :tangle suffix_text.h #endif #+end_src - diff --git a/tools/generator.py b/tools/generator.py index 30e06ad..4d5ba32 100644 --- a/tools/generator.py +++ b/tools/generator.py @@ -6,18 +6,13 @@ config_file = 'trex.json' trex_config = read_json(config_file) # --------------------------------------------------------------------------- # -# -------------------------------- [WIP] ------------------------------------ # -# for now remove rdm from config because it functions are hardcoded -del trex_config['rdm'] -# --------------------------------------------------------------------------- # - # -------------------- GET ATTRIBUTES FROM THE CONFIGURATION ---------------- # group_dict = get_group_dict(trex_config) detailed_nums = get_detailed_num_dict(trex_config) detailed_strs = get_detailed_str_dict(trex_config) # helper dictionaries that contain names of groups, nums or dsets as keys dsets = get_dset_dict(trex_config) -detailed_dsets_nostr, detailed_dsets_str = split_dset_dict_detailed(dsets) +detailed_dsets_nostr, detailed_dsets_str, detailed_dsets_sparse = split_dset_dict_detailed(dsets) detailed_dsets = detailed_dsets_nostr.copy() detailed_dsets.update(detailed_dsets_str) # consistency check for dimensioning variables @@ -26,9 +21,9 @@ check_dim_consistency(detailed_nums, dsets) # -------------------- GET TEMPLATED FILES TO BE POPULATED ------------------ # source = ['front', 'text', 'hdf5'] -# build helper dictionaries with paths per source directory +# build helper dictionaries with paths per source directory template_paths = get_template_paths(source) -# build helper dictionaries with source files per source directory +# build helper dictionaries with source files per source directory source_files = get_source_files(template_paths) # build helper dictionaries with templated files files_todo = get_files_todo(source_files) @@ -48,14 +43,18 @@ for fname in files_todo['attr_num']: for fname in files_todo['attr_str']: recursive_populate_file(fname, template_paths, detailed_strs) -# populate has/read/write_dset (numerical) functions with recursive scheme +# populate has/read/write_dset (numerical) functions with recursive scheme for fname in files_todo['dset_data']: recursive_populate_file(fname, template_paths, detailed_dsets_nostr) -# populate has/read/write_dset (strings) functions with recursive scheme +# populate has/read/write_dset (strings) functions with recursive scheme for fname in files_todo['dset_str']: recursive_populate_file(fname, template_paths, detailed_dsets_str) +# populate has/read/write_dset (sparse) functions with recursive scheme +for fname in files_todo['dset_sparse']: + recursive_populate_file(fname, template_paths, detailed_dsets_sparse) + # populate group-related functions with mixed (iterative+recursive) scheme [text backend] for fname in files_todo['group']: special_populate_text_group(fname, template_paths, group_dict, detailed_dsets, detailed_nums, detailed_strs) diff --git a/tools/generator_tools.py b/tools/generator_tools.py index d7cb9a0..f11b43c 100644 --- a/tools/generator_tools.py +++ b/tools/generator_tools.py @@ -4,7 +4,7 @@ from json import load as json_load def read_json(fname: str) -> dict: - """ + """ Read configuration from the input `fname` JSON file. Parameters: @@ -23,7 +23,7 @@ def read_json(fname: str) -> dict: def get_files_todo(source_files: dict) -> dict: - """ + """ Build dictionaries of templated files per objective. Parameters: @@ -36,21 +36,21 @@ def get_files_todo(source_files: dict) -> dict: for key in source_files.keys(): all_files += source_files[key] - files_todo = {} + files_todo = {} #files_todo['all'] = list(filter(lambda x: 'read' in x or 'write' in x or 'has' in x or 'hrw' in x or 'flush' in x or 'free' in x, all_files)) files_todo['all'] = [f for f in all_files if 'read' in f or 'write' in f or 'has' in f or 'flush' in f or 'free' in f or 'hrw' in f] - for key in ['dset_data', 'dset_str', 'attr_num', 'attr_str', 'group']: + for key in ['dset_data', 'dset_str', 'dset_sparse', 'attr_num', 'attr_str', 'group']: files_todo[key] = list(filter(lambda x: key in x, files_todo['all'])) files_todo['group'].append('struct_text_group_dset.h') # files that correspond to iterative population (e.g. the code is repeated within the function body but the function itself is unique) - files_todo['auxiliary'] = ['def_hdf5.c', 'basic_hdf5.c', 'basic_text_group.c', 'struct_hdf5.h', 'struct_text_group.h'] + files_todo['auxiliary'] = ['def_hdf5.c', 'basic_hdf5.c', 'basic_text_group.c', 'struct_hdf5.h', 'struct_text_group.h'] return files_todo def get_source_files(paths: dict) -> dict: - """ + """ Build dictionaries of all files per source directory. Parameters: @@ -67,7 +67,7 @@ def get_source_files(paths: dict) -> dict: def get_template_paths(source: list) -> dict: - """ + """ Build dictionary of the absolute paths to directory with templates per source. Parameters: @@ -86,7 +86,7 @@ def get_template_paths(source: list) -> dict: def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> None: - """ + """ Populate files containing basic read/write/has functions. Parameters: @@ -107,6 +107,8 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N 'group_num_f_dtype_default', 'group_num_f_dtype_double', 'group_num_f_dtype_single', 'group_num_dtype_default', 'group_num_dtype_double', 'group_num_dtype_single', 'group_num_h5_dtype', 'group_num_py_dtype', + 'group_sparse_dset', 'group_sparse_dset_format_scanf', 'group_sparse_dset_format_printf', + 'group_sparse_dset_line_length', 'group_sparse_dset_indices_printf', 'group_sparse_dset_indices_scanf', 'group_dset', 'group_num', 'group_str', 'group'] for item in detailed_source.keys(): @@ -133,9 +135,9 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N elif 'uncommented by the generator for dimensioning' in line: # only uncomment and write the line if `num` is in the name if 'dim' in detailed_source[item]['trex_json_int_type']: - templine = line.replace('//', '') + templine = line.replace('//', '') f_out.write(templine) - # general case of recursive replacement of inline triggers + # general case of recursive replacement of inline triggers else: populated_line = recursive_replace_line(line, triggers, detailed_source[item]) f_out.write(populated_line) @@ -144,8 +146,8 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N def recursive_replace_line (input_line: str, triggers: list, source: dict) -> str: - """ - Recursive replacer. Recursively calls itself as long as there is at least one "$" present in the `input_line`. + """ + Recursive replacer. Recursively calls itself as long as there is at least one "$" present in the `input_line`. Parameters: input_line (str) : input line @@ -154,10 +156,10 @@ def recursive_replace_line (input_line: str, triggers: list, source: dict) -> st Returns: output_line (str) : processed (replaced) line - """ + """ is_triggered = False output_line = input_line - + if '$' in input_line: for case in triggers: test_case = f'${case}$' @@ -175,12 +177,12 @@ def recursive_replace_line (input_line: str, triggers: list, source: dict) -> st else: print(output_line) raise ValueError('Recursion went wrong, not all cases considered') - + return output_line def iterative_populate_file (filename: str, paths: dict, groups: dict, datasets: dict, numbers: dict, strings: dict) -> None: - """ + """ Iteratively populate files with unique functions that contain templated variables. Parameters: @@ -200,7 +202,7 @@ def iterative_populate_file (filename: str, paths: dict, groups: dict, datasets: templ_path = get_template_path(filename, paths) filename_out = join('populated',f'pop_{filename}') # Note: it is important that special conditions like add_trigger above will be checked before standard triggers -# that contain only basic $-ed variable (like $group$). Otherwise, the standard triggers will be removed +# that contain only basic $-ed variable (like $group$). Otherwise, the standard triggers will be removed # from the template and the special condition will never be met. with open(join(templ_path,filename), 'r') as f_in : with open(join(templ_path,filename_out), 'a') as f_out : @@ -225,13 +227,13 @@ def iterative_populate_file (filename: str, paths: dict, groups: dict, datasets: f_out.write(populated_line) else: f_out.write(line) - + f_out.write("\n") def iterative_replace_line (input_line: str, case: str, source: dict, add_line: str) -> str: - """ - Iterative replacer. Iteratively copy-pastes `input_line` each time with a new substitution of a templated variable depending on the `case`. + """ + Iterative replacer. Iteratively copy-pastes `input_line` each time with a new substitution of a templated variable depending on the `case`. Parameters: input_line (str) : input line @@ -241,7 +243,7 @@ def iterative_replace_line (input_line: str, case: str, source: dict, add_line: Returns: output_block (str) : processed (replaced) block of text - """ + """ output_block = "" for item in source.keys(): templine1 = input_line.replace(case.upper(), item.upper()) @@ -270,12 +272,12 @@ def check_triggers (input_line: str, triggers: list) -> int: if trig in input_line or trig.upper() in input_line: out_id = id return out_id - + return out_id def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detailed_dset: dict, detailed_numbers: dict, detailed_strings: dict) -> None: - """ + """ Special population for group-related functions in the TEXT back end. Parameters: @@ -316,11 +318,11 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai elif 'START REPEAT GROUP_NUM' in line or 'START REPEAT GROUP_ATTR_STR' in line: subloop_num = True continue - + if 'END REPEAT GROUP_DSET' in line: for dset in detailed_dset.keys(): - if group != detailed_dset[dset]['group']: + if group != detailed_dset[dset]['group']: continue if ('REPEAT GROUP_DSET_STR' in line) and (detailed_dset[dset]['dtype'] != 'char*'): @@ -351,7 +353,7 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai elif 'END REPEAT GROUP_NUM' in line: for dim in detailed_numbers.keys(): - if group != detailed_numbers[dim]['group']: + if group != detailed_numbers[dim]['group']: continue save_body = loop_body @@ -364,7 +366,7 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai elif 'END REPEAT GROUP_ATTR_STR' in line: for str in detailed_strings.keys(): - if group != detailed_strings[str]['group']: + if group != detailed_strings[str]['group']: continue str_allocated.append(str) @@ -390,22 +392,22 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai continue if not subloop_num and not subloop_dset: - # NORMAL CASE WITHOUT SUBLOOPS + # NORMAL CASE WITHOUT SUBLOOPS if '$group_dset' in line: for dset in detailed_dset.keys(): - if group != detailed_dset[dset]['group']: + if group != detailed_dset[dset]['group']: continue populated_line = recursive_replace_line(line, triggers, detailed_dset[dset]) f_out.write(populated_line) elif '$group_str' in line: for str in detailed_strings.keys(): - if group != detailed_strings[str]['group']: + if group != detailed_strings[str]['group']: continue populated_line = recursive_replace_line(line, triggers, detailed_strings[str]) f_out.write(populated_line) elif '$group_num$' in line: for dim in detailed_numbers.keys(): - if group != detailed_numbers[dim]['group']: + if group != detailed_numbers[dim]['group']: continue populated_line = recursive_replace_line(line, triggers, detailed_numbers[dim]) f_out.write(populated_line) @@ -421,7 +423,7 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai def get_template_path (filename: str, path_dict: dict) -> str: - """ + """ Returns the absolute path to the directory with indicated `filename` template. Parameters: @@ -435,12 +437,12 @@ def get_template_path (filename: str, path_dict: dict) -> str: if dir_type in filename: path = path_dict[dir_type] return path - + raise ValueError('Filename should contain one of the keywords') def get_group_dict (configuration: dict) -> dict: - """ + """ Returns the dictionary of all groups. Parameters: @@ -457,9 +459,9 @@ def get_group_dict (configuration: dict) -> dict: def get_detailed_num_dict (configuration: dict) -> dict: - """ + """ Returns the dictionary of all `num`-suffixed variables. - Keys are names, values are subdictionaries containing corresponding group and group_num names. + Keys are names, values are subdictionaries containing corresponding group and group_num names. Parameters: configuration (dict) : configuration from `trex.json` @@ -503,7 +505,7 @@ def get_detailed_num_dict (configuration: dict) -> dict: tmp_dict['group_num_dtype_single'] = 'int32_t' tmp_dict['default_prec'] = '32' tmp_dict['group_num_std_dtype_out'] = '" PRId64 "' - tmp_dict['group_num_std_dtype_in'] = '" SCNd64 "' + tmp_dict['group_num_std_dtype_in'] = '" SCNd64 "' tmp_dict['group_num_py_dtype'] = 'int' tmp_dict['trex_json_int_type'] = v2[0] @@ -536,8 +538,8 @@ def get_detailed_str_dict (configuration: dict) -> dict: def get_dset_dict (configuration: dict) -> dict: - """ - Returns the dictionary of datasets. + """ + Returns the dictionary of datasets. Keys are names, values are lists containing datatype, list of dimensions and group name Parameters: @@ -559,8 +561,8 @@ def get_dset_dict (configuration: dict) -> dict: def split_dset_dict_detailed (datasets: dict) -> tuple: - """ - Returns the detailed dictionary of datasets. + """ + Returns the detailed dictionary of datasets. Keys are names, values are subdictionaries containing substitutes for templated variables Parameters: @@ -571,9 +573,11 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: """ dset_numeric_dict = {} dset_string_dict = {} + dset_sparse_dict = {} for k,v in datasets.items(): # create a temp dictionary tmp_dict = {} + is_sparse = False # specify details required to replace templated variables later if v[0] == 'float': datatype = 'double' @@ -599,7 +603,7 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: group_dset_dtype_single = 'int32_t' default_prec = '32' group_dset_std_dtype_out = '" PRId64 "' - group_dset_std_dtype_in = '" SCNd64 "' + group_dset_std_dtype_in = '" SCNd64 "' group_dset_py_dtype = 'int' elif v[0] == 'str': datatype = 'char*' @@ -612,11 +616,28 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: group_dset_dtype_single = '' default_prec = '' group_dset_std_dtype_out = 's' - group_dset_std_dtype_in = 's' + group_dset_std_dtype_in = 's' group_dset_py_dtype = 'str' - + elif 'sparse' in v[0]: + is_sparse = True + datatype = 'double' + group_dset_h5_dtype = 'native_double' + group_dset_f_dtype_default= 'real(8)' + group_dset_f_dtype_double = 'real(8)' + group_dset_f_dtype_single = 'real(4)' + group_dset_dtype_default= 'double' + group_dset_dtype_double = 'double' + group_dset_dtype_single = 'float' + default_prec = '64' + group_dset_std_dtype_out = '24.16e' + group_dset_std_dtype_in = 'lf' + group_dset_py_dtype = 'float' + # add the dset name for templates - tmp_dict['group_dset'] = k + if is_sparse: + tmp_dict['group_sparse_dset'] = k + else: + tmp_dict['group_dset'] = k # add flag to detect index types if 'index' == v[0]: tmp_dict['is_index'] = 'file->one_based' @@ -625,7 +646,7 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: # add the datatypes for templates tmp_dict['dtype'] = datatype tmp_dict['group_dset_dtype'] = datatype - tmp_dict['group_dset_h5_dtype'] = group_dset_h5_dtype + tmp_dict['group_dset_h5_dtype'] = group_dset_h5_dtype tmp_dict['group_dset_f_dtype_default'] = group_dset_f_dtype_default tmp_dict['group_dset_f_dtype_double'] = group_dset_f_dtype_double tmp_dict['group_dset_f_dtype_single'] = group_dset_f_dtype_single @@ -647,7 +668,7 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: for i in range(1, tmp_dict['rank']): dim_toadd = tmp_dict['dims'][i] dim_list += f', {dim_toadd}' - + tmp_dict['group_dset_dim_list'] = dim_list if tmp_dict['rank'] == 0: @@ -656,21 +677,30 @@ def split_dset_dict_detailed (datasets: dict) -> tuple: dim_f_list = "(*)" tmp_dict['group_dset_f_dims'] = dim_f_list + if is_sparse: + tmp_dict['group_sparse_dset_format_scanf'] = "%d %d %d %d %lf" + tmp_dict['group_sparse_dset_format_printf'] = "%10d %10d %10d %10d %24.16e" + tmp_dict['group_sparse_dset_line_length'] = "69" + tmp_dict['group_sparse_dset_indices_printf'] = "index_sparse[4*i], index_sparse[4*i+1], index_sparse[4*i+2], index_sparse[4*i+3]" + tmp_dict['group_sparse_dset_indices_scanf'] = "&index_sparse[4*i], &index_sparse[4*i+1], &index_sparse[4*i+2], &index_sparse[4*i+3]" + # add group name as a key-value pair to the dset dict tmp_dict['group'] = v[2] # split datasets in numeric- and string- based if (datatype == 'char*'): dset_string_dict[k] = tmp_dict + elif (is_sparse): + dset_sparse_dict[k] = tmp_dict else: dset_numeric_dict[k] = tmp_dict - return (dset_numeric_dict, dset_string_dict) + return (dset_numeric_dict, dset_string_dict, dset_sparse_dict) def check_dim_consistency(num: dict, dset: dict) -> None: - """ - Consistency check to make sure that each dimensioning variable exists as a num attribute of some group. + """ + Consistency check to make sure that each dimensioning variable exists as a num attribute of some group. Parameters: num (dict) : dictionary of numerical attributes