TREX/trexio
1
0
Fork 0
mirror of https://github.com/TREX-CoE/trexio.git synced 2025-03-04 06:10:23 +01:00
Code Issues Projects Releases Wiki Activity

Major refactoring of the generator script (#47)

Browse Source 
* minor cleaning

* create functions for dictionary parsers in the external file

* remove files_exclude since post-processing does the job

* no need to have group_group_dset when propagatin dsets

* oneliner for paths to templates

* add dset_per_group dict

* add function to iteratively populate string based on triggers list

* added recursive replacer for numbers

* add recursive replaces for datasets

* add function for text groups  [iterative+recursive upd]

* do not define triggers in the master script

* transition to helper dictionaries

* cleaning

* comment main code blocks

* rearrange parameters

* add documentation strings to the functions

* minor cleaning and changes

* adapt build_trexio script

* add consisteny check for dimensioning variables
This commit is contained in:
Evgeny Posenitskiy 2021-05-26 19:11:46 +02:00 committed by GitHub
parent 331e3d64fd
commit 29d927675e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 712 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
src/templates_front/templator_front.org
View File

@ -613,7 +613,7 @@ trexio_close (trexio_t* file)
trexio_exit_code rc = TREXIO_FAILURE;
assert(file->back_end < TREXIO_INVALID_BACK_END);
/* Terminate the back end */
switch (file->back_end) {
@ -681,6 +681,29 @@ interface
end interface
#+end_src
** C helper functions
#+begin_src c :tangle prefix_front.c
trexio_exit_code transform_str (char** dest, const char** src, uint64_t str_max_num, uint32_t str_max_len){
if (dest == NULL) return TREXIO_INVALID_ARG_1;
assert (str_max_num > 0);
assert (str_max_len > 0);
char* tmp_str = (char*)calloc(str_max_num*(str_max_len+1)+1,sizeof(char));
for (int i=0; i<str_max_num; i++){
dest[i] = tmp_str;
strncpy(tmp_str, src[i], str_max_len);
tmp_str += str_max_len + 1;
}
/*tmp_str cannot be freed here but it is taken case of when pointer to dest is deallocated */
return TREXIO_SUCCESS;
}
#+end_src c
* Templates for front end
Consider the following block of ~trex.json~:
@ -729,7 +752,7 @@ end interface
| ~$group_dset_f_dtype_single$~ | Single precision datatype of the dataset [Fortran] | ~real(4)/integer(4)~ |
| ~$group_dset_f_dtype_double$~ | Double precision datatype of the dataset [Fortran] | ~real(8)/integer(8)~ |
| ~$group_dset_f_dims$~ | Dimensions in Fortran | ~(:,:)~ |
Note: parent group name is always added to the child objects upon
construction of TREXIO (e.g. ~num~ of ~nucleus~ group becomes
@ -931,8 +954,8 @@ trexio_has_$group_num$ (trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
assert(file->back_end < TREXIO_INVALID_BACK_END);
assert(file->back_end < TREXIO_INVALID_BACK_END);
switch (file->back_end) {
@ -1034,13 +1057,13 @@ end interface
| Function name | Description | Precision |
|----------------------------------------+-------------------------------------+-----------|
| ~trexio_has_$group$_$group_dset$~ | Check if a dataset exists in a file | --- |
| ~trexio_read_$group$_$group_dset$~ | Read a dataset | Double |
| ~trexio_write_$group$_$group_dset$~ | Write a dataset | Double |
| ~trexio_read_$group$_$group_dset$_32~ | Read a dataset | Single |
| ~trexio_write_$group$_$group_dset$_32~ | Write a dataset | Single |
| ~trexio_read_$group$_$group_dset$_64~ | Read a dataset | Double |
| ~trexio_write_$group$_$group_dset$_64~ | Write a dataset | Double |
| ~trexio_has_$group_dset$~ | Check if a dataset exists in a file | --- |
| ~trexio_read_$group_dset$~ | Read a dataset | Double |
| ~trexio_write_$group_dset$~ | Write a dataset | Double |
| ~trexio_read_$group_dset$_32~ | Read a dataset | Single |
| ~trexio_write_$group_dset$_32~ | Write a dataset | Single |
| ~trexio_read_$group_dset$_64~ | Read a dataset | Double |
| ~trexio_write_$group_dset$_64~ | Write a dataset | Double |
*** C templates for front end
@ -1052,18 +1075,18 @@ end interface
#+begin_src c :tangle hrw_dset_front.h :exports none
trexio_exit_code trexio_has_$group$_$group_dset$(trexio_t* const file);
trexio_exit_code trexio_read_$group$_$group_dset$(trexio_t* const file, $group_dset_dtype_default$* const $group_dset$);
trexio_exit_code trexio_write_$group$_$group_dset$(trexio_t* const file, const $group_dset_dtype_default$* $group_dset$);
trexio_exit_code trexio_read_$group$_$group_dset$_32(trexio_t* const file, $group_dset_dtype_single$* const $group_dset$);
trexio_exit_code trexio_write_$group$_$group_dset$_32(trexio_t* const file, const $group_dset_dtype_single$* $group_dset$);
trexio_exit_code trexio_read_$group$_$group_dset$_64(trexio_t* const file, $group_dset_dtype_double$* const $group_dset$);
trexio_exit_code trexio_write_$group$_$group_dset$_64(trexio_t* const file, const $group_dset_dtype_double$* $group_dset$);
trexio_exit_code trexio_has_$group_dset$(trexio_t* const file);
trexio_exit_code trexio_read_$group_dset$(trexio_t* const file, $group_dset_dtype_default$* const $group_dset$);
trexio_exit_code trexio_write_$group_dset$(trexio_t* const file, const $group_dset_dtype_default$* $group_dset$);
trexio_exit_code trexio_read_$group_dset$_32(trexio_t* const file, $group_dset_dtype_single$* const $group_dset$);
trexio_exit_code trexio_write_$group_dset$_32(trexio_t* const file, const $group_dset_dtype_single$* $group_dset$);
trexio_exit_code trexio_read_$group_dset$_64(trexio_t* const file, $group_dset_dtype_double$* const $group_dset$);
trexio_exit_code trexio_write_$group_dset$_64(trexio_t* const file, const $group_dset_dtype_double$* $group_dset$);
#+end_src
#+begin_src c :tangle read_dset_64_front.c
trexio_exit_code
trexio_read_$group$_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* const $group_dset$)
trexio_read_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_double$* const $group_dset$)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
@ -1079,21 +1102,21 @@ trexio_read_$group$_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_dou
uint32_t rank = $group_dset_rank$;
uint64_t dims[$group_dset_rank$] = {$group_dset_dim_list$};
assert(file->back_end < TREXIO_INVALID_BACK_END);
assert(file->back_end < TREXIO_INVALID_BACK_END);
switch (file->back_end) {
case TREXIO_TEXT:
return trexio_text_read_$group$_$group_dset$(file, $group_dset$, rank, dims);
return trexio_text_read_$group_dset$(file, $group_dset$, rank, dims);
break;
case TREXIO_HDF5:
return trexio_hdf5_read_$group$_$group_dset$(file, $group_dset$, rank, dims);
return trexio_hdf5_read_$group_dset$(file, $group_dset$, rank, dims);
break;
/*
case TREXIO_JSON:
return trexio_json_read_$group$_$group_dset$(file, $group_dset$, rank, dims);
return trexio_json_read_$group_dset$(file, $group_dset$, rank, dims);
break;
,*/
}
@ -1103,7 +1126,7 @@ trexio_read_$group$_$group_dset$_64 (trexio_t* const file, $group_dset_dtype_dou
#+begin_src c :tangle write_dset_64_front.c
trexio_exit_code
trexio_write_$group$_$group_dset$_64 (trexio_t* const file, const $group_dset_dtype_double$* $group_dset$)
trexio_write_$group_dset$_64 (trexio_t* const file, const $group_dset_dtype_double$* $group_dset$)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
@ -1119,21 +1142,21 @@ trexio_write_$group$_$group_dset$_64 (trexio_t* const file, const $group_dset_dt
uint32_t rank = $group_dset_rank$;
uint64_t dims[$group_dset_rank$] = {$group_dset_dim_list$};
assert(file->back_end < TREXIO_INVALID_BACK_END);
assert(file->back_end < TREXIO_INVALID_BACK_END);
switch (file->back_end) {
case TREXIO_TEXT:
return trexio_text_write_$group$_$group_dset$(file, $group_dset$, rank, dims);
return trexio_text_write_$group_dset$(file, $group_dset$, rank, dims);
break;
case TREXIO_HDF5:
return trexio_hdf5_write_$group$_$group_dset$(file, $group_dset$, rank, dims);
return trexio_hdf5_write_$group_dset$(file, $group_dset$, rank, dims);
break;
/*
case TREXIO_JSON:
return trexio_json_write_$group$_$group_dset$(file, $group_dset$, rank, dims);
return trexio_json_write_$group_dset$(file, $group_dset$, rank, dims);
break;
,*/
}
@ -1143,7 +1166,7 @@ trexio_write_$group$_$group_dset$_64 (trexio_t* const file, const $group_dset_dt
#+begin_src c :tangle read_dset_32_front.c
trexio_exit_code
trexio_read_$group$_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_single$* const $group_dset$)
trexio_read_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_single$* const $group_dset$)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
@ -1167,7 +1190,7 @@ trexio_read_$group$_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_sin
$group_dset_dtype_double$* $group_dset$_64 = CALLOC(dim_size, $group_dset_dtype_double$);
if ($group_dset$_64 == NULL) return TREXIO_ALLOCATION_FAILED;
assert(file->back_end < TREXIO_INVALID_BACK_END);
rc = TREXIO_FAILURE;
@ -1175,15 +1198,15 @@ trexio_read_$group$_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_sin
switch (file->back_end) {
case TREXIO_TEXT:
rc = trexio_text_read_$group$_$group_dset$(file, $group_dset$_64, rank, dims);
rc = trexio_text_read_$group_dset$(file, $group_dset$_64, rank, dims);
break;
case TREXIO_HDF5:
rc = trexio_hdf5_read_$group$_$group_dset$(file, $group_dset$_64, rank, dims);
rc = trexio_hdf5_read_$group_dset$(file, $group_dset$_64, rank, dims);
break;
/*
case TREXIO_JSON:
rc = trexio_json_read_$group$_$group_dset$(file, $group_dset$_64, rank, dims);
rc = trexio_json_read_$group_dset$(file, $group_dset$_64, rank, dims);
break;
,*/
}
@ -1204,7 +1227,7 @@ trexio_read_$group$_$group_dset$_32 (trexio_t* const file, $group_dset_dtype_sin
#+begin_src c :tangle write_dset_32_front.c
trexio_exit_code
trexio_write_$group$_$group_dset$_32 (trexio_t* const file, const $group_dset_dtype_single$* $group_dset$)
trexio_write_$group_dset$_32 (trexio_t* const file, const $group_dset_dtype_single$* $group_dset$)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
@ -1233,22 +1256,22 @@ trexio_write_$group$_$group_dset$_32 (trexio_t* const file, const $group_dset_dt
for (uint64_t i=0; i<dim_size; ++i){
$group_dset$_64[i] = ($group_dset_dtype_double$) $group_dset$[i];
}
assert(file->back_end < TREXIO_INVALID_BACK_END);
rc = TREXIO_FAILURE;
switch (file->back_end) {
case TREXIO_TEXT:
rc = trexio_text_write_$group$_$group_dset$(file, $group_dset$_64, rank, dims);
rc = trexio_text_write_$group_dset$(file, $group_dset$_64, rank, dims);
break;
case TREXIO_HDF5:
rc = trexio_hdf5_write_$group$_$group_dset$(file, $group_dset$_64, rank, dims);
rc = trexio_hdf5_write_$group_dset$(file, $group_dset$_64, rank, dims);
break;
/*
case TREXIO_JSON:
rc = trexio_json_write_$group$_$group_dset$(file, $group_dset$_64, rank, dims);
rc = trexio_json_write_$group_dset$(file, $group_dset$_64, rank, dims);
break;
,*/
}
@ -1263,48 +1286,48 @@ trexio_write_$group$_$group_dset$_32 (trexio_t* const file, const $group_dset_dt
#+begin_src c :tangle read_dset_def_front.c
trexio_exit_code
trexio_read_$group$_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const $group_dset$)
trexio_read_$group_dset$ (trexio_t* const file, $group_dset_dtype_default$* const $group_dset$)
{
return trexio_read_$group$_$group_dset$_$default_prec$(file, $group_dset$);
return trexio_read_$group_dset$_$default_prec$(file, $group_dset$);
}
#+end_src
#+begin_src c :tangle write_dset_def_front.c
trexio_exit_code
trexio_write_$group$_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default$* $group_dset$)
trexio_write_$group_dset$ (trexio_t* const file, const $group_dset_dtype_default$* $group_dset$)
{
return trexio_write_$group$_$group_dset$_$default_prec$(file, $group_dset$);
return trexio_write_$group_dset$_$default_prec$(file, $group_dset$);
}
#+end_src
#+begin_src c :tangle has_dset_front.c
trexio_exit_code
trexio_has_$group$_$group_dset$ (trexio_t* const file)
trexio_has_$group_dset$ (trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
assert(file->back_end < TREXIO_INVALID_BACK_END);
switch (file->back_end) {
case TREXIO_TEXT:
return trexio_text_has_$group$_$group_dset$(file);
return trexio_text_has_$group_dset$(file);
break;
case TREXIO_HDF5:
return trexio_hdf5_has_$group$_$group_dset$(file);
return trexio_hdf5_has_$group_dset$(file);
break;
/*
case TREXIO_JSON:
return trexio_json_has_$group$_$group_dset$(file);
return trexio_json_has_$group_dset$(file);
break;
,*/
}
return TREXIO_FAILURE;
}
#+end_src
*** Fortran templates for front end
The ~Fortran~ templates that provide an access to the ~C~ API calls from ~Fortran~.
@ -1312,70 +1335,70 @@ trexio_has_$group$_$group_dset$ (trexio_t* const file)
#+begin_src f90 :tangle write_dset_64_front_fortran.f90
interface
integer function trexio_write_$group$_$group_dset$_64 (trex_file, dset) bind(C)
integer function trexio_write_$group_dset$_64 (trex_file, dset) bind(C)
use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file
$group_dset_f_dtype_double$, intent(in) :: dset$group_dset_f_dims$
end function trexio_write_$group$_$group_dset$_64
end function trexio_write_$group_dset$_64
end interface
#+end_src
#+begin_src f90 :tangle read_dset_64_front_fortran.f90
interface
integer function trexio_read_$group$_$group_dset$_64 (trex_file, dset) bind(C)
integer function trexio_read_$group_dset$_64 (trex_file, dset) bind(C)
use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file
$group_dset_f_dtype_double$, intent(out) :: dset$group_dset_f_dims$
end function trexio_read_$group$_$group_dset$_64
end function trexio_read_$group_dset$_64
end interface
#+end_src
#+begin_src f90 :tangle write_dset_32_front_fortran.f90
interface
integer function trexio_write_$group$_$group_dset$_32 (trex_file, dset) bind(C)
integer function trexio_write_$group_dset$_32 (trex_file, dset) bind(C)
use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file
$group_dset_f_dtype_single$, intent(in) :: dset$group_dset_f_dims$
end function trexio_write_$group$_$group_dset$_32
end function trexio_write_$group_dset$_32
end interface
#+end_src
#+begin_src f90 :tangle read_dset_32_front_fortran.f90
interface
integer function trexio_read_$group$_$group_dset$_32 (trex_file, dset) bind(C)
integer function trexio_read_$group_dset$_32 (trex_file, dset) bind(C)
use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file
$group_dset_f_dtype_single$, intent(out) :: dset$group_dset_f_dims$
end function trexio_read_$group$_$group_dset$_32
end function trexio_read_$group_dset$_32
end interface
#+end_src
#+begin_src f90 :tangle write_dset_def_front_fortran.f90
interface
integer function trexio_write_$group$_$group_dset$ (trex_file, dset) bind(C)
integer function trexio_write_$group_dset$ (trex_file, dset) bind(C)
use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file
$group_dset_f_dtype_default$, intent(in) :: dset$group_dset_f_dims$
end function trexio_write_$group$_$group_dset$
end function trexio_write_$group_dset$
end interface
#+end_src
#+begin_src f90 :tangle read_dset_def_front_fortran.f90
interface
integer function trexio_read_$group$_$group_dset$ (trex_file, dset) bind(C)
integer function trexio_read_$group_dset$ (trex_file, dset) bind(C)
use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file
$group_dset_f_dtype_default$, intent(out) :: dset$group_dset_f_dims$
end function trexio_read_$group$_$group_dset$
end function trexio_read_$group_dset$
end interface
#+end_src
#+begin_src f90 :tangle has_dset_front_fortran.f90
interface
integer function trexio_has_$group$_$group_dset$ (trex_file) bind(C)
integer function trexio_has_$group_dset$ (trex_file) bind(C)
use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file
end function trexio_has_$group$_$group_dset$
end function trexio_has_$group_dset$
end interface
#+end_src

28
src/templates_hdf5/templator_hdf5.org
View File

@ -55,7 +55,7 @@
#+begin_src c :tangle def_hdf5.c
#define $GROUP$_GROUP_NAME "$group$"
#define $GROUP_NUM$_NAME "$group_num$"
#define $GROUP$_$GROUP_DSET$_NAME "$group_dset$"
#define $GROUP_DSET$_NAME "$group_dset$"
#+end_src
** Template for HDF5 structures
@ -282,14 +282,14 @@ trexio_hdf5_has_$group_num$ (trexio_t* const file)
** Template for HDF5 has/read/write a dataset
#+begin_src c :tangle hrw_dset_hdf5.h :exports none
trexio_exit_code trexio_hdf5_has_$group$_$group_dset$(trexio_t* const file);
trexio_exit_code trexio_hdf5_read_$group$_$group_dset$(trexio_t* const file, $group_dset_dtype$* const $group_dset$, const uint32_t rank, const uint64_t* dims);
trexio_exit_code trexio_hdf5_write_$group$_$group_dset$(trexio_t* const file, const $group_dset_dtype$* $group_dset$, const uint32_t rank, const uint64_t* dims);
trexio_exit_code trexio_hdf5_has_$group_dset$(trexio_t* const file);
trexio_exit_code trexio_hdf5_read_$group_dset$(trexio_t* const file, $group_dset_dtype$* const $group_dset$, const uint32_t rank, const uint64_t* dims);
trexio_exit_code trexio_hdf5_write_$group_dset$(trexio_t* const file, const $group_dset_dtype$* $group_dset$, const uint32_t rank, const uint64_t* dims);
#+end_src
#+begin_src c :tangle read_dset_hdf5.c
trexio_exit_code
trexio_hdf5_read_$group$_$group_dset$ (trexio_t* const file, $group_dset_dtype$* const $group_dset$,
trexio_hdf5_read_$group_dset$ (trexio_t* const file, $group_dset_dtype$* const $group_dset$,
const uint32_t rank, const uint64_t* dims)
{
@ -301,14 +301,14 @@ trexio_hdf5_read_$group$_$group_dset$ (trexio_t* const file, $group_dset_dtype$*
herr_t status;
int rrank;
// get the rank of the dataset in a file
status = H5LTget_dataset_ndims (f->$group$_group, $GROUP$_$GROUP_DSET$_NAME, &rrank);
status = H5LTget_dataset_ndims (f->$group$_group, $GROUP_DSET$_NAME, &rrank);
if (status < 0) return TREXIO_FAILURE;
if (rrank != (int) rank) return TREXIO_INVALID_ARG_3;
// open the dataset to get its dimensions
hid_t dset_id = H5Dopen(f->$group$_group, $GROUP$_$GROUP_DSET$_NAME, H5P_DEFAULT);
hid_t dset_id = H5Dopen(f->$group$_group, $GROUP_DSET$_NAME, H5P_DEFAULT);
if (dset_id <= 0) return TREXIO_INVALID_ID;
// allocate space for the dimensions to be read
@ -334,7 +334,7 @@ trexio_hdf5_read_$group$_$group_dset$ (trexio_t* const file, $group_dset_dtype$*
/* High-level H5LT API. No need to deal with dataspaces and datatypes */
status = H5LTread_dataset(f->$group$_group,
$GROUP$_$GROUP_DSET$_NAME,
$GROUP_DSET$_NAME,
H5T_NATIVE_$GROUP_DSET_H5_DTYPE$,
$group_dset$);
if (status < 0) return TREXIO_FAILURE;
@ -345,7 +345,7 @@ trexio_hdf5_read_$group$_$group_dset$ (trexio_t* const file, $group_dset_dtype$*
#+begin_src c :tangle write_dset_hdf5.c
trexio_exit_code
trexio_hdf5_write_$group$_$group_dset$ (trexio_t* const file, const $group_dset_dtype$* $group_dset$,
trexio_hdf5_write_$group_dset$ (trexio_t* const file, const $group_dset_dtype$* $group_dset$,
const uint32_t rank, const uint64_t* dims)
{
@ -360,10 +360,10 @@ trexio_hdf5_write_$group$_$group_dset$ (trexio_t* const file, const $group_dset_
trexio_hdf5_t* f = (trexio_hdf5_t*) file;
if ( H5LTfind_dataset(f->$group$_group, $GROUP$_$GROUP_DSET$_NAME) != 1 ) {
if ( H5LTfind_dataset(f->$group$_group, $GROUP_DSET$_NAME) != 1 ) {
const herr_t status = H5LTmake_dataset(f->$group$_group,
$GROUP$_$GROUP_DSET$_NAME,
$GROUP_DSET$_NAME,
(int) rank, (const hsize_t*) dims,
H5T_NATIVE_$GROUP_DSET_H5_DTYPE$,
$group_dset$);
@ -371,7 +371,7 @@ trexio_hdf5_write_$group$_$group_dset$ (trexio_t* const file, const $group_dset_
} else {
hid_t dset_id = H5Dopen(f->$group$_group, $GROUP$_$GROUP_DSET$_NAME, H5P_DEFAULT);
hid_t dset_id = H5Dopen(f->$group$_group, $GROUP_DSET$_NAME, H5P_DEFAULT);
if (dset_id <= 0) return TREXIO_INVALID_ID;
const herr_t status = H5Dwrite(dset_id,
@ -391,14 +391,14 @@ trexio_hdf5_write_$group$_$group_dset$ (trexio_t* const file, const $group_dset_
#+begin_src c :tangle has_dset_hdf5.c
trexio_exit_code
trexio_hdf5_has_$group$_$group_dset$ (trexio_t* const file)
trexio_hdf5_has_$group_dset$ (trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
trexio_hdf5_t* f = (trexio_hdf5_t*) file;
herr_t status = H5LTfind_dataset(f->$group$_group, $GROUP$_$GROUP_DSET$_NAME);
herr_t status = H5LTfind_dataset(f->$group$_group, $GROUP_DSET$_NAME);
/* H5LTfind_dataset returns 1 if dataset exists, 0 otherwise */
if (status == 1){
return TREXIO_SUCCESS;

5
tools/build_trexio.sh
View File

@ -71,9 +71,10 @@ cd ..
# Populate templates with TREXIO structure according to trex.json file
echo "run generator script to populate templates"
cp ${TOOLS}/generator.py ${SRC}
cp ${TOOLS}/generator.py ${TOOLS}/generator_tools.py ${SRC}
python3 generator.py
rm -f -- ${SRC}/generator.py
rm -f -- ${SRC}/generator.py ${SRC}/generator_tools.py
rm -f -r -- ${SRC}/__pycache__/
# Put pieces of source files together
echo "compile populated files in the lib source files "

449
tools/generator.py
View File

@ -1,406 +1,57 @@
#!/usr/bin/env python3
from generator_tools import *
import json
from os import listdir, scandir, remove
from os.path import isfile, join, dirname, abspath
fileDir = dirname(abspath(__file__))
parentDir = dirname(fileDir)
with open(join(parentDir,'trex.json'), 'r') as f:
config = json.load(f)
# --------------------- GET CONFIGURATION FROM THE TREX.JSON ---------------- #
config_file = 'trex.json'
trex_config = read_json(config_file)
# --------------------------------------------------------------------------- #
# -------------------------------- [WIP] ------------------------------------ #
print('Metadata I/O currently not supported')
# TODO, for now remove metadata-related stuff
del config['metadata']
# for now remove rdm because it is hardcoded
del config['rdm']
groups = [group for group in config.keys()]
dim_variables = {}
dim_list = []
dim_dict = {}
for k1,v1 in config.items():
grname = k1
for v2 in v1.values():
for dim in v2[1]:
if not dim.isdigit():
tmp = dim.replace('.','_')
dim_variables[tmp] = 0
if dim not in dim_list:
dim_list.append(tmp)
dim_dict[grname] = dim_list
dim_list = []
datasets = {}
numbers = {}
for k1,v1 in config.items():
for k2,v2 in v1.items():
if len(v2[1]) > 0:
datasets[f'{k1}_{k2}'] = v2
else:
var_name = f'{k1}_{k2}'
if var_name not in dim_variables.keys():
numbers[var_name] = v2[0]
del trex_config['metadata']
# for now remove rdm from config because it functions are hardcoded
del trex_config['rdm']
# TODO, for now remove char-related stuff
print('Strings I/O currently not supported')
datasets_nostr = {}
for k,v in datasets.items():
tmp_dict = {}
if 'char' not in v[0]:
if v[0] == 'float':
datatype = 'double'
elif v[0] == 'int':
datatype = 'int64_t'
tmp_dict['dtype'] = datatype
tmp_dict['dims'] = [dim.replace('.','_') for dim in v[1]]
tmp_dict['rank'] = len(v[1])
dim_str = tmp_dict['dims'][0]
if tmp_dict['rank'] > 1:
for i in range(1, tmp_dict['rank']):
dim_toadd = tmp_dict['dims'][i]
dim_str += f', {dim_toadd}'
tmp_dict['dim_list'] = dim_str
datasets_nostr[k] = tmp_dict
#put also dimensioning variables in numbers
numbers.update(dim_variables)
templ_path_text = join(fileDir,'templates_text')
templ_path_hdf5 = join(fileDir,'templates_hdf5')
templ_path_front = join(fileDir,'templates_front')
files_exclude = ['prefix_hdf5.c', 'prefix_hdf5.h', 'suffix_hdf5.h',
'prefix_text.c', 'prefix_text.h', 'suffix_text.h',
'prefix_front.c', 'prefix_front.h', 'suffix_front.h',
'prefix_fortran.f90', 'suffix_fortran.f90',
'prefix_s_front.h', 'suffix_s_front.h',
'templator_front.org', 'templator_hdf5.org', 'templator_text.org']
files_text = [f for f in listdir(templ_path_text) if isfile(join(templ_path_text, f)) and f not in files_exclude]
files_hdf5 = [f for f in listdir(templ_path_hdf5) if isfile(join(templ_path_hdf5, f)) and f not in files_exclude]
files_front = [f for f in listdir(templ_path_front) if isfile(join(templ_path_front, f)) and f not in files_exclude]
files = files_text + files_hdf5 + files_front
files_funcs = [f for f in files if 'read_' in f or 'write_' in f or 'flush_' in f or 'free_' in f or 'hrw_' in f or 'has_' in f]
files_funcs_dsets = [f for f in files_funcs if 'dset' in f]
files_funcs_nums = [f for f in files_funcs if 'num' in f]
files_funcs_groups = [f for f in files_funcs if 'group' in f]
files_auxil = [f for f in files if not ('read_' in f or 'write_' in f or 'hrw_' in f or 'has_' in f)]
files_funcs_groups.append('struct_text_group_dset.h')
# build files with functions for text groups
for fname in files_funcs_groups:
fname_new = join('populated',f'pop_{fname}')
if '_text' in fname:
templ_path = templ_path_text
groups_done = []
for group in config.keys():
#grname = group.split('_')[0]
grname = group
if grname in groups_done:
continue
else:
groups_done.append(grname)
subloop = False
do_dset = False
do_num = False
loop_body = ''
dset_allocated = []
with open(join(templ_path,fname), 'r') as f_in :
with open(join(templ_path,fname_new), 'a') as f_out :
for line in f_in :
if 'END REPEAT' in line:
if do_dset:
for dset,params in datasets_nostr.items():
#dset_grname = dset.split('_')[0]
if grname not in dset: #dset_grname != grname:
continue
dset_allocated.append(dset)
templine1 = loop_body.replace('$group_dset_dtype$', params['dtype'])
templine2 = templine1
if 'FREE($group$->$group_dset$)' in loop_body:
tmp_string = ''
for dset_alloc in dset_allocated:
tmp_string += f'FREE({grname}->{dset_alloc});\n '
templine1 = templine2.replace('FREE($group$->$group_dset$);',tmp_string)
templine2 = templine1
templine1 = templine2.replace('$group_dset$', dset)
templine2 = templine1.replace('$group$', grname)
if params['dtype'] == 'double':
std_dtype_out = '24.16e'
std_dtype_in = 'lf'
elif params['dtype'] == 'int64_t':
std_dtype_out = '" PRId64 "'
std_dtype_in = '" SCNd64 "'
templine1 = templine2.replace('$group_dset_std_dtype_out$', std_dtype_out)
templine2 = templine1.replace('$group_dset_std_dtype_in$', std_dtype_in)
f_out.write(templine2)
elif do_num:
#for dim in dim_variables.keys():
for dim in numbers.keys():
#num_grname = dim.split('_')[0]
if grname not in dim: #num_grname != grname:
continue
templine1 = loop_body.replace('$group_num$', dim)
templine2 = templine1.replace('$group$', grname)
f_out.write(templine2)
else:
print('fishy')
loop_body = ''
dset_allocated = []
subloop = False
do_dset = False
do_num = False
continue
if subloop:
loop_body += line
if 'START REPEAT' in line:
if 'GROUP_DSET' in line:
do_dset = True
if 'GROUP_NUM' in line:
do_num = True
subloop = True
if '$group_dset' in line and not subloop:
for dset,params in datasets_nostr.items():
#dset_grname = dset.split('_')[0]
if grname not in dset: #dset_grname != grname:
continue
templine1 = line.replace('$group_dset$', dset)
templine2 = templine1
templine1 = templine2.replace('$group_dset_dtype$', params['dtype'])
templine2 = templine1
templine1 = templine2.replace('$group$', grname)
templine2 = templine1.replace('$GROUP$', grname.upper())
f_out.write(templine2)
elif '$group_num' in line and not subloop:
#for dim in dim_variables.keys():
for dim in numbers.keys():
#num_grname = dim.split('_')[0]
if grname not in dim: #num_grname != grname:
continue
templine1 = line.replace('$GROUP_NUM$', dim.upper())
templine2 = templine1.replace('$group_num$', dim)
templine1 = templine2.replace('$group$', grname)
templine2 = templine1.replace('$GROUP$', grname.upper())
f_out.write(templine2)
elif '$group$' in line and not subloop:
templine1 = line.replace('$group$', grname)
templine2 = templine1.replace('$GROUP$', grname.upper())
f_out.write(templine2)
elif not subloop:
f_out.write(line)
# build files with functions
for fname in files_funcs_dsets:
fname_new = join('populated',f'pop_{fname}')
if '_hdf5' in fname:
templ_path = templ_path_hdf5
if '_front' in fname:
templ_path = templ_path_front
if '_text' in fname:
templ_path = templ_path_text
for dset,params in datasets_nostr.items():
#grname = dset.split('_')[0]
# the problem was when group name has underscores in it, special case needed!
for group_tmp in config.keys():
if group_tmp in dset:
grname = group_tmp
with open(join(templ_path,fname), 'r') as f_in :
with open(join(templ_path,fname_new), 'a') as f_out :
num_written = []
for line in f_in :
if '$' in line:
if '$group_dset_dim$' in line:
rc_line = 'if (rc != TREXIO_SUCCESS) return rc;\n'
indentlevel = len(line) - len(line.lstrip())
for dim in params['dims']:
if not dim.isdigit() and not dim in num_written:
num_written.append(dim)
templine1 = line.replace('$group_dset_dim$', dim)
templine2 = templine1
if '_read' in templine2: # and 'hdf5' in fname:
templine1 = indentlevel*" " + rc_line
templine2 += templine1
f_out.write(templine2)
num_written = []
continue
templine1 = line.replace('$GROUP$_$GROUP_DSET$', dset.upper())
templine2 = templine1.replace('$group$_$group_dset$', dset)
templine1 = templine2.replace('$group_dset$', dset)
templine2 = templine1
templine1 = templine2.replace('$group_dset_dtype$', params['dtype'])
templine2 = templine1
if params['dtype'] == 'double':
h5_dtype = 'double'
f_dtype_default= 'real(8)'
f_dtype_double = 'real(8)'
f_dtype_single = 'real(4)'
c_dtype_default= 'double'
c_dtype_double = 'double'
c_dtype_single = 'float'
default_prec = '64'
elif params['dtype'] == 'int64_t':
h5_dtype = 'int64'
f_dtype_default= 'integer(4)'
f_dtype_double = 'integer(8)'
f_dtype_single = 'integer(4)'
c_dtype_default= 'int32_t'
c_dtype_double = 'int64_t'
c_dtype_single = 'int32_t'
default_prec = '32'
templine1 = templine2.replace('$group_dset_dtype_double$', c_dtype_double)
templine2 = templine1.replace('$group_dset_dtype_single$', c_dtype_single)
templine1 = templine2.replace('$group_dset_h5_dtype$', h5_dtype)
templine2 = templine1.replace('$group_dset_h5_dtype$'.upper(), h5_dtype.upper())
templine1 = templine2.replace('$group_dset_f_dtype_double$', f_dtype_double)
templine2 = templine1.replace('$group_dset_f_dtype_single$', f_dtype_single)
templine1 = templine2.replace('$group_dset_f_dtype_default$', f_dtype_default)
templine2 = templine1.replace('$group_dset_dtype_default$', c_dtype_default)
templine1 = templine2.replace('$default_prec$', default_prec)
templine2 = templine1
templine1 = templine2.replace('$group_dset_rank$', str(params['rank']))
templine2 = templine1
templine1 = templine2.replace('$group_dset_dim_list$', params['dim_list'])
templine2 = templine1
templine1 = templine2.replace('$group$', grname)
templine2 = templine1.replace('$GROUP$', grname.upper())
if "$group_dset_f_dims$" in templine2:
dims = "(" + ",".join([":" for _ in range(params['rank'])]) + ")"
if dims == "()":
dims = ""
else:
dims = "(*)"
templine1 = templine2.replace("$group_dset_f_dims$", dims)
templine2 = templine1
f_out.write(templine2)
else:
f_out.write(line)
# build files with functions
for fname in files_funcs_nums:
fname_new = join('populated',f'pop_{fname}')
if '_hdf5' in fname:
templ_path = templ_path_hdf5
if '_front' in fname:
templ_path = templ_path_front
if '_text' in fname:
templ_path = templ_path_text
#for dim in dim_variables.keys():
for dim in numbers.keys():
grname = dim.split('_')[0]
with open(join(templ_path,fname), 'r') as f_in :
with open(join(templ_path,fname_new), 'a') as f_out :
for line in f_in :
if '$' in line:
templine1 = line.replace('$GROUP_NUM$', dim.upper())
templine2 = templine1.replace('$group_num$', dim)
templine1 = templine2.replace('$group$', grname)
templine2 = templine1.replace('$GROUP$', grname.upper())
f_out.write(templine2)
else:
f_out.write(line)
# build files with $group$ and $group$-based
for fname in ['def_hdf5.c', 'basic_hdf5.c', 'basic_text_group.c',
'struct_hdf5.h', 'struct_text_group.h'] :
fname_new = join('populated',f'pop_{fname}')
if '_hdf5' in fname:
templ_path = templ_path_hdf5
if '_front' in fname:
templ_path = templ_path_front
if '_text' in fname:
templ_path = templ_path_text
with open(join(templ_path,fname), 'r') as f_in :
with open(join(templ_path,fname_new), 'a') as f_out :
for line in f_in :
if '$group_dset$' in line or '$GROUP_DSET$' in line :
for dset in datasets_nostr.keys():
templine1 = line.replace('$GROUP$_$GROUP_DSET$', dset.upper())
templine2 = templine1.replace('$group_dset$', dset)
f_out.write(templine2)
elif '$group_num$' in line or '$GROUP_NUM$' in line :
#for num in dim_variables.keys():
for num in numbers.keys():
templine1 = line.replace('$GROUP_NUM$', num.upper())
templine2 = templine1.replace('$group_num$', num)
f_out.write(templine2)
# special case for proper error handling when deallocting text groups
elif 'rc = trexio_text_free_$group$' in line:
for grname in config.keys():
templine1 = line.replace('$group$', grname)
templine2 = templine1 + ' if (rc != TREXIO_SUCCESS) return rc;\n'
f_out.write(templine2)
elif '$group$' in line or '$GROUP$' in line :
for grname in config.keys():
templine1 = line.replace('$group$', grname)
templine2 = templine1.replace('$GROUP$', grname.upper())
f_out.write(templine2)
else:
f_out.write(line)
# --------------------------------------------------------------------------- #
# -------------------- GET ATTRIBUTES FROM THE CONFIGURATION ---------------- #
group_dict = get_group_dict(trex_config)
detailed_num = get_detailed_num_dict(trex_config)
# helper dictionaries contain group, num or dset names as keys
datasets = get_dset_dict(trex_config)
detailed_dset_nostr, detailed_dset_str = split_dset_dict_detailed(datasets)
# consistency check for dimensioning variables
check_dim_consistency(detailed_num, datasets)
# --------------------------------------------------------------------------- #
# -------------------- GET TEMPLATED FILES TO BE POPULATED ------------------ #
source = ['front', 'text', 'hdf5']
# build helper dictionaries with paths per source directory
template_paths = get_template_paths(source)
# build helper dictionaries with source files per source directory
source_files = get_source_files(template_paths)
# build helper dictionaries with templated files
files_todo = get_files_todo(source_files)
# --------------------------------------------------------------------------- #
# ----------------------- POPULATE TEMPLATED FILES -------------------------- #
# populate files with iterative scheme, i.e. for unique functions
for fname in files_todo['auxiliary']:
iterative_populate_file(fname, template_paths, group_dict, detailed_dset_nostr, detailed_num)
# populate has/read/write_num functions with recursive scheme
for fname in files_todo['num']:
recursive_populate_file(fname, template_paths, detailed_num)
# populate has/read/write_dset functions with recursive scheme
for fname in files_todo['dset']:
recursive_populate_file(fname, template_paths, detailed_dset_nostr)
# populate group-related functions with mixed (iterative+recursive) scheme [text backend]
for fname in files_todo['group']:
special_populate_text_group(fname, template_paths, group_dict, detailed_dset_nostr, detailed_num)
# --------------------------------------------------------------------------- #

558
tools/generator_tools.py Normal file
View File

@ -0,0 +1,558 @@
from os import listdir
from os.path import join, dirname, abspath, isfile
from json import load as json_load
def read_json(fname: str) -> dict:
"""
Read configuration from the input `fname` JSON file.
Parameters:
fname (str) : JSON file name
Returns:
config (dict) : full configuration dictionary loaded from the input file
"""
fileDir = dirname(abspath(__file__))
parentDir = dirname(fileDir)
with open(join(parentDir,fname), 'r') as f:
config = json_load(f)
return config
def get_files_todo(source_files: dict) -> dict:
"""
Build dictionaries of templated files per objective.
Parameters:
source_files (dict) : dictionary with source files per source directory
Returns:
file_todo (dict) : dictionary with objective title : [list of files] as key-value pairs
"""
all_files = []
for key in source_files.keys():
all_files += source_files[key]
files_todo = {}
#files_todo['all'] = list(filter(lambda x: 'read' in x or 'write' in x or 'has' in x or 'hrw' in x or 'flush' in x or 'free' in x, all_files))
files_todo['all'] = [f for f in all_files if 'read' in f or 'write' in f or 'has' in f or 'flush' in f or 'free' in f or 'hrw' in f]
for key in ['dset', 'num', 'group']:
files_todo[key] = list(filter(lambda x: key in x, files_todo['all']))
files_todo['group'].append('struct_text_group_dset.h')
# files that correspond to todo1 group (e.g. only iterative population within the function body)
files_todo['auxiliary'] = ['def_hdf5.c', 'basic_hdf5.c', 'basic_text_group.c', 'struct_hdf5.h', 'struct_text_group.h']
return files_todo
def get_source_files(paths: dict) -> dict:
"""
Build dictionaries of all files per source directory.
Parameters:
paths (dict) : dictionary with paths to source directories
Returns:
file_dict (dict) : dictionary with source title : [list of files] as key-value pairs
"""
file_dict = {}
for key in paths.keys():
file_dict[key] = [f for f in listdir(paths[key]) if isfile(join(paths[key], f))]
return file_dict
def get_template_paths(source: list) -> dict:
"""
Build dictionary of the absolute paths to directory with templates per source.
Parameters:
source (list) : list of source titles, i.e. ['front', 'text', 'hdf5']
Returns:
path_dict (dict) : dictionary with source title : absolute path as key-value pairs
"""
fileDir = dirname(abspath(__file__))
path_dict = {}
for dir in source:
path_dict[dir] = join(fileDir,f'templates_{dir}')
return path_dict
def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> None:
"""
Populate files containing basic read/write/has functions.
Parameters:
filename (str) : template file to be populated
paths (dict) : dictionary of paths per source directory
detailed_source (dict) : dictionary of variables with substitution details (usually either datasets or numbers)
Returns:
None
"""
fname_new = join('populated',f'pop_{fname}')
templ_path = get_template_path(fname, paths)
triggers = ['group_dset_dtype', 'group_dset_h5_dtype', 'default_prec',
'group_dset_f_dtype_default', 'group_dset_f_dtype_double', 'group_dset_f_dtype_single',
'group_dset_dtype_default', 'group_dset_dtype_double', 'group_dset_dtype_single',
'group_dset_rank', 'group_dset_dim_list', 'group_dset_f_dims',
'group_dset', 'group_num', 'group']
for item in detailed_source.keys():
with open(join(templ_path,fname), 'r') as f_in :
with open(join(templ_path,fname_new), 'a') as f_out :
num_written = []
for line in f_in :
# special case to add error handling for read/write of dimensioning variables
if '$group_dset_dim$' in line:
rc_line = 'if (rc != TREXIO_SUCCESS) return rc;\n'
indentlevel = len(line) - len(line.lstrip())
for dim in detailed_source[item]['dims']:
if not dim.isdigit() and not dim in num_written:
num_written.append(dim)
templine = line.replace('$group_dset_dim$', dim)
if '_read' in templine:
line_toadd = indentlevel*" " + rc_line
templine += line_toadd
f_out.write(templine)
num_written = []
continue
# general case of recursive replacement of inline triggers
else:
populated_line = recursive_replace_line(line, triggers, detailed_source[item])
f_out.write(populated_line)
def recursive_replace_line (input_line: str, triggers: list, source: dict) -> str:
"""
Recursive replacer. Recursively calls itself as long as there is at least one "$" present in the `input_line`.
Parameters:
input_line (str) : input line
triggers (list) : list of triggers (templated variables to be replaced)
source (dict) : dictionary of variables with substitution details (usually either datasets or numbers)
Returns:
output_line (str) : processed (replaced) line
"""
is_triggered = False
output_line = input_line
if '$' in input_line:
for case in triggers:
test_case = f'${case}$'
if test_case in input_line:
output_line = input_line.replace(test_case, source[case])
is_triggered = True
break
elif test_case.upper() in input_line:
output_line = input_line.replace(test_case.upper(), source[case].upper())
is_triggered = True
break
if is_triggered:
return recursive_replace_line(output_line, triggers, source)
else:
print(output_line)
raise ValueError('Recursion went wrong, not all cases considered')
return output_line
def iterative_populate_file (filename: str, paths: dict, groups: dict, datasets: dict, numbers: dict) -> None:
"""
Iteratively populate files with unique functions that contain templated variables.
Parameters:
filename (str) : template file to be populated
paths (dict) : dictionary of paths per source directory
groups (dict) : dictionary of groups
datasets (dict) : dictionary of datasets with substitution details
numbers (dict) : dictionary of numbers with substitution details
Returns:
None
"""
add_trigger = 'rc = trexio_text_free_$group$'
triggers = [add_trigger, '$group_dset$', '$group_num$', '$group$']
templ_path = get_template_path(filename, paths)
filename_out = join('populated',f'pop_{filename}')
# Note: it is important that special conditions like add_trigger above will be checked before standard triggers
# that contain only basic $-ed variable (like $group$). Otherwise, the standard triggers will be removed
# from the template and the special condition will never be met.
with open(join(templ_path,filename), 'r') as f_in :
with open(join(templ_path,filename_out), 'a') as f_out :
for line in f_in :
id = check_triggers(line, triggers)
if id == 0:
# special case for proper error handling when deallocting text groups
error_handler = ' if (rc != TREXIO_SUCCESS) return rc;\n'
populated_line = iterative_replace_line(line, triggers[3], groups, add_line=error_handler)
f_out.write(populated_line)
elif id == 1:
populated_line = iterative_replace_line(line, triggers[id], datasets, None)
f_out.write(populated_line)
elif id == 2:
populated_line = iterative_replace_line(line, triggers[id], numbers, None)
f_out.write(populated_line)
elif id == 3:
populated_line = iterative_replace_line(line, triggers[id], groups, None)
f_out.write(populated_line)
else:
f_out.write(line)
def iterative_replace_line (input_line: str, case: str, source: dict, add_line: str) -> str:
"""
Iterative replacer. Iteratively copy-pastes `input_line` each time with a new substitution of a templated variable depending on the `case`.
Parameters:
input_line (str) : input line
case (str) : single trigger case (templated variable to be replaced)
source (dict) : dictionary of variables with substitution details
add_line (str) : special line to be added (e.g. for error handling)
Returns:
output_block (str) : processed (replaced) block of text
"""
output_block = ""
for item in source.keys():
templine1 = input_line.replace(case.upper(), item.upper())
templine2 = templine1.replace(case, item)
if add_line != None:
templine2 += add_line
output_block += templine2
return output_block
def check_triggers (input_line: str, triggers: list) -> int:
"""
Check the presence of the trigger in the `input_line`.
Parameters:
input_line (str) : string to be checked
triggers (list) : list of triggers (templated variables)
Returns:
out_id (int) : id of the trigger item in the list
"""
out_id = -1
for id,trig in enumerate(triggers):
if trig in input_line or trig.upper() in input_line:
out_id = id
return out_id
return out_id
def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detailed_dset: dict, detailed_numbers: dict) -> None:
"""
Special population for group-related functions in the TEXT back end.
Parameters:
fname (str) : template file to be populated
paths (dict) : dictionary of paths per source directory
group_dict (dict) : dictionary of groups
detailed_dset (dict) : dictionary of datasets with substitution details
detailed_numbers (dict) : dictionary of numbers with substitution details
Returns:
None
"""
fname_new = join('populated',f'pop_{fname}')
templ_path = get_template_path(fname, paths)
triggers = ['group_dset_dtype', 'group_dset_std_dtype_out', 'group_dset_std_dtype_in',
'group_dset', 'group_num', 'group']
for group in group_dict.keys():
with open(join(templ_path,fname), 'r') as f_in :
with open(join(templ_path,fname_new), 'a') as f_out :
subloop_dset = False
subloop_num = False
loop_body = ''
dset_allocated = []
for line in f_in :
if 'START REPEAT GROUP_DSET' in line:
subloop_dset = True
continue
elif 'START REPEAT GROUP_NUM' in line:
subloop_num = True
continue
if 'END REPEAT GROUP_DSET' in line:
for dset in detailed_dset.keys():
if group != detailed_dset[dset]['group']:
continue
dset_allocated.append(dset)
if 'FREE($group$->$group_dset$)' in loop_body:
tmp_string = ''
for dset_alloc in dset_allocated:
tmp_string += f'FREE({group}->{dset_alloc});\n '
tmp_body = loop_body.replace('FREE($group$->$group_dset$);',tmp_string)
populated_body = recursive_replace_line(tmp_body, triggers, detailed_dset[dset])
f_out.write(populated_body)
else:
save_body = loop_body
populated_body = recursive_replace_line(save_body, triggers, detailed_dset[dset])
f_out.write(populated_body)
subloop_dset = False
loop_body = ''
dset_allocated = []
continue
elif 'END REPEAT GROUP_NUM' in line:
for dim in detailed_numbers.keys():
if group != detailed_numbers[dim]['group']:
continue
save_body = loop_body
populated_body = recursive_replace_line(save_body, triggers, detailed_numbers[dim])
f_out.write(populated_body)
subloop_num = False
loop_body = ''
continue
if not subloop_num and not subloop_dset:
# NORMAL CASE WITHOUT SUBLOOPS
if '$group_dset' in line:
for dset in detailed_dset.keys():
if group != detailed_dset[dset]['group']:
continue
populated_line = recursive_replace_line(line, triggers, detailed_dset[dset])
f_out.write(populated_line)
elif '$group_num$' in line:
for dim in detailed_numbers.keys():
if group != detailed_numbers[dim]['group']:
continue
populated_line = recursive_replace_line(line, triggers, detailed_numbers[dim])
f_out.write(populated_line)
elif '$group$' in line:
populated_line = line.replace('$group$', group)
f_out.write(populated_line)
else:
f_out.write(line)
else:
loop_body += line
def get_template_path (filename: str, path_dict: dict) -> str:
"""
Returns the absolute path to the directory with indicated `filename` template.
Parameters:
filename (str) : template file to be populated
path_dict (dict) : dictionary of paths per source directory
Returns:
path (str) : resulting path
"""
for dir_type in path_dict.keys():
if dir_type in filename:
path = path_dict[dir_type]
return path
raise ValueError('Filename should contain one of the keywords')
def get_group_dict (configuration: dict) -> dict:
"""
Returns the dictionary of all groups.
Parameters:
configuration (dict) : configuration from `trex.json`
Returns:
group_dict (dict) : dictionary of groups
"""
group_dict = {}
for k in configuration.keys():
group_dict[k] = 0
return group_dict
def get_detailed_num_dict (configuration: dict) -> dict:
"""
Returns the dictionary of all `num`-suffixed variables.
Keys are names, values are subdictionaries containing corresponding group and group_num names.
Parameters:
configuration (dict) : configuration from `trex.json`
Returns:
num_dict (dict) : dictionary of num-suffixed variables
"""
num_dict = {}
for k1,v1 in configuration.items():
for k2,v2 in v1.items():
if len(v2[1]) == 0:
tmp_num = f'{k1}_{k2}'
if 'str' not in v2[0]:
tmp_dict = {}
tmp_dict['group'] = k1
tmp_dict['group_num'] = tmp_num
num_dict[tmp_num] = tmp_dict
return num_dict
def get_dset_dict (configuration: dict) -> dict:
"""
Returns the dictionary of datasets.
Keys are names, values are lists containing datatype, list of dimensions and group name
Parameters:
configuration (dict) : configuration from `trex.json`
Returns:
dset_dict (dict) : dictionary of datasets
"""
dset_dict = {}
for k1,v1 in configuration.items():
for k2,v2 in v1.items():
if len(v2[1]) != 0:
tmp_dset = f'{k1}_{k2}'
dset_dict[tmp_dset] = v2
# append a group name for postprocessing
dset_dict[tmp_dset].append(k1)
return dset_dict
def split_dset_dict_detailed (datasets: dict) -> tuple:
"""
Returns the detailed dictionary of datasets.
Keys are names, values are subdictionaries containing substitutes for templated variables
Parameters:
configuration (dict) : configuration from `trex.json`
Returns:
dset_numeric_dict, dset_string_dict (tuple) : dictionaries corresponding to all numeric- and string-based datasets, respectively.
"""
dset_numeric_dict = {}
dset_string_dict = {}
for k,v in datasets.items():
# create a temp dictionary
tmp_dict = {}
# specify details required to replace templated variables later
if v[0] == 'float':
datatype = 'double'
group_dset_h5_dtype = 'double'
group_dset_f_dtype_default= 'real(8)'
group_dset_f_dtype_double = 'real(8)'
group_dset_f_dtype_single = 'real(4)'
group_dset_dtype_default= 'double'
group_dset_dtype_double = 'double'
group_dset_dtype_single = 'float'
default_prec = '64'
group_dset_std_dtype_out = '24.16e'
group_dset_std_dtype_in = 'lf'
elif v[0] == 'int':
datatype = 'int64_t'
group_dset_h5_dtype = 'int64'
group_dset_f_dtype_default= 'integer(4)'
group_dset_f_dtype_double = 'integer(8)'
group_dset_f_dtype_single = 'integer(4)'
group_dset_dtype_default= 'int32_t'
group_dset_dtype_double = 'int64_t'
group_dset_dtype_single = 'int32_t'
default_prec = '32'
group_dset_std_dtype_out = '" PRId64 "'
group_dset_std_dtype_in = '" SCNd64 "'
elif v[0] == 'str':
# TODO
datatype = 'string'
# add the dset name for templates
tmp_dict['group_dset'] = k
# add the datatypes for templates
tmp_dict['dtype'] = datatype
tmp_dict['group_dset_dtype'] = datatype
tmp_dict['group_dset_h5_dtype'] = group_dset_h5_dtype
tmp_dict['group_dset_f_dtype_default'] = group_dset_f_dtype_default
tmp_dict['group_dset_f_dtype_double'] = group_dset_f_dtype_double
tmp_dict['group_dset_f_dtype_single'] = group_dset_f_dtype_single
tmp_dict['group_dset_dtype_default'] = group_dset_dtype_default
tmp_dict['group_dset_dtype_double'] = group_dset_dtype_double
tmp_dict['group_dset_dtype_single'] = group_dset_dtype_single
tmp_dict['default_prec'] = default_prec
tmp_dict['group_dset_std_dtype_in'] = group_dset_std_dtype_in
tmp_dict['group_dset_std_dtype_out'] = group_dset_std_dtype_out
# add the rank
tmp_dict['rank'] = len(v[1])
tmp_dict['group_dset_rank'] = str(tmp_dict['rank'])
# add the list of dimensions
tmp_dict['dims'] = [dim.replace('.','_') for dim in v[1]]
# build a list of dimensions to be inserted in the dims array initialization, e.g. {ao_num, ao_num}
dim_list = tmp_dict['dims'][0]
if tmp_dict['rank'] > 1:
for i in range(1, tmp_dict['rank']):
dim_toadd = tmp_dict['dims'][i]
dim_list += f', {dim_toadd}'
tmp_dict['group_dset_dim_list'] = dim_list
if tmp_dict['rank'] == 0:
dim_f_list = ""
else:
dim_f_list = "(*)"
tmp_dict['group_dset_f_dims'] = dim_f_list
# add group name as a key-value pair to the dset dict
tmp_dict['group'] = v[2]
# split datasets in numeric- and string- based
if (datatype == 'string'):
dset_string_dict[k] = tmp_dict
else:
dset_numeric_dict[k] = tmp_dict
return (dset_numeric_dict, dset_string_dict)
def check_dim_consistency(num: dict, dset: dict) -> None:
"""
Consistency check to make sure that each dimensioning variable exists as a num attribute of some group.
Parameters:
num (dict) : dictionary of num-suffixed variables
dset (dict) : dictionary of datasets
Returns:
None
"""
dim_tocheck = []
for v in dset.values():
tmp_dim_list = [dim.replace('.','_') for dim in v[1] if not dim.isdigit()]
for dim in tmp_dim_list:
if dim not in dim_tocheck:
dim_tocheck.append(dim)
for dim in dim_tocheck:
if not dim in num.keys():
raise ValueError(f"Dimensioning variable {dim} is not a num attribute of any group.\n")