TREX/trexio
1
0
Fork 0
mirror of https://github.com/TREX-CoE/trexio.git synced 2025-04-28 11:24:44 +02:00
Code Issues Projects Releases Wiki Activity

[WIP] more general templated variables for sparse data

Browse Source 
+ renamed templated variables with std_dtype_in|out suffix into _format_scanf|printf for clarity
This commit is contained in:
q-posev 2021-12-03 15:42:40 +01:00
parent c520175cbe
commit ed3bde973e
3 changed files with 101 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
src/templates_front/templator_front.org
View File

@ -1200,8 +1200,8 @@ def close(trexio_file):
| ~$group_dset_dim_list$~ | All dimensions of the dataset | ~{nucleus_num, 3}~ | | ~$group_dset_dim_list$~ | All dimensions of the dataset | ~{nucleus_num, 3}~ |
| ~$group_dset_dtype$~ | Basic type of the dataset (int/float/char) | ~float~ | | ~$group_dset_dtype$~ | Basic type of the dataset (int/float/char) | ~float~ |
| ~$group_dset_h5_dtype$~ | Type of the dataset in HDF5 | ~double~ | | ~$group_dset_h5_dtype$~ | Type of the dataset in HDF5 | ~double~ |
| ~$group_dset_std_dtype_in$~ | Input type of the dataset in TEXT [fscanf] | ~%lf~ | | ~$group_dset_format_scanf$~ | Input type of the dataset in TEXT [fscanf] | ~%lf~ |
| ~$group_dset_std_dtype_out$~ | Output type of the dataset in TEXT [fprintf] | ~%24.16e~ | | ~$group_dset_format_printf$~ | Output type of the dataset in TEXT [fprintf] | ~%24.16e~ |
| ~$group_dset_dtype_default$~ | Default datatype of the dataset [C] | ~double/int32_t~ | | ~$group_dset_dtype_default$~ | Default datatype of the dataset [C] | ~double/int32_t~ |
| ~$group_dset_dtype_single$~ | Single precision datatype of the dataset [C] | ~float/int32_t~ | | ~$group_dset_dtype_single$~ | Single precision datatype of the dataset [C] | ~float/int32_t~ |
| ~$group_dset_dtype_double$~ | Double precision datatype of the dataset [C] | ~double/int64_t~ | | ~$group_dset_dtype_double$~ | Double precision datatype of the dataset [C] | ~double/int64_t~ |
@ -2413,17 +2413,17 @@ def has_$group_dset$(trexio_file) -> bool:
**** Function declarations **** Function declarations
#+begin_src c :tangle hrw_dset_sparse_front.h :exports none #+begin_src c :tangle hrw_dset_sparse_front.h :exports none
trexio_exit_code trexio_has_$group_sparse_dset$(trexio_t* const file); trexio_exit_code trexio_has_$group_dset$(trexio_t* const file);
trexio_exit_code trexio_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t buffer_size, int32_t* const index_sparse, double* const value_sparse); trexio_exit_code trexio_read_$group_dset$(trexio_t* const file, const int64_t offset_file, const int64_t buffer_size, int32_t* const index_sparse, double* const value_sparse);
trexio_exit_code trexio_read_$group_sparse_dset$_size(trexio_t* const file, int64_t* const size_max); trexio_exit_code trexio_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max);
trexio_exit_code trexio_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t buffer_size, const int32_t* index_sparse, const double* value_sparse); trexio_exit_code trexio_write_$group_dset$(trexio_t* const file, const int64_t offset_file, const int64_t buffer_size, const int32_t* index_sparse, const double* value_sparse);
#+end_src #+end_src
**** Source code for default functions **** Source code for default functions
#+begin_src c :tangle read_dset_sparse_front.c #+begin_src c :tangle read_dset_sparse_front.c
trexio_exit_code trexio_exit_code
trexio_read_$group_sparse_dset$(trexio_t* const file, trexio_read_$group_dset$(trexio_t* const file,
const int64_t offset_file, const int64_t offset_file,
const int64_t buffer_size, const int64_t buffer_size,
int32_t* const index_sparse, int32_t* const index_sparse,
@ -2435,7 +2435,7 @@ trexio_read_$group_sparse_dset$(trexio_t* const file,
if (buffer_size <= 0L) return TREXIO_INVALID_ARG_3; if (buffer_size <= 0L) return TREXIO_INVALID_ARG_3;
if (index_sparse == NULL) return TREXIO_INVALID_ARG_4; if (index_sparse == NULL) return TREXIO_INVALID_ARG_4;
if (value_sparse == NULL) return TREXIO_INVALID_ARG_5; if (value_sparse == NULL) return TREXIO_INVALID_ARG_5;
if (trexio_has_$group_sparse_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING;
const uint32_t rank = $group_dset_rank$; // To be set by generator : number of indices const uint32_t rank = $group_dset_rank$; // To be set by generator : number of indices
@ -2443,7 +2443,7 @@ trexio_read_$group_sparse_dset$(trexio_t* const file,
trexio_exit_code rc; trexio_exit_code rc;
/* Read the max number of integrals stored in the file */ /* Read the max number of integrals stored in the file */
rc = trexio_read_$group_sparse_dset$_size(file, &size_max); rc = trexio_read_$group_dset$_size(file, &size_max);
if (rc != TREXIO_SUCCESS) return rc; if (rc != TREXIO_SUCCESS) return rc;
/* Cannot read more data points than there is already in the file */ /* Cannot read more data points than there is already in the file */
if (buffer_size > size_max) return TREXIO_INVALID_ARG_3; if (buffer_size > size_max) return TREXIO_INVALID_ARG_3;
@ -2451,19 +2451,19 @@ trexio_read_$group_sparse_dset$(trexio_t* const file,
switch (file->back_end) { switch (file->back_end) {
case TREXIO_TEXT: case TREXIO_TEXT:
return trexio_text_read_$group_sparse_dset$(file, offset_file, buffer_size, size_max, index_sparse, value_sparse); return trexio_text_read_$group_dset$(file, offset_file, buffer_size, size_max, index_sparse, value_sparse);
break; break;
case TREXIO_HDF5: case TREXIO_HDF5:
#ifdef HAVE_HDF5 #ifdef HAVE_HDF5
return trexio_hdf5_read_$group_sparse_dset$(file, offset_file, buffer_size, size_max, index_sparse, value_sparse); return trexio_hdf5_read_$group_dset$(file, offset_file, buffer_size, size_max, index_sparse, value_sparse);
break; break;
#else #else
return TREXIO_BACK_END_MISSING; return TREXIO_BACK_END_MISSING;
#endif #endif
/* /*
case TREXIO_JSON: case TREXIO_JSON:
return trexio_json_read_$group_sparse_dset$(...); return trexio_json_read_$group_dset$(...);
break; break;
,*/ ,*/
default: default:
@ -2475,27 +2475,27 @@ trexio_read_$group_sparse_dset$(trexio_t* const file,
#+begin_src c :tangle read_dset_sparse_size_front.c #+begin_src c :tangle read_dset_sparse_size_front.c
trexio_exit_code trexio_exit_code
trexio_read_$group_sparse_dset$_size(trexio_t* const file, int64_t* const size_max) trexio_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max)
{ {
if (file == NULL) return TREXIO_INVALID_ARG_1; if (file == NULL) return TREXIO_INVALID_ARG_1;
if (trexio_has_$group_sparse_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING; if (trexio_has_$group_dset$(file) != TREXIO_SUCCESS) return TREXIO_DSET_MISSING;
switch (file->back_end) { switch (file->back_end) {
case TREXIO_TEXT: case TREXIO_TEXT:
return trexio_text_read_$group_sparse_dset$_size(file, size_max); return trexio_text_read_$group_dset$_size(file, size_max);
break; break;
case TREXIO_HDF5: case TREXIO_HDF5:
#ifdef HAVE_HDF5 #ifdef HAVE_HDF5
return trexio_hdf5_read_$group_sparse_dset$_size(file, size_max); return trexio_hdf5_read_$group_dset$_size(file, size_max);
break; break;
#else #else
return TREXIO_BACK_END_MISSING; return TREXIO_BACK_END_MISSING;
#endif #endif
/* /*
case TREXIO_JSON: case TREXIO_JSON:
return trexio_json_read_$group_sparse_dset$_size(...); return trexio_json_read_$group_dset$_size(...);
break; break;
,*/ ,*/
default: default:
@ -2507,7 +2507,7 @@ trexio_read_$group_sparse_dset$_size(trexio_t* const file, int64_t* const size_m
#+begin_src c :tangle write_dset_sparse_front.c #+begin_src c :tangle write_dset_sparse_front.c
trexio_exit_code trexio_exit_code
trexio_write_$group_sparse_dset$(trexio_t* const file, trexio_write_$group_dset$(trexio_t* const file,
const int64_t offset_file, const int64_t offset_file,
const int64_t buffer_size, const int64_t buffer_size,
const int32_t* index_sparse, const int32_t* index_sparse,
@ -2526,26 +2526,26 @@ trexio_write_$group_sparse_dset$(trexio_t* const file,
trexio_exit_code rc; trexio_exit_code rc;
/* Read the max number of integrals stored in the file */ /* Read the max number of integrals stored in the file */
rc = trexio_read_$group_sparse_dset$_size(file, &size_max); rc = trexio_read_$group_dset$_size(file, &size_max);
if (rc != TREXIO_SUCCESS && rc != TREXIO_DSET_MISSING) return rc; if (rc != TREXIO_SUCCESS && rc != TREXIO_DSET_MISSING) return rc;
if (rc == TREXIO_DSET_MISSING) size_max = 0L; if (rc == TREXIO_DSET_MISSING) size_max = 0L;
switch (file->back_end) { switch (file->back_end) {
case TREXIO_TEXT: case TREXIO_TEXT:
return trexio_text_write_$group_sparse_dset$(file, offset_file, buffer_size, size_max, index_sparse, value_sparse); return trexio_text_write_$group_dset$(file, offset_file, buffer_size, size_max, index_sparse, value_sparse);
break; break;
case TREXIO_HDF5: case TREXIO_HDF5:
#ifdef HAVE_HDF5 #ifdef HAVE_HDF5
return trexio_hdf5_write_$group_sparse_dset$(file, offset_file, buffer_size, size_max, index_sparse, value_sparse); return trexio_hdf5_write_$group_dset$(file, offset_file, buffer_size, size_max, index_sparse, value_sparse);
break; break;
#else #else
return TREXIO_BACK_END_MISSING; return TREXIO_BACK_END_MISSING;
#endif #endif
/* /*
case TREXIO_JSON: case TREXIO_JSON:
return trexio_json_write_$group_sparse_dset$(...); return trexio_json_write_$group_dset$(...);
break; break;
*/ */
default: default:
@ -2557,7 +2557,7 @@ trexio_write_$group_sparse_dset$(trexio_t* const file,
#+begin_src c :tangle has_dset_sparse_front.c #+begin_src c :tangle has_dset_sparse_front.c
trexio_exit_code trexio_exit_code
trexio_has_$group_sparse_dset$ (trexio_t* const file) trexio_has_$group_dset$ (trexio_t* const file)
{ {
if (file == NULL) return TREXIO_INVALID_ARG_1; if (file == NULL) return TREXIO_INVALID_ARG_1;
@ -2567,19 +2567,19 @@ trexio_has_$group_sparse_dset$ (trexio_t* const file)
switch (file->back_end) { switch (file->back_end) {
case TREXIO_TEXT: case TREXIO_TEXT:
return trexio_text_has_$group_sparse_dset$(file); return trexio_text_has_$group_dset$(file);
break; break;
case TREXIO_HDF5: case TREXIO_HDF5:
#ifdef HAVE_HDF5 #ifdef HAVE_HDF5
return trexio_hdf5_has_$group_sparse_dset$(file); return trexio_hdf5_has_$group_dset$(file);
break; break;
#else #else
return TREXIO_BACK_END_MISSING; return TREXIO_BACK_END_MISSING;
#endif #endif
/* /*
case TREXIO_JSON: case TREXIO_JSON:
return trexio_json_has_$group_sparse_dset$(file); return trexio_json_has_$group_dset$(file);
break; break;
,*/ ,*/
} }
@ -2595,7 +2595,7 @@ trexio_has_$group_sparse_dset$ (trexio_t* const file)
#+begin_src f90 :tangle write_dset_sparse_front_fortran.f90 #+begin_src f90 :tangle write_dset_sparse_front_fortran.f90
interface interface
integer function trexio_write_$group_sparse_dset$ (trex_file, & integer function trexio_write_$group_dset$ (trex_file, &
offset_file, buffer_size, & offset_file, buffer_size, &
index_sparse, value_sparse) bind(C) index_sparse, value_sparse) bind(C)
use, intrinsic :: iso_c_binding use, intrinsic :: iso_c_binding
@ -2604,13 +2604,13 @@ interface
integer(8), intent(in), value :: buffer_size integer(8), intent(in), value :: buffer_size
integer(4), intent(in) :: index_sparse(*) integer(4), intent(in) :: index_sparse(*)
double precision, intent(in) :: value_sparse(*) double precision, intent(in) :: value_sparse(*)
end function trexio_write_$group_sparse_dset$ end function trexio_write_$group_dset$
end interface end interface
#+end_src #+end_src
#+begin_src f90 :tangle read_dset_sparse_front_fortran.f90 #+begin_src f90 :tangle read_dset_sparse_front_fortran.f90
interface interface
integer function trexio_read_$group_sparse_dset$ (trex_file, & integer function trexio_read_$group_dset$ (trex_file, &
offset_file, buffer_size, & offset_file, buffer_size, &
index_sparse, value_sparse) bind(C) index_sparse, value_sparse) bind(C)
use, intrinsic :: iso_c_binding use, intrinsic :: iso_c_binding
@ -2619,27 +2619,27 @@ interface
integer(8), intent(in), value :: buffer_size integer(8), intent(in), value :: buffer_size
integer(4), intent(in) :: index_sparse(*) integer(4), intent(in) :: index_sparse(*)
double precision, intent(out) :: value_sparse(*) double precision, intent(out) :: value_sparse(*)
end function trexio_read_$group_sparse_dset$ end function trexio_read_$group_dset$
end interface end interface
#+end_src #+end_src
#+begin_src f90 :tangle read_dset_sparse_size_front_fortran.f90 #+begin_src f90 :tangle read_dset_sparse_size_front_fortran.f90
interface interface
integer function trexio_read_$group_sparse_dset$_size (trex_file, & integer function trexio_read_$group_dset$_size (trex_file, &
size_max) bind(C) size_max) bind(C)
use, intrinsic :: iso_c_binding use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file integer(8), intent(in), value :: trex_file
integer(8), intent(out) :: size_max integer(8), intent(out) :: size_max
end function trexio_read_$group_sparse_dset$_size end function trexio_read_$group_dset$_size
end interface end interface
#+end_src #+end_src
#+begin_src f90 :tangle has_dset_sparse_front_fortran.f90 #+begin_src f90 :tangle has_dset_sparse_front_fortran.f90
interface interface
integer function trexio_has_$group_sparse_dset$ (trex_file) bind(C) integer function trexio_has_$group_dset$ (trex_file) bind(C)
use, intrinsic :: iso_c_binding use, intrinsic :: iso_c_binding
integer(8), intent(in), value :: trex_file integer(8), intent(in), value :: trex_file
end function trexio_has_$group_sparse_dset$ end function trexio_has_$group_dset$
end interface end interface
#+end_src #+end_src

68
src/templates_text/templator_text.org
View File

@ -396,7 +396,7 @@ trexio_text_read_$group$ (trexio_text_t* const file)
return NULL; return NULL;
} }
rc = fscanf(f, "%$group_num_std_dtype_in$", &($group$->$group_num$)); rc = fscanf(f, "%$group_num_format_scanf$", &($group$->$group_num$));
assert(!(rc != 1)); assert(!(rc != 1));
if (rc != 1) { if (rc != 1) {
FREE(buffer); FREE(buffer);
@ -484,7 +484,7 @@ trexio_text_read_$group$ (trexio_text_t* const file)
} }
for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) { for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
rc = fscanf(f, "%$group_dset_std_dtype_in$", &($group$->$group_dset$[i])); rc = fscanf(f, "%$group_dset_format_scanf$", &($group$->$group_dset$[i]));
assert(!(rc != 1)); assert(!(rc != 1));
if (rc != 1) { if (rc != 1) {
FREE(buffer); FREE(buffer);
@ -598,7 +598,7 @@ trexio_text_flush_$group$ (trexio_text_t* const file)
// START REPEAT GROUP_NUM // START REPEAT GROUP_NUM
fprintf(f, "$group_num$_isSet %u \n", $group$->$group_num$_isSet); fprintf(f, "$group_num$_isSet %u \n", $group$->$group_num$_isSet);
if ($group$->$group_num$_isSet == true) fprintf(f, "$group_num$ %$group_num_std_dtype_out$ \n", $group$->$group_num$); if ($group$->$group_num$_isSet == true) fprintf(f, "$group_num$ %$group_num_format_printf$ \n", $group$->$group_num$);
// END REPEAT GROUP_NUM // END REPEAT GROUP_NUM
// START REPEAT GROUP_ATTR_STR // START REPEAT GROUP_ATTR_STR
@ -612,7 +612,7 @@ trexio_text_flush_$group$ (trexio_text_t* const file)
fprintf(f, "$group_dset$\n"); fprintf(f, "$group_dset$\n");
for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) { for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
fprintf(f, "%$group_dset_std_dtype_out$\n", $group$->$group_dset$[i]); fprintf(f, "%$group_dset_format_printf$\n", $group$->$group_dset$[i]);
} }
// END REPEAT GROUP_DSET_ALL // END REPEAT GROUP_DSET_ALL
@ -1004,15 +1004,15 @@ trexio_text_has_$group_str$ (trexio_t* const file)
** Template for has/read/write the dataset of sparse data ** Template for has/read/write the dataset of sparse data
#+begin_src c :tangle hrw_dset_sparse_text.h :exports none #+begin_src c :tangle hrw_dset_sparse_text.h :exports none
trexio_exit_code trexio_text_has_$group_sparse_dset$(trexio_t* const file); trexio_exit_code trexio_text_has_$group_dset$(trexio_t* const file);
trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t size, const int64_t size_max, int32_t* const index_sparse, double* const value_sparse); trexio_exit_code trexio_text_read_$group_dset$(trexio_t* const file, const int64_t offset_file, const int64_t size, const int64_t size_max, int32_t* const index_sparse, double* const value_sparse);
trexio_exit_code trexio_text_read_$group_sparse_dset$_size(trexio_t* const file, int64_t* const size_max); trexio_exit_code trexio_text_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max);
trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t size, const int64_t size_max, const int32_t* index_sparse, const double* value_sparse); trexio_exit_code trexio_text_write_$group_dset$(trexio_t* const file, const int64_t offset_file, const int64_t size, const int64_t size_max, const int32_t* index_sparse, const double* value_sparse);
#+end_src #+end_src
#+begin_src c :tangle write_dset_sparse_text.c #+begin_src c :tangle write_dset_sparse_text.c
trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, trexio_exit_code trexio_text_write_$group_dset$(trexio_t* const file,
const int64_t offset_file, const int64_t offset_file,
const int64_t size, const int64_t size,
const int64_t size_max, const int64_t size_max,
@ -1022,16 +1022,16 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file,
if (file == NULL) return TREXIO_INVALID_ARG_1; if (file == NULL) return TREXIO_INVALID_ARG_1;
/* Build the name of the file with sparse data*/ /* Build the name of the file with sparse data*/
/* The $group_sparse_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed? */ /* The $group_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed? */
const char $group_sparse_dset$_file_name[256] = "/$group_sparse_dset$.txt"; const char $group_dset$_file_name[256] = "/$group_dset$.txt";
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */ /* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH]; char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */ /* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH); strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */ /* Append name of the file with sparse data */
strncat (file_full_path, $group_sparse_dset$_file_name, strncat (file_full_path, $group_dset$_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen($group_sparse_dset$_file_name)); TREXIO_MAX_FILENAME_LENGTH-strlen($group_dset$_file_name));
/* Open the file in "a" (append) mode to guarantee that no truncation happens upon consecutive writes */ /* Open the file in "a" (append) mode to guarantee that no truncation happens upon consecutive writes */
FILE* f = fopen(file_full_path, "a"); FILE* f = fopen(file_full_path, "a");
@ -1041,7 +1041,7 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file,
the line_length is 69 because 10 per index + 4 spaces + 24 for floating point value + 1 for the new line char. the line_length is 69 because 10 per index + 4 spaces + 24 for floating point value + 1 for the new line char.
CURRENTLY NO OFFSET IS USED WHEN WRITING ! CURRENTLY NO OFFSET IS USED WHEN WRITING !
*/ */
const int64_t line_length = $group_sparse_dset_line_length$L; const int64_t line_length = $group_dset_sparse_line_length$L;
/* Get the starting position of the IO stream to be written in the .size file. /* Get the starting position of the IO stream to be written in the .size file.
This is error-prone due to the fact that for large files (>2 GB) in 32-bit systems ftell will fail. This is error-prone due to the fact that for large files (>2 GB) in 32-bit systems ftell will fail.
@ -1057,8 +1057,8 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file,
int rc; int rc;
for (uint64_t i=0L; i<size; ++i) { for (uint64_t i=0L; i<size; ++i) {
rc = fprintf(f, "$group_sparse_dset_format_printf$\n", rc = fprintf(f, "$group_dset_format_printf$\n",
$group_sparse_dset_indices_printf$, $group_dset_sparse_indices_printf$,
value_sparse[i]); value_sparse[i]);
if(rc <= 0) { if(rc <= 0) {
@ -1099,7 +1099,7 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file,
#+begin_src c :tangle read_dset_sparse_text.c #+begin_src c :tangle read_dset_sparse_text.c
trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, trexio_exit_code trexio_text_read_$group_dset$(trexio_t* const file,
const int64_t offset_file, const int64_t offset_file,
const int64_t size, const int64_t size,
const int64_t size_max, const int64_t size_max,
@ -1109,17 +1109,17 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file,
if (file == NULL) return TREXIO_INVALID_ARG_1; if (file == NULL) return TREXIO_INVALID_ARG_1;
/* Build the name of the file with sparse data. /* Build the name of the file with sparse data.
The $group_sparse_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed? The $group_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed?
*/ */
const char $group_sparse_dset$_file_name[256] = "/$group_sparse_dset$.txt"; const char $group_dset$_file_name[256] = "/$group_dset$.txt";
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */ /* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH]; char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */ /* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH); strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */ /* Append name of the file with sparse data */
strncat (file_full_path, $group_sparse_dset$_file_name, strncat (file_full_path, $group_dset$_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen($group_sparse_dset$_file_name)); TREXIO_MAX_FILENAME_LENGTH-strlen($group_dset$_file_name));
/* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */ /* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */
FILE* f = fopen(file_full_path, "r"); FILE* f = fopen(file_full_path, "r");
@ -1128,7 +1128,7 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file,
/* Specify the line length in order to offset properly. For example, for 4-index quantities /* Specify the line length in order to offset properly. For example, for 4-index quantities
the line_length is 69 because 10 per index + 4 spaces + 24 for floating point value + 1 for the new line char the line_length is 69 because 10 per index + 4 spaces + 24 for floating point value + 1 for the new line char
*/ */
const uint64_t line_length = $group_sparse_dset_line_length$L; const uint64_t line_length = $group_dset_sparse_line_length$L;
fseek(f, (long) offset_file * line_length, SEEK_SET); fseek(f, (long) offset_file * line_length, SEEK_SET);
@ -1136,8 +1136,8 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file,
int rc; int rc;
for (uint64_t i=0L; i<size; ++i) { for (uint64_t i=0L; i<size; ++i) {
rc = fscanf(f, "$group_sparse_dset_format_scanf$", rc = fscanf(f, "$group_dset_format_scanf$",
$group_sparse_dset_indices_scanf$, $group_dset_sparse_indices_scanf$,
&value_sparse[i]); &value_sparse[i]);
// TODO: find a way to indicate the number of elements being read (useful?) // TODO: find a way to indicate the number of elements being read (useful?)
@ -1167,22 +1167,22 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file,
#+begin_src c :tangle read_dset_sparse_text.c #+begin_src c :tangle read_dset_sparse_text.c
trexio_exit_code trexio_text_read_$group_sparse_dset$_size(trexio_t* const file, int64_t* const size_max) trexio_exit_code trexio_text_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max)
{ {
if (file == NULL) return TREXIO_INVALID_ARG_1; if (file == NULL) return TREXIO_INVALID_ARG_1;
/* Build the name of the file with sparse data. /* Build the name of the file with sparse data.
The $group_sparse_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed? The $group_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed?
,*/ ,*/
const char $group_sparse_dset$_file_name[256] = "/$group_sparse_dset$.txt.size"; const char $group_dset$_file_name[256] = "/$group_dset$.txt.size";
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */ /* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH]; char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */ /* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH); strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */ /* Append name of the file with sparse data */
strncat (file_full_path, $group_sparse_dset$_file_name, strncat (file_full_path, $group_dset$_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen($group_sparse_dset$_file_name)); TREXIO_MAX_FILENAME_LENGTH-strlen($group_dset$_file_name));
/* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */ /* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */
FILE* f = fopen(file_full_path, "r"); FILE* f = fopen(file_full_path, "r");
@ -1217,22 +1217,22 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$_size(trexio_t* const file,
#+end_src #+end_src
#+begin_src c :tangle has_dset_sparse_text.c #+begin_src c :tangle has_dset_sparse_text.c
trexio_exit_code trexio_text_has_$group_sparse_dset$(trexio_t* const file) trexio_exit_code trexio_text_has_$group_dset$(trexio_t* const file)
{ {
if (file == NULL) return TREXIO_INVALID_ARG_1; if (file == NULL) return TREXIO_INVALID_ARG_1;
/* Build the name of the file with sparse data. /* Build the name of the file with sparse data.
The $group_sparse_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed? The $group_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed?
,*/ ,*/
const char $group_sparse_dset$_file_name[256] = "/$group_sparse_dset$.txt"; const char $group_dset$_file_name[256] = "/$group_dset$.txt";
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */ /* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH]; char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */ /* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH); strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */ /* Append name of the file with sparse data */
strncat (file_full_path, $group_sparse_dset$_file_name, strncat (file_full_path, $group_dset$_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen($group_sparse_dset$_file_name)); TREXIO_MAX_FILENAME_LENGTH-strlen($group_dset$_file_name));
/* Check the return code of access function to determine whether the file with sparse data exists or not */ /* Check the return code of access function to determine whether the file with sparse data exists or not */
if (access(file_full_path, F_OK) == 0){ if (access(file_full_path, F_OK) == 0){

72
tools/generator_tools.py
View File

@ -107,8 +107,8 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N
'group_num_f_dtype_default', 'group_num_f_dtype_double', 'group_num_f_dtype_single', 'group_num_f_dtype_default', 'group_num_f_dtype_double', 'group_num_f_dtype_single',
'group_num_dtype_default', 'group_num_dtype_double', 'group_num_dtype_single', 'group_num_dtype_default', 'group_num_dtype_double', 'group_num_dtype_single',
'group_num_h5_dtype', 'group_num_py_dtype', 'group_num_h5_dtype', 'group_num_py_dtype',
'group_sparse_dset', 'group_sparse_dset_format_scanf', 'group_sparse_dset_format_printf', 'group_dset_format_scanf', 'group_dset_format_printf',
'group_sparse_dset_line_length', 'group_sparse_dset_indices_printf', 'group_sparse_dset_indices_scanf', 'group_dset_sparse_line_length', 'group_dset_sparse_indices_printf', 'group_dset_sparse_indices_scanf',
'group_dset', 'group_num', 'group_str', 'group'] 'group_dset', 'group_num', 'group_str', 'group']
for item in detailed_source.keys(): for item in detailed_source.keys():
@ -294,8 +294,8 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai
fname_new = join('populated',f'pop_{fname}') fname_new = join('populated',f'pop_{fname}')
templ_path = get_template_path(fname, paths) templ_path = get_template_path(fname, paths)
triggers = ['group_dset_dtype', 'group_dset_std_dtype_out', 'group_dset_std_dtype_in', triggers = ['group_dset_dtype', 'group_dset_format_printf', 'group_dset_format_scanf',
'group_num_dtype_double', 'group_num_std_dtype_out', 'group_num_std_dtype_in', 'group_num_dtype_double', 'group_num_format_printf', 'group_num_format_scanf',
'group_dset', 'group_num', 'group_str', 'group'] 'group_dset', 'group_num', 'group_str', 'group']
for group in group_dict.keys(): for group in group_dict.keys():
@ -491,8 +491,8 @@ def get_detailed_num_dict (configuration: dict) -> dict:
tmp_dict['group_num_dtype_double'] = 'double' tmp_dict['group_num_dtype_double'] = 'double'
tmp_dict['group_num_dtype_single'] = 'float' tmp_dict['group_num_dtype_single'] = 'float'
tmp_dict['default_prec'] = '64' tmp_dict['default_prec'] = '64'
tmp_dict['group_num_std_dtype_out'] = '24.16e' tmp_dict['group_num_format_printf'] = '24.16e'
tmp_dict['group_num_std_dtype_in'] = 'lf' tmp_dict['group_num_format_scanf'] = 'lf'
tmp_dict['group_num_py_dtype'] = 'float' tmp_dict['group_num_py_dtype'] = 'float'
elif v2[0] in ['int', 'dim']: elif v2[0] in ['int', 'dim']:
tmp_dict['datatype'] = 'int64_t' tmp_dict['datatype'] = 'int64_t'
@ -504,8 +504,8 @@ def get_detailed_num_dict (configuration: dict) -> dict:
tmp_dict['group_num_dtype_double'] = 'int64_t' tmp_dict['group_num_dtype_double'] = 'int64_t'
tmp_dict['group_num_dtype_single'] = 'int32_t' tmp_dict['group_num_dtype_single'] = 'int32_t'
tmp_dict['default_prec'] = '32' tmp_dict['default_prec'] = '32'
tmp_dict['group_num_std_dtype_out'] = '" PRId64 "' tmp_dict['group_num_format_printf'] = '" PRId64 "'
tmp_dict['group_num_std_dtype_in'] = '" SCNd64 "' tmp_dict['group_num_format_scanf'] = '" SCNd64 "'
tmp_dict['group_num_py_dtype'] = 'int' tmp_dict['group_num_py_dtype'] = 'int'
tmp_dict['trex_json_int_type'] = v2[0] tmp_dict['trex_json_int_type'] = v2[0]
@ -589,8 +589,8 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
group_dset_dtype_double = 'double' group_dset_dtype_double = 'double'
group_dset_dtype_single = 'float' group_dset_dtype_single = 'float'
default_prec = '64' default_prec = '64'
group_dset_std_dtype_out = '24.16e' group_dset_format_printf = '24.16e'
group_dset_std_dtype_in = 'lf' group_dset_format_scanf = 'lf'
group_dset_py_dtype = 'float' group_dset_py_dtype = 'float'
elif v[0] in ['int', 'index']: elif v[0] in ['int', 'index']:
datatype = 'int64_t' datatype = 'int64_t'
@ -602,8 +602,8 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
group_dset_dtype_double = 'int64_t' group_dset_dtype_double = 'int64_t'
group_dset_dtype_single = 'int32_t' group_dset_dtype_single = 'int32_t'
default_prec = '32' default_prec = '32'
group_dset_std_dtype_out = '" PRId64 "' group_dset_format_printf = '" PRId64 "'
group_dset_std_dtype_in = '" SCNd64 "' group_dset_format_scanf = '" SCNd64 "'
group_dset_py_dtype = 'int' group_dset_py_dtype = 'int'
elif v[0] == 'str': elif v[0] == 'str':
datatype = 'char*' datatype = 'char*'
@ -615,29 +615,25 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
group_dset_dtype_double = '' group_dset_dtype_double = ''
group_dset_dtype_single = '' group_dset_dtype_single = ''
default_prec = '' default_prec = ''
group_dset_std_dtype_out = 's' group_dset_format_printf = 's'
group_dset_std_dtype_in = 's' group_dset_format_scanf = 's'
group_dset_py_dtype = 'str' group_dset_py_dtype = 'str'
elif 'sparse' in v[0]: elif 'sparse' in v[0]:
is_sparse = True is_sparse = True
datatype = 'double' datatype = 'double'
group_dset_h5_dtype = 'native_double' group_dset_h5_dtype = ''
group_dset_f_dtype_default= 'real(8)' group_dset_f_dtype_default= ''
group_dset_f_dtype_double = 'real(8)' group_dset_f_dtype_double = ''
group_dset_f_dtype_single = 'real(4)' group_dset_f_dtype_single = ''
group_dset_dtype_default= 'double' group_dset_dtype_default= ''
group_dset_dtype_double = 'double' group_dset_dtype_double = ''
group_dset_dtype_single = 'float' group_dset_dtype_single = ''
default_prec = '64' default_prec = ''
group_dset_std_dtype_out = '24.16e' group_dset_format_printf = '%10" PRId32 " %10" PRId32 " %10" PRId32 " %10" PRId32 " %24.16e'
group_dset_std_dtype_in = 'lf' group_dset_format_scanf = '%" SCNd32 " %" SCNd32 " %" SCNd32 " %" SCNd32 " %lf'
group_dset_py_dtype = 'float' group_dset_py_dtype = ''
# add the dset name for templates tmp_dict['group_dset'] = k
if is_sparse:
tmp_dict['group_sparse_dset'] = k
else:
tmp_dict['group_dset'] = k
# add flag to detect index types # add flag to detect index types
if 'index' == v[0]: if 'index' == v[0]:
tmp_dict['is_index'] = 'file->one_based' tmp_dict['is_index'] = 'file->one_based'
@ -654,8 +650,8 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
tmp_dict['group_dset_dtype_double'] = group_dset_dtype_double tmp_dict['group_dset_dtype_double'] = group_dset_dtype_double
tmp_dict['group_dset_dtype_single'] = group_dset_dtype_single tmp_dict['group_dset_dtype_single'] = group_dset_dtype_single
tmp_dict['default_prec'] = default_prec tmp_dict['default_prec'] = default_prec
tmp_dict['group_dset_std_dtype_in'] = group_dset_std_dtype_in tmp_dict['group_dset_format_printf'] = group_dset_format_printf
tmp_dict['group_dset_std_dtype_out'] = group_dset_std_dtype_out tmp_dict['group_dset_format_scanf'] = group_dset_format_scanf
tmp_dict['group_dset_py_dtype'] = group_dset_py_dtype tmp_dict['group_dset_py_dtype'] = group_dset_py_dtype
# add the rank # add the rank
tmp_dict['rank'] = len(v[1]) tmp_dict['rank'] = len(v[1])
@ -678,19 +674,17 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
tmp_dict['group_dset_f_dims'] = dim_f_list tmp_dict['group_dset_f_dims'] = dim_f_list
if is_sparse: if is_sparse:
tmp_dict['group_sparse_dset_format_scanf'] = "%d %d %d %d %lf" tmp_dict['group_dset_sparse_line_length'] = "69"
tmp_dict['group_sparse_dset_format_printf'] = "%10d %10d %10d %10d %24.16e" tmp_dict['group_dset_sparse_indices_printf'] = "*(index_sparse + 4*i), *(index_sparse + 4*i+1), *(index_sparse + 4*i+2), *(index_sparse + 4*i+3)"
tmp_dict['group_sparse_dset_line_length'] = "69" tmp_dict['group_dset_sparse_indices_scanf'] = "index_sparse + 4*i, index_sparse + 4*i+1, index_sparse + 4*i+2, index_sparse + 4*i+3"
tmp_dict['group_sparse_dset_indices_printf'] = "*(index_sparse + 4*i), *(index_sparse + 4*i+1), *(index_sparse + 4*i+2), *(index_sparse + 4*i+3)"
tmp_dict['group_sparse_dset_indices_scanf'] = "index_sparse + 4*i, index_sparse + 4*i+1, index_sparse + 4*i+2, index_sparse + 4*i+3"
# add group name as a key-value pair to the dset dict # add group name as a key-value pair to the dset dict
tmp_dict['group'] = v[2] tmp_dict['group'] = v[2]
# split datasets in numeric- and string- based # split datasets in numeric- and string- based
if (datatype == 'char*'): if datatype == 'char*':
dset_string_dict[k] = tmp_dict dset_string_dict[k] = tmp_dict
elif (is_sparse): elif is_sparse:
dset_sparse_dict[k] = tmp_dict dset_sparse_dict[k] = tmp_dict
else: else:
dset_numeric_dict[k] = tmp_dict dset_numeric_dict[k] = tmp_dict