[WIP] working generator for sparse functions

+ remove trailing whitespaces

src/templates_front/templator_front.org

@@ -2394,21 +2394,21 @@ def has_$group_dset$(trexio_file) -> bool:
 *** C templates for front end
 **** Function declarations
 
-     #+begin_src c :tangle hrw_sparse_front.h :exports none
+     #+begin_src c :tangle hrw_dset_sparse_front.h :exports none
 trexio_exit_code trexio_has_$group_sparse_dset$(trexio_t* const file);
-trexio_exit_code trexio_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int64_t size, int32_t* const index_sparse, double* const value_sparse);
+trexio_exit_code trexio_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_data, const int64_t size, int32_t* const index_sparse, double* const value_sparse);
-trexio_exit_code trexio_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int64_t size, const int32_t* index_sparse, const double* value_sparse);
+trexio_exit_code trexio_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_data, const int64_t size, const int32_t* index_sparse, const double* value_sparse);
 //trexio_exit_code trexio_read_$group_sparse_dset$_value(trexio_t* const file, const uint64_t offset, const uint_64_t size, int32_t* const value_sparse);
 //trexio_exit_code trexio_write_$group_sparse_dset$_value(trexio_t* const file, const uint64_t offset, const uint_64_t size, double* const value_sparse);
      #+end_src
 
 **** Source code for default functions
 
-  #+begin_src c :tangle read_sparse_front.c
+  #+begin_src c :tangle read_dset_sparse_front.c
 trexio_exit_code
 trexio_read_$group_sparse_dset$(trexio_t* const file,
                            const int64_t offset_file,
-                           const int64_t offset_dset,
+                           const int64_t offset_data,
                            const int64_t size,
                            int32_t* const index_sparse,
                            double* const value_sparse
@@ -2416,28 +2416,35 @@ trexio_read_$group_sparse_dset$(trexio_t* const file,
 {
   if (file  == NULL) return TREXIO_INVALID_ARG_1;
   if (offset_file < 0L) return TREXIO_INVALID_ARG_2;
-  if (offset_dset < 0L) return TREXIO_INVALID_ARG_3;
+  if (size        <= 0L) return TREXIO_INVALID_ARG_3;
-  if (size        <= 0L) return TREXIO_INVALID_ARG_4;
+  if (index_sparse == NULL) return TREXIO_INVALID_ARG_4;
-  if (index_sparse == NULL) return TREXIO_INVALID_ARG_5;
+  if (value_sparse == NULL) return TREXIO_INVALID_ARG_5;
-  if (value_sparse == NULL) return TREXIO_INVALID_ARG_6;
 
-  const uint32_t rank = $group_sparse_dset_rank$;  // To be set by generator : number of indices
+  const uint32_t rank = $group_dset_rank$;  // To be set by generator : number of indices
 
   int64_t size_max;         // Max number of integrals (already in the file)
   trexio_exit_code rc;
 
+  // temporary
+  size_max = size;
+  /* TODO
   rc = trexio_read_$group_sparse_dset$_num(file, &size_max);
   if (rc != TREXIO_SUCCESS) return rc;
+  */
 
   switch (file->back_end) {
 
   case TREXIO_TEXT:
-    return trexio_text_read_$group_sparse_dset$(file, offset_file, offset_dset, size, size_max, rank, index_sparse, value_sparse);
+    return trexio_text_read_$group_sparse_dset$(file, offset_file, offset_data, size, size_max, index_sparse, value_sparse);
     break;
 
   case TREXIO_HDF5:
-    return trexio_hdf5_read_$group_sparse_dset$(file, offset_file, offset_dset, size, size_max, rank, index_sparse, value_sparse);
+#ifdef HAVE_HDF5
+    return trexio_hdf5_read_$group_sparse_dset$(file, offset_file, offset_data, size, size_max, index_sparse, value_sparse);
     break;
+#else
+    return TREXIO_BACK_END_MISSING;
+#endif
 /*
   case TREXIO_JSON:
     return trexio_json_read_$group_sparse_dset$(...);
@@ -2449,11 +2456,11 @@ trexio_read_$group_sparse_dset$(trexio_t* const file,
 }
   #+end_src
 
-  #+begin_src c :tangle write_sparse_front.c
+  #+begin_src c :tangle write_dset_sparse_front.c
 trexio_exit_code
 trexio_write_$group_sparse_dset$(trexio_t* const file,
                            const int64_t offset_file,
-                           const int64_t offset_dset,
+                           const int64_t offset_data,
                            const int64_t size,
                            const int32_t* index_sparse,
                            const double* value_sparse
@@ -2461,28 +2468,35 @@ trexio_write_$group_sparse_dset$(trexio_t* const file,
 {
   if (file  == NULL) return TREXIO_INVALID_ARG_1;
   if (offset_file < 0L) return TREXIO_INVALID_ARG_2;
-  if (offset_dset < 0L) return TREXIO_INVALID_ARG_3;
+  if (size        <= 0L) return TREXIO_INVALID_ARG_3;
-  if (size        <= 0L) return TREXIO_INVALID_ARG_4;
+  if (index_sparse == NULL) return TREXIO_INVALID_ARG_4;
-  if (index_sparse == NULL) return TREXIO_INVALID_ARG_5;
+  if (value_sparse == NULL) return TREXIO_INVALID_ARG_5;
-  if (value_sparse == NULL) return TREXIO_INVALID_ARG_6;
 
-  const uint32_t rank = $group_sparse_dset_rank$;  // To be set by generator : number of indices
+  const uint32_t rank = $group_dset_rank$;  // To be set by generator : number of indices
 
   int64_t size_max;         // Max number of integrals (already in the file)
   trexio_exit_code rc;
 
+  // temporary
+  size_max = size;
+  /* TODO
   rc = trexio_read_$group_sparse_dset$_num(file, &size_max);
   if (rc != TREXIO_SUCCESS) return rc;
+  */
 
   switch (file->back_end) {
 
   case TREXIO_TEXT:
-    return trexio_text_write_$group_sparse_dset$(file, offset_file, offset_dset, size, size_max, rank, index_sparse, value_sparse);
+    return trexio_text_write_$group_sparse_dset$(file, offset_file, offset_data, size, size_max, index_sparse, value_sparse);
     break;
 
   case TREXIO_HDF5:
-    return trexio_hdf5_write_$group_sparse_dset$(file, offset_file, offset_dset, size, size_max, rank, index_sparse, value_sparse);
+#ifdef HAVE_HDF5
+    return trexio_hdf5_write_$group_sparse_dset$(file, offset_file, offset_data, size, size_max, index_sparse, value_sparse);
     break;
+#else
+    return TREXIO_BACK_END_MISSING;
+#endif
 /*
   case TREXIO_JSON:
     return trexio_json_write_$group_sparse_dset$(...);
@@ -2495,6 +2509,39 @@ trexio_write_$group_sparse_dset$(trexio_t* const file,
   #+end_src
 
 
+  #+begin_src c :tangle has_dset_sparse_front.c
+trexio_exit_code
+trexio_has_$group_sparse_dset$ (trexio_t* const file)
+{
+
+  if (file  == NULL) return TREXIO_INVALID_ARG_1;
+
+  assert(file->back_end < TREXIO_INVALID_BACK_END);
+
+  switch (file->back_end) {
+
+  case TREXIO_TEXT:
+    return trexio_text_has_$group_sparse_dset$(file);
+    break;
+
+  case TREXIO_HDF5:
+#ifdef HAVE_HDF5
+    return trexio_hdf5_has_$group_sparse_dset$(file);
+    break;
+#else
+    return TREXIO_BACK_END_MISSING;
+#endif
+/*
+  case TREXIO_JSON:
+    return trexio_json_has_$group_sparse_dset$(file);
+    break;
+,*/
+  }
+
+  return TREXIO_FAILURE;
+}
+     #+end_src
+
 ** Templates for front end has/read/write a dataset of strings
 *** Introduction
    This section concerns API calls related to datasets of strings.

src/templates_text/build.sh

@@ -19,23 +19,26 @@ cat populated/pop_flush_group_text.h >> trexio_text.h
 
 cat populated/pop_has_dset_data_text.c >> trexio_text.c
 cat populated/pop_has_dset_str_text.c >> trexio_text.c
+cat populated/pop_has_dset_sparse_text.c >> trexio_text.c
 cat populated/pop_has_attr_num_text.c >> trexio_text.c
 cat populated/pop_has_attr_str_text.c >> trexio_text.c
+
 cat populated/pop_read_dset_data_text.c >> trexio_text.c
 cat populated/pop_read_dset_str_text.c >> trexio_text.c
+cat populated/pop_read_dset_sparse_text.c >> trexio_text.c
 cat populated/pop_read_attr_str_text.c >> trexio_text.c
 cat populated/pop_read_attr_num_text.c >> trexio_text.c
+
 cat populated/pop_write_dset_data_text.c >> trexio_text.c
 cat populated/pop_write_dset_str_text.c >> trexio_text.c
+cat populated/pop_write_dset_sparse_text.c >> trexio_text.c
 cat populated/pop_write_attr_str_text.c >> trexio_text.c
 cat populated/pop_write_attr_num_text.c >> trexio_text.c
+
 cat populated/pop_hrw_dset_data_text.h >> trexio_text.h
 cat populated/pop_hrw_dset_str_text.h >> trexio_text.h
+cat populated/pop_hrw_dset_sparse_text.h >> trexio_text.h
 cat populated/pop_hrw_attr_num_text.h >> trexio_text.h
 cat populated/pop_hrw_attr_str_text.h >> trexio_text.h
 
-cat rdm_text.c >> trexio_text.c
-cat rdm_text.h >> trexio_text.h
-
 cat suffix_text.h   >> trexio_text.h
-

src/templates_text/templator_text.org

@@ -1003,13 +1003,13 @@ trexio_text_has_$group_str$ (trexio_t* const file)
    #+end_src
 ** Template for has/read/write the dataset of sparse data
 
-   #+begin_src c :tangle hrw_sparse_text.h :exports none
+   #+begin_src c :tangle hrw_dset_sparse_text.h :exports none
 trexio_exit_code trexio_text_has_$group_sparse_dset$(trexio_t* const file);
-trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int_64_t size, const int64_t size_max, int32_t* const index_sparse, double* const value_sparse);
+trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int64_t size, const int64_t size_max, int32_t* const index_sparse, double* const value_sparse);
-trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int_64_t size, const int64_t size_max, const int32_t* index_sparse, const double* value_sparse);
+trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file, const int64_t offset_file, const int64_t offset_dset, const int64_t size, const int64_t size_max, const int32_t* index_sparse, const double* value_sparse);
    #+end_src
 
-   #+begin_src c :tangle write_sparse_text.c
+   #+begin_src c :tangle write_dset_sparse_text.c
 trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file,
                                                   const int64_t offset_file,
                                                   const int64_t offset_dset,
@@ -1022,7 +1022,7 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file,
 
  /* Build the name of the file with sparse data*/
   const char* $group_sparse_dset$_file_name = "/$group_sparse_dset$.txt";
-  const char  file_abs_path[TREXIO_MAX_FILENAME_LENGTH];
+  char  file_abs_path[TREXIO_MAX_FILENAME_LENGTH];
 
   strncpy (file_abs_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
   strncat (file_abs_path, $group_sparse_dset$_file_name,
@@ -1039,10 +1039,11 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file,
   // in general: 10*n_indices + n_indices + 24 + 1
   const uint64_t line_length = $group_sparse_dset_line_length$L;
 
-  fseek(f, (long) offset_file * line_length, SEEK_SET);
+  //fseek(f, (long) offset_file * line_length, SEEK_SET);
 
-  for (uint64_t i=0L+offset_data ; i<size+offset_data ; ++i) {
+  int rc;
-      int rc = fprintf(f, "$group_sparse_dset_format_printf$\n",
+  for (uint64_t i=0L+offset_dset ; i<size+offset_dset ; ++i) {
+      rc = fprintf(f, "$group_sparse_dset_format_printf$\n",
          $group_sparse_dset_indices_printf$,
          value_sparse[i]);
       assert(rc > 0);
@@ -1056,13 +1057,15 @@ trexio_exit_code trexio_text_write_$group_sparse_dset$(trexio_t* const file,
            value[i]);
 */
 
-  fclose(f);
+  rc = fclose(f);
+  assert(rc == 0);
+
   return TREXIO_SUCCESS;
 }
    #+end_src
 
 
-   #+begin_src c :tangle read_sparse_text.c
+   #+begin_src c :tangle read_dset_sparse_text.c
 trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file,
                                                  const int64_t offset_file,
                                                  const int64_t offset_dset,
@@ -1075,14 +1078,14 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file,
 
  /* Build the name of the file with sparse data*/
   const char* $group_sparse_dset$_file_name = "/$group_sparse_dset$.txt";
-  const char  file_abs_path[TREXIO_MAX_FILENAME_LENGTH];
+  char  file_abs_path[TREXIO_MAX_FILENAME_LENGTH];
 
   strncpy (file_abs_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
   strncat (file_abs_path, $group_sparse_dset$_file_name,
 	   TREXIO_MAX_FILENAME_LENGTH-strlen($group_sparse_dset$_file_name));
 
 
-  FILE* f = fopen(file_abs_path, "a");
+  FILE* f = fopen(file_abs_path, "r");
   //TODO ERROR HANDLING
   assert(f != NULL);
 
@@ -1094,10 +1097,11 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file,
 
   fseek(f, (long) offset_file * line_length, SEEK_SET);
 
-  for (uint64_t i=0L+offset_data ; i<size+offset_data ; ++i) {
+  int rc;
-      int rc = fprintf(f, "$group_sparse_dset_format_scanf$\n",
+  for (uint64_t i=0L+offset_dset ; i<size+offset_dset ; ++i) {
+      rc = fscanf(f, "$group_sparse_dset_format_scanf$",
          $group_sparse_dset_indices_scanf$,
-         value_sparse[i]);
+         &value_sparse[i]);
 
 // TODO: find a way to indicate the number of elements being read (useful?)
       if (rc == EOF){
@@ -1106,47 +1110,32 @@ trexio_exit_code trexio_text_read_$group_sparse_dset$(trexio_t* const file,
         assert(rc > 0);
       }
   }
-/*
-        int rc = fscanf(f, "%d %d %d %d %lf",
-           &index[4*i],
-           &index[4*i+1],
-           &index[4*i+2],N OPEN
-           &index[4*i+3],
-           &value[i]);
-,*/
 
-  fclose(f);
+  rc = fclose(f);
+  assert(rc==0);
+
   return TREXIO_SUCCESS;
 
 }
    #+end_src
 
-   #+begin_src c :tangle has_sparse_text.c
+   #+begin_src c :tangle has_dset_sparse_text.c
 trexio_exit_code trexio_text_has_$group_sparse_dset$(trexio_t* const file)
 {
   if (file == NULL) return TREXIO_FILE_ERROR;
 
  /* Build the name of the file with sparse data*/
   const char* $group_sparse_dset$_file_name = "/$group_sparse_dset$.txt";
-  const char  file_abs_path[TREXIO_MAX_FILENAME_LENGTH];
+  char  file_abs_path[TREXIO_MAX_FILENAME_LENGTH];
 
   strncpy (file_abs_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
   strncat (file_abs_path, $group_sparse_dset$_file_name,
 	   TREXIO_MAX_FILENAME_LENGTH-strlen($group_sparse_dset$_file_name));
 
-  /*struct stat buffer;
+  if (access(file_abs_path, F_OK) == 0){
-  int rc = stat(file_abs_path, &buffer)
-  if (rc == 0) {
     return TREXIO_SUCCESS;
   } else {
     return TREXIO_HAS_NOT;
-  }*/
-
-  int fd = open(file_abs_path, O_CREAT | O_EXCL | O_RDONLY);
-  if (fd < 0) {
-    if(errno == EEXIST) return TREXIO_SUCCESS;
-  } else {
-    return TREXIO_HAS_NOT;
   }
 }
    #+end_src
@@ -1156,4 +1145,3 @@ trexio_exit_code trexio_text_has_$group_sparse_dset$(trexio_t* const file)
   #+begin_src c :tangle suffix_text.h
 #endif
   #+end_src
-

tools/generator.py

@@ -6,18 +6,13 @@ config_file = 'trex.json'
 trex_config = read_json(config_file)
 # --------------------------------------------------------------------------- #
 
-# -------------------------------- [WIP] ------------------------------------ #
-# for now remove rdm from config because it functions are hardcoded
-del trex_config['rdm']
-# --------------------------------------------------------------------------- #
-
 # -------------------- GET ATTRIBUTES FROM THE CONFIGURATION ---------------- #
 group_dict = get_group_dict(trex_config)
 detailed_nums = get_detailed_num_dict(trex_config)
 detailed_strs = get_detailed_str_dict(trex_config)
 # helper dictionaries that contain names of groups, nums or dsets as keys
 dsets = get_dset_dict(trex_config)
-detailed_dsets_nostr, detailed_dsets_str = split_dset_dict_detailed(dsets)
+detailed_dsets_nostr, detailed_dsets_str, detailed_dsets_sparse = split_dset_dict_detailed(dsets)
 detailed_dsets = detailed_dsets_nostr.copy()
 detailed_dsets.update(detailed_dsets_str)
 # consistency check for dimensioning variables
@@ -56,6 +51,10 @@ for fname in files_todo['dset_data']:
 for fname in files_todo['dset_str']:
     recursive_populate_file(fname, template_paths, detailed_dsets_str)
 
+# populate has/read/write_dset (sparse) functions with recursive scheme
+for fname in files_todo['dset_sparse']:
+    recursive_populate_file(fname, template_paths, detailed_dsets_sparse)
+
 # populate group-related functions with mixed (iterative+recursive) scheme [text backend]
 for fname in files_todo['group']:
     special_populate_text_group(fname, template_paths, group_dict, detailed_dsets, detailed_nums, detailed_strs)

tools/generator_tools.py

@@ -39,7 +39,7 @@ def get_files_todo(source_files: dict) -> dict:
     files_todo = {}
     #files_todo['all'] = list(filter(lambda x: 'read' in x or 'write' in x or 'has' in x or 'hrw' in x or 'flush' in x or 'free' in x, all_files))
     files_todo['all'] = [f for f in all_files if 'read' in f or 'write' in f or 'has' in f or 'flush' in f or 'free' in f or 'hrw' in f]
-    for key in ['dset_data', 'dset_str', 'attr_num', 'attr_str', 'group']:
+    for key in ['dset_data', 'dset_str', 'dset_sparse', 'attr_num', 'attr_str', 'group']:
         files_todo[key] = list(filter(lambda x: key in x, files_todo['all']))
 
     files_todo['group'].append('struct_text_group_dset.h')
@@ -107,6 +107,8 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N
                 'group_num_f_dtype_default', 'group_num_f_dtype_double', 'group_num_f_dtype_single',
                 'group_num_dtype_default', 'group_num_dtype_double', 'group_num_dtype_single',
                 'group_num_h5_dtype', 'group_num_py_dtype',
+                'group_sparse_dset', 'group_sparse_dset_format_scanf', 'group_sparse_dset_format_printf',
+                'group_sparse_dset_line_length', 'group_sparse_dset_indices_printf', 'group_sparse_dset_indices_scanf',
                 'group_dset', 'group_num', 'group_str', 'group']
 
     for item in detailed_source.keys():
@@ -571,9 +573,11 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
     """
     dset_numeric_dict = {}
     dset_string_dict = {}
+    dset_sparse_dict = {}
     for k,v in datasets.items():
         # create a temp dictionary
         tmp_dict = {}
+        is_sparse = False
         # specify details required to replace templated variables later
         if v[0] == 'float':
             datatype = 'double'
@@ -614,8 +618,25 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
             group_dset_std_dtype_out = 's'
             group_dset_std_dtype_in  = 's'
             group_dset_py_dtype = 'str'
+        elif 'sparse' in v[0]:
+            is_sparse = True
+            datatype = 'double'
+            group_dset_h5_dtype       = 'native_double'
+            group_dset_f_dtype_default= 'real(8)'
+            group_dset_f_dtype_double = 'real(8)'
+            group_dset_f_dtype_single = 'real(4)'
+            group_dset_dtype_default= 'double'
+            group_dset_dtype_double = 'double'
+            group_dset_dtype_single = 'float'
+            default_prec   = '64'
+            group_dset_std_dtype_out = '24.16e'
+            group_dset_std_dtype_in = 'lf'
+            group_dset_py_dtype = 'float'
 
         # add the dset name for templates
+        if is_sparse:
+            tmp_dict['group_sparse_dset'] = k
+        else:
             tmp_dict['group_dset'] = k
         # add flag to detect index types
         if 'index' == v[0]:
@@ -656,16 +677,25 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
             dim_f_list = "(*)"
         tmp_dict['group_dset_f_dims'] = dim_f_list
 
+        if is_sparse:
+            tmp_dict['group_sparse_dset_format_scanf'] = "%d %d %d %d %lf"
+            tmp_dict['group_sparse_dset_format_printf'] = "%10d %10d %10d %10d %24.16e"
+            tmp_dict['group_sparse_dset_line_length'] = "69"
+            tmp_dict['group_sparse_dset_indices_printf'] = "index_sparse[4*i], index_sparse[4*i+1], index_sparse[4*i+2], index_sparse[4*i+3]"
+            tmp_dict['group_sparse_dset_indices_scanf']  = "&index_sparse[4*i], &index_sparse[4*i+1], &index_sparse[4*i+2], &index_sparse[4*i+3]"
+
         # add group name as a key-value pair to the dset dict
         tmp_dict['group'] = v[2]
 
         # split datasets in numeric- and string- based
         if (datatype == 'char*'):
             dset_string_dict[k] = tmp_dict
+        elif (is_sparse):
+            dset_sparse_dict[k] = tmp_dict
         else:
             dset_numeric_dict[k] = tmp_dict
 
-    return (dset_numeric_dict, dset_string_dict)
+    return (dset_numeric_dict, dset_string_dict, dset_sparse_dict)
 
 
 def check_dim_consistency(num: dict, dset: dict) -> None: