diff --git a/src/templates_hdf5/templator_hdf5.org b/src/templates_hdf5/templator_hdf5.org index 8ff47d1..3cfc914 100644 --- a/src/templates_hdf5/templator_hdf5.org +++ b/src/templates_hdf5/templator_hdf5.org @@ -201,15 +201,15 @@ trexio_hdf5_write_$group_num$ (trexio_t* const file, const $group_num_dtype_doub /* Write the dimensioning variables */ const hid_t dtype = H5Tcopy(H5T_$GROUP_NUM_H5_DTYPE$); const hid_t dspace = H5Screate(H5S_SCALAR); - - const hid_t num_id = H5Acreate(f->$group$_group, $GROUP_NUM$_NAME, + + const hid_t num_id = H5Acreate(f->$group$_group, $GROUP_NUM$_NAME, dtype, dspace, H5P_DEFAULT, H5P_DEFAULT); if (num_id <= 0) { H5Sclose(dspace); H5Tclose(dtype); return TREXIO_INVALID_ID; } - + const herr_t status = H5Awrite(num_id, dtype, &(num)); if (status < 0) { H5Aclose(num_id); @@ -217,7 +217,7 @@ trexio_hdf5_write_$group_num$ (trexio_t* const file, const $group_num_dtype_doub H5Tclose(dtype); return TREXIO_FAILURE; } - + H5Sclose(dspace); H5Aclose(num_id); H5Tclose(dtype); @@ -372,6 +372,202 @@ trexio_hdf5_has_$group_dset$ (trexio_t* const file) } #+end_src +** Template for HDF5 has/read/write the dataset of sparse data + + Sparse data is stored using extensible datasets of HDF5. Extensibility is required + due to the fact that the sparse data will be written in chunks of user-defined size. + + #+begin_src c :tangle hrw_dset_sparse_hdf5.h :exports none +trexio_exit_code trexio_hdf5_has_$group_dset$(trexio_t* const file); +trexio_exit_code trexio_hdf5_read_$group_dset$(trexio_t* const file, const int64_t offset_file, const int64_t size, const int64_t size_max, int32_t* const index_sparse, double* const value_sparse); +trexio_exit_code trexio_hdf5_write_$group_dset$(trexio_t* const file, const int64_t offset_file, const int64_t size, const int64_t size_max, const int32_t* index_sparse, const double* value_sparse); +trexio_exit_code trexio_hdf5_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max); + #+end_src + + + #+begin_src c :tangle write_dset_sparse_hdf5.c +trexio_exit_code +trexio_hdf5_write_$group_dset$ (trexio_t* const file, + const int64_t offset_file, + const int64_t size, + const int64_t size_max, + const int32_t* index_sparse, + const double* value_sparse) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + + trexio_hdf5_t* f = (trexio_hdf5_t*) file; + + const uint32_t rank = 1; // 4; + const hsize_t chunk_dims[1] = {size*4}; //[4] = {size, size, size, size}; + // TODO: generator + hsize_t maxdims[1] = {H5S_UNLIMITED}; // [4] = {H5S_UNLIMITED, H5S_UNLIMITED, H5S_UNLIMITED, H5S_UNLIMITED}; + + if ( H5LTfind_dataset(f->$group$_group, $GROUP_DSET$_NAME) != 1 ) { + + hid_t dspace = H5Screate_simple(rank, chunk_dims, maxdims); + hid_t prop = H5Pcreate(H5P_DATASET_CREATE); + herr_t status = H5Pset_chunk(prop, rank, chunk_dims); + + hid_t dset_id = H5Dcreate(f->$group$_group, + $GROUP_DSET$_NAME, + H5T_NATIVE_INT32, + dspace, + H5P_DEFAULT, + prop, + H5P_DEFAULT); + + assert(dset_id >= 0); + printf(" HERE HERE HERE !\n"); + status = H5Dwrite(dset_id, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, index_sparse); + + /*const herr_t status = H5LTmake_dataset(f->$group$_group, + $GROUP_DSET$_NAME, + (int) rank, (const hsize_t*) dims, + H5T_$GROUP_DSET_H5_DTYPE$, + $group_dset$); + if (status < 0) return TREXIO_FAILURE;*/ + H5Pclose(prop); + H5Dclose(dset_id); + H5Sclose(dspace); + + } else { + + hid_t dset_id = H5Dopen(f->$group$_group, $GROUP_DSET$_NAME, H5P_DEFAULT); + if (dset_id <= 0) return TREXIO_INVALID_ID; + + + hid_t fspace = H5Dget_space(dset_id); + hsize_t offset[1] = {offset_file*4}; //[4] = {offset_file, offset_file, offset_file, offset_file}; + + // allocate space for the dimensions to be read + hsize_t ddims[1] = {0}; + + // get the rank and dimensions of the dataset + int rrank = H5Sget_simple_extent_dims(fspace, ddims, NULL); + ddims[0] += chunk_dims[0]; + + printf("SIZE = %ld\n", ddims[0]); + + // extend the dset size + herr_t status = H5Dset_extent(dset_id, ddims); + + H5Sclose(fspace); + + fspace = H5Dget_space(dset_id); + + status = H5Sselect_hyperslab(fspace, H5S_SELECT_SET, offset, NULL, chunk_dims, NULL); + hid_t dspace = H5Screate_simple(rank, chunk_dims, NULL); + + status = H5Dwrite(dset_id, + H5T_NATIVE_INT32, + dspace, fspace, H5P_DEFAULT, + index_sparse); + assert(status >= 0); + // TODO: CLOSE ALL OPENED + H5Dclose(dset_id); + H5Sclose(dspace); + H5Sclose(fspace); + //if (status < 0) return TREXIO_FAILURE; + + } + + return TREXIO_SUCCESS; + +} + #+end_src + + + #+begin_src c :tangle read_dset_sparse_hdf5.c +trexio_exit_code +trexio_hdf5_read_$group_dset$ (trexio_t* const file, + const int64_t offset_file, + const int64_t size, + const int64_t size_max, + int32_t* const index_sparse, + double* const value_sparse) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + + const trexio_hdf5_t* f = (const trexio_hdf5_t*) file; + + // open the dataset to get its dimensions + hid_t dset_id = H5Dopen(f->$group$_group, $GROUP_DSET$_NAME, H5P_DEFAULT); + if (dset_id <= 0) return TREXIO_INVALID_ID; + + const uint32_t rank = 4; + + // allocate space for the dimensions to be read + hsize_t* ddims = CALLOC( (int) rank, hsize_t); + if (ddims == NULL) return TREXIO_FAILURE; + + // get the dataspace of the dataset + hid_t dspace_id = H5Dget_space(dset_id); + // get the rank and dimensions of the dataset + int rrank = H5Sget_simple_extent_dims(dspace_id, ddims, NULL); + + // check that dimensions are consistent + if (rrank != (int) rank) { + FREE(ddims); + H5Sclose(dspace_id); + H5Dclose(dset_id); + return TREXIO_INVALID_ARG_3; + } + + free(ddims); + H5Sclose(dspace_id); + H5Dclose(dset_id); + + /* High-level H5LT API. No need to deal with dataspaces and datatypes */ + /*herr_t status = H5LTread_dataset(f->$group$_group, + $GROUP_DSET$_NAME, + H5T_$GROUP_DSET_H5_DTYPE$, + $group_dset$); + if (status < 0) return TREXIO_FAILURE;*/ + + return TREXIO_SUCCESS; +} + #+end_src + + + #+begin_src c :tangle read_dset_sparse_hdf5.c +trexio_exit_code +trexio_hdf5_read_$group_dset$_size (trexio_t* const file, int64_t* const size_max) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + + // TODO + + return TREXIO_SUCCESS; +} + #+end_src + + + #+begin_src c :tangle has_dset_sparse_hdf5.c +trexio_exit_code +trexio_hdf5_has_$group_dset$ (trexio_t* const file) +{ + + if (file == NULL) return TREXIO_INVALID_ARG_1; + + trexio_hdf5_t* f = (trexio_hdf5_t*) file; + + herr_t status = H5LTfind_dataset(f->$group$_group, $GROUP_DSET$_NAME); + /* H5LTfind_dataset returns 1 if dataset exists, 0 otherwise */ + if (status == 1){ + return TREXIO_SUCCESS; + } else if (status == 0) { + return TREXIO_HAS_NOT; + } else { + return TREXIO_FAILURE; + } + +} + #+end_src + ** Template for HDF5 has/read/write the dataset of strings #+begin_src c :tangle hrw_dset_str_hdf5.h :exports none @@ -403,10 +599,10 @@ trexio_hdf5_read_$group_dset$ (trexio_t* const file, char* const $group_dset$, c return TREXIO_ALLOCATION_FAILED; } - hid_t dspace = H5Dget_space(dset_id); + hid_t dspace = H5Dget_space(dset_id); if (dset_id <= 0) { FREE(ddims); - H5Dclose(dset_id); + H5Dclose(dset_id); return TREXIO_INVALID_ID; } @@ -442,7 +638,7 @@ trexio_hdf5_read_$group_dset$ (trexio_t* const file, char* const $group_dset$, c if (rdata == NULL) { H5Dclose(dset_id); H5Sclose(dspace); - H5Tclose(memtype); + H5Tclose(memtype); return TREXIO_ALLOCATION_FAILED; } @@ -451,7 +647,7 @@ trexio_hdf5_read_$group_dset$ (trexio_t* const file, char* const $group_dset$, c FREE(rdata); H5Dclose(dset_id); H5Sclose(dspace); - H5Tclose(memtype); + H5Tclose(memtype); return TREXIO_FAILURE; } @@ -474,11 +670,11 @@ trexio_hdf5_read_$group_dset$ (trexio_t* const file, char* const $group_dset$, c FREE(rdata); H5Dclose(dset_id); H5Sclose(dspace); - H5Tclose(memtype); + H5Tclose(memtype); return TREXIO_FAILURE; } - FREE(rdata); + FREE(rdata); H5Dclose(dset_id); H5Sclose(dspace); H5Tclose(memtype); @@ -509,7 +705,7 @@ trexio_hdf5_write_$group_dset$ (trexio_t* const file, const char** $group_dset$, if ( H5LTfind_dataset(f->$group$_group, $GROUP_DSET$_NAME) != 1 ) { - /* code to create dataset */ + /* code to create dataset */ hid_t filetype = H5Tcopy (H5T_FORTRAN_S1); if (filetype <= 0) return TREXIO_INVALID_ID; @@ -577,7 +773,7 @@ trexio_hdf5_has_$group_dset$ (trexio_t* const file) } #+end_src - + ** Template for HDF5 has/read/write the string attribute #+begin_src c :tangle hrw_attr_str_hdf5.h :exports none @@ -655,7 +851,7 @@ trexio_hdf5_write_$group_str$ (trexio_t* const file, const char* str) const hid_t dspace_id = H5Screate(H5S_SCALAR); if (dspace_id <= 0) return TREXIO_INVALID_ID; - + /* Create the $group_str$ attribute of $group$ group */ const hid_t str_id = H5Acreate(f->$group$_group, $GROUP_STR$_NAME, dtype_id, dspace_id, H5P_DEFAULT, H5P_DEFAULT); @@ -665,7 +861,7 @@ trexio_hdf5_write_$group_str$ (trexio_t* const file, const char* str) H5Tclose(dtype_id); return TREXIO_INVALID_ID; } - + status = H5Awrite(str_id, dtype_id, str); if (status < 0) { H5Aclose(str_id); @@ -673,7 +869,7 @@ trexio_hdf5_write_$group_str$ (trexio_t* const file, const char* str) H5Tclose(dtype_id); return TREXIO_FAILURE; } - + H5Aclose(str_id); H5Sclose(dspace_id); H5Tclose(dtype_id); @@ -709,5 +905,3 @@ trexio_hdf5_has_$group_str$ (trexio_t* const file) #endif #+end_src - - diff --git a/tools/generator.py b/tools/generator.py index 4d5ba32..f6cd9b8 100644 --- a/tools/generator.py +++ b/tools/generator.py @@ -15,6 +15,12 @@ dsets = get_dset_dict(trex_config) detailed_dsets_nostr, detailed_dsets_str, detailed_dsets_sparse = split_dset_dict_detailed(dsets) detailed_dsets = detailed_dsets_nostr.copy() detailed_dsets.update(detailed_dsets_str) +# build a big dictionary with all pre-processed data +detailed_all = {} +detailed_all['datasets'] = dict(detailed_dsets_nostr, **detailed_dsets_str, **detailed_dsets_sparse) +detailed_all['groups'] = group_dict +detailed_all['numbers'] = detailed_nums +detailed_all['strings'] = detailed_strs # consistency check for dimensioning variables check_dim_consistency(detailed_nums, dsets) # --------------------------------------------------------------------------- # @@ -33,7 +39,7 @@ files_todo = get_files_todo(source_files) # populate files with iterative scheme, i.e. for unique functions for fname in files_todo['auxiliary']: - iterative_populate_file(fname, template_paths, group_dict, detailed_dsets, detailed_nums, detailed_strs) + iterative_populate_file(fname, template_paths, detailed_all) # populate has/read/write_num functions with recursive scheme for fname in files_todo['attr_num']: diff --git a/tools/generator_tools.py b/tools/generator_tools.py index 166183b..f7a9215 100644 --- a/tools/generator_tools.py +++ b/tools/generator_tools.py @@ -181,17 +181,18 @@ def recursive_replace_line (input_line: str, triggers: list, source: dict) -> st return output_line -def iterative_populate_file (filename: str, paths: dict, groups: dict, datasets: dict, numbers: dict, strings: dict) -> None: +def iterative_populate_file (filename: str, paths: dict, detailed_all: dict) -> None: """ Iteratively populate files with unique functions that contain templated variables. Parameters: filename (str) : template file to be populated paths (dict) : dictionary of paths per source directory - groups (dict) : dictionary of groups - datasets (dict) : dictionary of datasets with substitution details - numbers (dict) : dictionary of numbers with substitution details - strings (dict) : dictionary of strings with substitution details + detailed_all(dict) : dictionary with substitution details with the following keys: + 'groups' : dictionary of groups with substitution details + 'datasets' : dictionary of datasets with substitution details + 'numbers' : dictionary of numbers with substitution details + 'strings' : dictionary of strings with substitution details Returns: None @@ -211,19 +212,19 @@ def iterative_populate_file (filename: str, paths: dict, groups: dict, datasets: if id == 0: # special case for proper error handling when deallocting text groups error_handler = ' if (rc != TREXIO_SUCCESS) return rc;\n' - populated_line = iterative_replace_line(line, '$group$', groups, add_line=error_handler) + populated_line = iterative_replace_line(line, '$group$', detailed_all['groups'], add_line=error_handler) f_out.write(populated_line) elif id == 1: - populated_line = iterative_replace_line(line, triggers[id], datasets, None) + populated_line = iterative_replace_line(line, triggers[id], detailed_all['datasets'], None) f_out.write(populated_line) elif id == 2: - populated_line = iterative_replace_line(line, triggers[id], numbers, None) + populated_line = iterative_replace_line(line, triggers[id], detailed_all['numbers'], None) f_out.write(populated_line) elif id == 3: - populated_line = iterative_replace_line(line, triggers[id], strings, None) + populated_line = iterative_replace_line(line, triggers[id], detailed_all['strings'], None) f_out.write(populated_line) elif id == 4: - populated_line = iterative_replace_line(line, triggers[id], groups, None) + populated_line = iterative_replace_line(line, triggers[id], detailed_all['groups'], None) f_out.write(populated_line) else: f_out.write(line)