TREX/trexio
1
0
Fork 0
mirror of https://github.com/TREX-CoE/trexio.git synced 2024-08-24 22:21:43 +02:00
Code Issues Projects Releases Wiki Activity

Works for int32_t arrays

Browse Source
This commit is contained in:
q-posev 2023-05-07 16:02:41 +02:00
parent dff995c13f
commit 8da662772c
No known key found for this signature in database
2 changed files with 225 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
src/templates_front/templator_front.org
View File

@ -1938,7 +1938,7 @@ trexio_pre_close (trexio_t* file)
4) ~dimensions~ - ~uint64_t*~ pointer to the array with the number of elements per dimension
4) ~datatype~ - ~char*~ string specifying the datatype (e.g. ~double/float~, ~int32/int64~)
5) ~name~ - ~char*~ lowercase string with the name of the data block (e.g. how it will appear in the TREXIO file)
**important for ~trexio_has_external~ to work!**
**important for ~trexio_has_external_[array|attribute]~ to work!**
**Output:**
- ~trexio_exit_code~
@ -1970,6 +1970,7 @@ for (suffix, c_type, _, _) in table:
# trexio_has function does not require datatype and is thus unique
result.append("trexio_exit_code trexio_has_external_array(trexio_t* const file, const char* name);")
result.append("trexio_exit_code trexio_has_external(trexio_t* const file);")
result.append("#+end_src")
return '\n'.join(result)
#+end_src
@ -1986,13 +1987,37 @@ trexio_exit_code trexio_read_external_float32_array(trexio_t* const file, float*
trexio_exit_code trexio_write_external_float64_array(trexio_t* const file, const double* array, const uint32_t rank, const uint64_t* dimensions, const char* name);
trexio_exit_code trexio_read_external_float64_array(trexio_t* const file, double* const array, const char* name);
trexio_exit_code trexio_has_external_array(trexio_t* const file, const char* name);
trexio_exit_code trexio_has_external(trexio_t* const file);
#+end_src
:end:
#+begin_src c :tangle prefix_front.c
trexio_exit_code
trexio_has_external(trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
assert(file->back_end < TREXIO_INVALID_BACK_END);
switch (file->back_end) {
case TREXIO_TEXT:
return TREXIO_NOT_SUPPORTED;
case TREXIO_HDF5:
#ifdef HAVE_HDF5
return trexio_hdf5_has_external(file);
#else
return TREXIO_BACK_END_MISSING;
#endif
}
return TREXIO_FAILURE;
}
#+end_src
#+begin_src c :tangle prefix_front.c
trexio_exit_code
trexio_write_external_int32_array(trexio_t* const file, const int32_t* array, const uint32_t rank, const uint64_t* dimensions, const char* name);
trexio_write_external_int32_array(trexio_t* const file, const int32_t* array, const uint32_t rank, const uint64_t* dimensions, const char* name)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (array == NULL) return TREXIO_INVALID_ARG_2;
@ -2003,7 +2028,7 @@ trexio_write_external_int32_array(trexio_t* const file, const int32_t* array, co
if (dimensions[i] == 0) return TREXIO_INVALID_ARG_4;
}
trexio_exit_code rc = trexio_has_external_array(file, name)
trexio_exit_code rc = trexio_has_external_array(file, name);
if (rc == TREXIO_SUCCESS && file->mode != 'u') return TREXIO_DSET_ALREADY_EXISTS;
if (rc != TREXIO_HAS_NOT && rc != TREXIO_SUCCESS) return rc;
@ -2023,13 +2048,13 @@ trexio_write_external_int32_array(trexio_t* const file, const int32_t* array, co
}
trexio_exit_code
trexio_read_external_int32_array(trexio_t* const file, int32_t* const array, const char* name);
trexio_read_external_int32_array(trexio_t* const file, int32_t* const array, const char* name)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (array == NULL) return TREXIO_INVALID_ARG_2;
if (name == NULL) return TREXIO_INVALID_ARG_3;
trexio_exit_code rc = trexio_has_external_array(file, name)
trexio_exit_code rc = trexio_has_external_array(file, name);
if (rc == TREXIO_HAS_NOT) return TREXIO_DSET_MISSING;
if (rc != TREXIO_HAS_NOT && rc != TREXIO_SUCCESS) return rc;
@ -2049,7 +2074,7 @@ trexio_read_external_int32_array(trexio_t* const file, int32_t* const array, con
}
trexio_exit_code
trexio_has_external_array(trexio_t* const file, const char* name);
trexio_has_external_array(trexio_t* const file, const char* name)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (name == NULL) return TREXIO_INVALID_ARG_2;

196
src/templates_hdf5/templator_hdf5.org
View File

@ -46,6 +46,7 @@
<<header>>
#include "trexio_hdf5.h"
#define EXTERNAL_GROUP_NAME "external"
#+end_src
* Template for HDF5 definitions
@ -69,6 +70,7 @@ typedef struct trexio_hdf5_s {
trexio_t parent ;
hid_t file_id;
hid_t $group$_group;
hid_t external_group;
} trexio_hdf5_t;
#+end_src
@ -158,18 +160,26 @@ trexio_hdf5_init (trexio_t* const file)
case 'r':
if (H5Lexists(f->file_id, $GROUP$_GROUP_NAME, H5P_DEFAULT) > 0) f->$group$_group = H5Gopen(f->file_id, $GROUP$_GROUP_NAME, H5P_DEFAULT);
if (H5Lexists(f->file_id, $GROUP$_GROUP_NAME, H5P_DEFAULT) == 0) f->$group$_group = (hid_t) 0;
/* Manual addition for the "external" group */
if (H5Lexists(f->file_id, EXTERNAL_GROUP_NAME, H5P_DEFAULT) > 0) f->external_group = H5Gopen(f->file_id, EXTERNAL_GROUP_NAME, H5P_DEFAULT);
if (H5Lexists(f->file_id, EXTERNAL_GROUP_NAME, H5P_DEFAULT) == 0) f->external_group = (hid_t) 0;
break;
case 'u':
case 'w':
if (f_exists == 1) {
if (H5Lexists(f->file_id, $GROUP$_GROUP_NAME, H5P_DEFAULT) > 0) f->$group$_group = H5Gopen(f->file_id, $GROUP$_GROUP_NAME, H5P_DEFAULT);
if (H5Lexists(f->file_id, $GROUP$_GROUP_NAME, H5P_DEFAULT) == 0) f->$group$_group = H5Gcreate(f->file_id, $GROUP$_GROUP_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
/* Manual addition for the "external" group */
if (H5Lexists(f->file_id, EXTERNAL_GROUP_NAME, H5P_DEFAULT) > 0) f->external_group = H5Gopen(f->file_id, EXTERNAL_GROUP_NAME, H5P_DEFAULT);
if (H5Lexists(f->file_id, EXTERNAL_GROUP_NAME, H5P_DEFAULT) == 0) f->external_group = H5Gcreate(f->file_id, EXTERNAL_GROUP_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
} else {
f->$group$_group = H5Gcreate(f->file_id, $GROUP$_GROUP_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
f->external_group = H5Gcreate(f->file_id, EXTERNAL_GROUP_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
break;
}
if (f->$group$_group < (hid_t) 0) return TREXIO_INVALID_ID;
if (f->external_group < (hid_t) 0) return TREXIO_INVALID_ID;
return TREXIO_SUCCESS;
}
@ -185,6 +195,9 @@ trexio_hdf5_deinit (trexio_t* const file)
if (f->$group$_group != (hid_t) 0) H5Gclose(f->$group$_group);
f->$group$_group = 0;
if (f->external_group != (hid_t) 0) H5Gclose(f->external_group);
f->external_group = 0;
H5Fclose(f->file_id);
f->file_id = 0;
@ -383,7 +396,7 @@ trexio_hdf5_read_$group_dset$ (trexio_t* const file, $group_dset_dtype$* const $
/* Read dataset */
herr_t status = H5Dread(dset_id,
H5T_$GROUP_DSET_H5_DTYPE$,
H5S_ALL, H5S_ALL, H5P_DEFAULT,
H5S_ALL, H5S_ALL, H5P_DEFAULT,
$group_dset$);
H5Sclose(dspace_id);
@ -413,7 +426,7 @@ trexio_hdf5_write_$group_dset$ (trexio_t* const file, const $group_dset_dtype$*
Consider using HDF5-native h5repack utility after deleting/overwriting big datasets.
,*/
if ((trexio_hdf5_has_$group_dset$(file) == TREXIO_SUCCESS) && (file->mode == 'u')) {
herr_t status_del = H5Ldelete(f->$group$_group, $GROUP_DSET$_NAME, H5P_DEFAULT);
if (status_del < 0) return TREXIO_FAILURE;
@ -1270,6 +1283,185 @@ trexio_exit_code trexio_hdf5_has_determinant_list(trexio_t* const file)
}
#+end_src
* Source code for external group (generic I/O of arbitrary data)
~trexio_hdf5_[write|read|has]_external_[datatype]_array~
~trexio_hdf5_[write|read|has]_external_[datatype]_attribute~
write|read|check for existence of an arbitrary data block in a given TREXIO file.
~external~ here means that the data does not correspond to the TREXIO format definition.
Thus, it is not present in the =trex.json= specification and does not support any of the
advanced TREXIO capabilities (e.g. sparse I/O, verification of dimensions etc).
**Input parameters:**
1) ~trexio_file~ - ~trexio_t*~ pointer to the TREXIO file
2) ~array~ - ~void*~ pointer to the flat array of data to be written
3) ~rank~ - ~uint32_t~ value: rank (number of dimensions) of an array
4) ~dimensions~ - ~uint64_t*~ pointer to the array with the number of elements per dimension
4) ~datatype~ - ~char*~ string specifying the datatype (e.g. ~double/float~, ~int32/int64~)
5) ~name~ - ~char*~ lowercase string with the name of the data block (e.g. how it will appear in the TREXIO file)
**important for ~trexio_has_external~ to work!**
**Output:**
- ~trexio_exit_code~
**Note 1:** experimental functionality. Please report any issues that occur.
**Note 2:** I/O of strings is not supported yet.
#+NAME: table-external-datatypes
| TREXIO suffix | C | Fortran | Python |
|---------------+-----------+---------+--------|
| ~int32~ | ~int32_t~ | | |
| ~int64~ | ~int64_t~ | | |
| ~float32~ | ~float~ | | |
| ~float64~ | ~double~ | | |
*** C
#+begin_src python :var table=table-external-datatypes :results drawer
""" This script generates the C and Fortran functions for generic I/O """
template_write_func = "trexio_exit_code trexio_hdf5_write_external_$suffix$_array(trexio_t* const file, const $c_type$* array, const uint32_t rank, const uint64_t* dimensions, const char* name);"
template_read_func = "trexio_exit_code trexio_hdf5_read_external_$suffix$_array(trexio_t* const file, $c_type$* const array, const char* name);"
result = []
result.append("#+begin_src c :tangle prefix_hdf5.h :exports none")
for (suffix, c_type, _, _) in table:
result.append(template_write_func.replace("$suffix$", suffix.replace("~","")).replace("$c_type$", c_type.replace("~","")))
result.append(template_read_func.replace("$suffix$", suffix.replace("~","")).replace("$c_type$", c_type.replace("~","")))
# trexio_has function does not require datatype and is thus unique
result.append("trexio_exit_code trexio_hdf5_has_external_array(trexio_t* const file, const char* name);")
result.append("trexio_exit_code trexio_hdf5_has_external(trexio_t* const file);")
result.append("#+end_src")
return '\n'.join(result)
#+end_src
#+RESULTS:
:results:
#+begin_src c :tangle prefix_hdf5.h :exports none
trexio_exit_code trexio_hdf5_write_external_int32_array(trexio_t* const file, const int32_t* array, const uint32_t rank, const uint64_t* dimensions, const char* name);
trexio_exit_code trexio_hdf5_read_external_int32_array(trexio_t* const file, int32_t* const array, const char* name);
trexio_exit_code trexio_hdf5_write_external_int64_array(trexio_t* const file, const int64_t* array, const uint32_t rank, const uint64_t* dimensions, const char* name);
trexio_exit_code trexio_hdf5_read_external_int64_array(trexio_t* const file, int64_t* const array, const char* name);
trexio_exit_code trexio_hdf5_write_external_float32_array(trexio_t* const file, const float* array, const uint32_t rank, const uint64_t* dimensions, const char* name);
trexio_exit_code trexio_hdf5_read_external_float32_array(trexio_t* const file, float* const array, const char* name);
trexio_exit_code trexio_hdf5_write_external_float64_array(trexio_t* const file, const double* array, const uint32_t rank, const uint64_t* dimensions, const char* name);
trexio_exit_code trexio_hdf5_read_external_float64_array(trexio_t* const file, double* const array, const char* name);
trexio_exit_code trexio_hdf5_has_external_array(trexio_t* const file, const char* name);
trexio_exit_code trexio_hdf5_has_external(trexio_t* const file);
#+end_src
:end:
#+begin_src c :tangle basic_hdf5.c :exports none
trexio_exit_code
trexio_hdf5_has_external (trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
const trexio_hdf5_t* f = (const trexio_hdf5_t*) file;
struct H5G_info_t group_info;
/* H5Gget_info return info about the HDF5 group as a group_info struct */
herr_t status = H5Gget_info(f->external_group, &group_info);
if (status < 0) return TREXIO_FAILURE;
/* If nlinks==0 --> the group is empty, i.e. non-existent */
if (group_info.nlinks == (hsize_t) 0) {
return TREXIO_HAS_NOT;
} else {
return TREXIO_SUCCESS;
}
}
#+end_src
#+begin_src c :tangle basic_hdf5.c :exports none
trexio_exit_code
trexio_hdf5_has_external_array(trexio_t* const file, const char* name)
{
trexio_hdf5_t* f = (trexio_hdf5_t*) file;
if (f->external_group == (hsize_t) 0) return TREXIO_HAS_NOT;
htri_t exists = H5Lexists(f->external_group, name, H5P_DEFAULT);
if (exists > 0) {
return TREXIO_SUCCESS;
} else if (exists < 0) {
return TREXIO_FAILURE;
} else {
return TREXIO_HAS_NOT;
}
}
trexio_exit_code
trexio_hdf5_read_external_int32_array(trexio_t* const file, int32_t* const array, const char* name)
{
const trexio_hdf5_t* f = (const trexio_hdf5_t*) file;
// open the dataset to get its dimensions
hid_t dset_id = H5Dopen(f->external_group, name, H5P_DEFAULT);
if (dset_id <= 0) return TREXIO_INVALID_ID;
/* Read dataset */
herr_t status = H5Dread(dset_id,
H5T_NATIVE_INT32,
H5S_ALL, H5S_ALL, H5P_DEFAULT,
array);
H5Dclose(dset_id);
if (status < 0) return TREXIO_FAILURE;
return TREXIO_SUCCESS;
}
trexio_exit_code
trexio_hdf5_write_external_int32_array(trexio_t* const file, const int32_t* array, const uint32_t rank, const uint64_t* dimensions, const char* name)
{
trexio_hdf5_t* f = (trexio_hdf5_t*) file;
/*
Try to delete an existing dataset by unlinking it from the group (UNSAFE mode).
NOTE: In principle, HDF5 should see the deallocated (unused) file space and free it,
thus reducing the size of the HDF5 file. In practic, this is not always the case.
Consider using HDF5-native h5repack utility after deleting/overwriting big datasets.
,*/
if ((trexio_hdf5_has_external_array(file, name) == TREXIO_SUCCESS) && (file->mode == 'u')) {
herr_t status_del = H5Ldelete(f->external_group, name, H5P_DEFAULT);
if (status_del < 0) return TREXIO_FAILURE;
}
hid_t dspace_id = H5Screate_simple( (int) rank, (const hsize_t*) dimensions, NULL);
if (dspace_id <= 0) return TREXIO_INVALID_ID;
hid_t dset_id = H5Dcreate(f->external_group,
name,
H5T_NATIVE_INT32,
dspace_id,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (dset_id <= 0) {
H5Sclose(dspace_id);
return TREXIO_INVALID_ID;
}
herr_t status = H5Dwrite(dset_id,
H5T_NATIVE_INT32,
H5S_ALL,
dspace_id,
H5P_DEFAULT,
array);
H5Dclose(dset_id);
H5Sclose(dspace_id);
if (status < 0) return TREXIO_FAILURE;
return TREXIO_SUCCESS;
}
#+end_src
* Helper functions
#+begin_src c :tangle helpers_hdf5.c