TREX/trexio
1
0
Fork 0
mirror of https://github.com/TREX-CoE/trexio.git synced 2024-11-03 20:54:07 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #101 from TREX-CoE/float-buffered

Browse Source 
Float buffered, stateless formatting and flexible sparse compression
This commit is contained in:
Evgeny Posenitskiy 2022-10-06 15:34:45 +02:00 committed by GitHub
commit 65269c6e69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 970 additions and 963 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
python/test/test_api.py
View File

@ -182,15 +182,11 @@ class TestIO:
assert int_num == int64_num
# write the data for the ground state
offset = 0
trexio.write_state_id(self.test_file, 0)
trexio.write_determinant_list(self.test_file, offset, det_num, dets)
assert trexio.has_determinant_list(self.test_file)
trexio.write_determinant_coefficient(self.test_file, offset, det_num, coeffs)
assert trexio.has_determinant_coefficient(self.test_file)
# write the data for some other state
self.test_file.set_state(2)
trexio.write_determinant_coefficient(self.test_file, offset, det_num, coeffs_s2)
assert trexio.has_determinant_coefficient(self.test_file)
self.test_file.set_state(0)
# manually check the consistency between coefficient_size and number of determinants
assert trexio.read_determinant_coefficient_size(self.test_file) == trexio.read_determinant_num(self.test_file)

874
src/templates_front/templator_front.org
View File

File diff suppressed because it is too large Load Diff

289
src/templates_hdf5/templator_hdf5.org
View File

@ -670,6 +670,141 @@ trexio_hdf5_has_$group_dset$ (trexio_t* const file)
}
#+end_src
* Template for HDF5 has/read/write a dataset of buffered vectors
Chunked I/O in HDF5 for ~buffered~ data.
#+begin_src c :tangle hrw_buffered_hdf5.h :exports none
trexio_exit_code trexio_hdf5_has_$group_dset$(trexio_t* const file);
trexio_exit_code trexio_hdf5_read_$group_dset$(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, int64_t* const eof_read_size, double* const dset);
trexio_exit_code trexio_hdf5_write_$group_dset$(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, const double* dset);
trexio_exit_code trexio_hdf5_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max);
#+end_src
#+begin_src c :tangle read_buffered_hdf5.c
trexio_exit_code trexio_hdf5_read_$group_dset$(trexio_t* const file,
const int64_t offset_file,
const uint32_t rank,
const uint64_t* dims,
int64_t* const eof_read_size,
double* const dset)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (eof_read_size == NULL) return TREXIO_INVALID_ARG_5;
if (dset == NULL) return TREXIO_INVALID_ARG_6;
const char dset_name[256] = "$group_dset$";
const trexio_hdf5_t* f = (const trexio_hdf5_t*) file;
hsize_t offset[1] = {(hsize_t) offset_file};
hsize_t count[1] = {(hsize_t) dims[0]};
/* Attempt to read values (if EOF -> eof_read_size is modified with the number of elements read and return code is TREXIO_END)
0 argument below is requires to skip internal treatment specific to sparse indices (i.e. their de-compression).*/
return trexio_hdf5_open_read_dset_sparse(f->$group$_group, dset_name, 1, offset, count, eof_read_size, 0, dset);
}
#+end_src
#+begin_src c :tangle write_buffered_hdf5.c
trexio_exit_code trexio_hdf5_write_$group_dset$(trexio_t* const file,
const int64_t offset_file,
const uint32_t rank,
const uint64_t* dims,
const double* dset)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (dset == NULL) return TREXIO_INVALID_ARG_5;
const char dset_name[256] = "$group_dset$";
trexio_hdf5_t* f = (trexio_hdf5_t*) file;
hid_t dtype = H5T_NATIVE_DOUBLE;
/* Arrays of chunk dims that will be used for chunking the dataset */
const hsize_t chunk_dims[1] = {(hsize_t) dims[0]};
trexio_exit_code rc_write = TREXIO_FAILURE;
/* NOTE: chunk size is set upon creation of the HDF5 dataset and cannot be changed ! */
if ( H5LTfind_dataset(f->$group$_group, dset_name) != 1 ) {
/* If the file does not exist -> create it and write */
/* Create chunked dataset with dtype datatype and write indices into it */
rc_write = trexio_hdf5_create_write_dset_sparse(f->$group$_group, dset_name, dtype, chunk_dims, dset);
if (rc_write != TREXIO_SUCCESS) return rc_write;
} else {
/* If the file exists -> open it and write */
hsize_t offset_data[1] = {(hsize_t) offset_file};
/* Create chunked dataset with dtype datatype and write indices into it */
rc_write = trexio_hdf5_open_write_dset_sparse(f->$group$_group, dset_name, dtype, chunk_dims, offset_data, dset);
if (rc_write != TREXIO_SUCCESS) return rc_write;
}
return TREXIO_SUCCESS;
}
trexio_exit_code
trexio_hdf5_read_$group_dset$_size (trexio_t* const file, int64_t* const size_max)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (size_max == NULL) return TREXIO_INVALID_ARG_2;
const char dset_name[256] = "$group_dset$";
const trexio_hdf5_t* f = (const trexio_hdf5_t*) file;
hid_t dset_id = H5Dopen(f->$group$_group, dset_name, H5P_DEFAULT);
if (dset_id <= 0) return TREXIO_INVALID_ID;
hid_t fspace_id = H5Dget_space(dset_id);
if (fspace_id < 0) {
H5Dclose(dset_id);
return TREXIO_INVALID_ID;
}
// allocate space for the dimensions to be read
hsize_t ddims[1] = {0};
// get the rank and dimensions of the dataset
H5Sget_simple_extent_dims(fspace_id, ddims, NULL);
H5Dclose(dset_id);
H5Sclose(fspace_id);
*size_max = (int64_t) ddims[0];
return TREXIO_SUCCESS;
}
#+end_src
#+begin_src c :tangle has_buffered_hdf5.c
trexio_exit_code trexio_hdf5_has_$group_dset$(trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
trexio_hdf5_t* f = (trexio_hdf5_t*) file;
if (f->$group$_group == (hsize_t) 0) return TREXIO_HAS_NOT;
const char dset_name[256] = "$group_dset$";
herr_t status = H5LTfind_dataset(f->$group$_group, dset_name);
/* H5LTfind_dataset returns 1 if dataset exists, 0 otherwise */
if (status == 1){
return TREXIO_SUCCESS;
} else if (status == 0) {
return TREXIO_HAS_NOT;
} else {
return TREXIO_FAILURE;
}
}
#+end_src
* Template for HDF5 has/read/write a dataset of strings
#+begin_src c :tangle hrw_dset_str_hdf5.h :exports none
@ -1045,10 +1180,6 @@ trexio_hdf5_delete_$group$ (trexio_t* const file)
trexio_exit_code trexio_hdf5_has_determinant_list(trexio_t* const file);
trexio_exit_code trexio_hdf5_read_determinant_list(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, int64_t* const eof_read_size, int64_t* const list);
trexio_exit_code trexio_hdf5_write_determinant_list(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, const int64_t* list);
trexio_exit_code trexio_hdf5_has_determinant_coefficient(trexio_t* const file);
trexio_exit_code trexio_hdf5_read_determinant_coefficient(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, int64_t* const eof_read_size, double* const coeff);
trexio_exit_code trexio_hdf5_write_determinant_coefficient(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, const double* coeff);
trexio_exit_code trexio_hdf5_read_determinant_coefficient_size(trexio_t* const file, int64_t* const size_max);
#+end_src
#+begin_src c :tangle read_determinant_hdf5.c
@ -1074,37 +1205,6 @@ trexio_exit_code trexio_hdf5_read_determinant_list(trexio_t* const file,
0 argument below is requires to skip internal treatment specific to sparse indices (i.e. their de-compression).*/
return trexio_hdf5_open_read_dset_sparse(f->determinant_group, dset_det_name, (uint32_t) dims[1], offset, count, eof_read_size, 0, list);
}
trexio_exit_code trexio_hdf5_read_determinant_coefficient(trexio_t* const file,
const int64_t offset_file,
const uint32_t rank,
const uint64_t* dims,
int64_t* const eof_read_size,
double* const coeff)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (eof_read_size == NULL) return TREXIO_INVALID_ARG_5;
if (coeff == NULL) return TREXIO_INVALID_ARG_6;
char dset_coeff_name[128];
memset(dset_coeff_name, 0, sizeof(dset_coeff_name));
const int32_t trexio_state = file->state;
if (trexio_state != 0) {
sprintf(dset_coeff_name, "determinant_coefficient_state_%" PRId32, trexio_state);
} else {
strncpy(dset_coeff_name, "determinant_coefficient", 24);
}
const trexio_hdf5_t* f = (const trexio_hdf5_t*) file;
hsize_t offset[1] = {(hsize_t) offset_file};
hsize_t count[1] = {(hsize_t) dims[0]};
/* Attempt to read determinants (if EOF -> eof_read_size is modified with the number of elements read and return code is TREXIO_END)
0 argument below is requires to skip internal treatment specific to sparse indices (i.e. their de-compression).*/
return trexio_hdf5_open_read_dset_sparse(f->determinant_group, dset_coeff_name, 1, offset, count, eof_read_size, 0, coeff);
}
#+end_src
#+begin_src c :tangle write_determinant_hdf5.c
@ -1150,97 +1250,6 @@ trexio_exit_code trexio_hdf5_write_determinant_list(trexio_t* const file,
return TREXIO_SUCCESS;
}
trexio_exit_code trexio_hdf5_write_determinant_coefficient(trexio_t* const file,
const int64_t offset_file,
const uint32_t rank,
const uint64_t* dims,
const double* coeff)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (coeff == NULL) return TREXIO_INVALID_ARG_5;
char dset_coeff_name[128];
memset(dset_coeff_name, 0, sizeof(dset_coeff_name));
const int32_t trexio_state = file->state;
if (trexio_state != 0) {
sprintf(dset_coeff_name, "determinant_coefficient_state_%" PRId32, trexio_state);
} else {
strncpy(dset_coeff_name, "determinant_coefficient", 24);
}
trexio_hdf5_t* f = (trexio_hdf5_t*) file;
hid_t det_dtype = H5T_NATIVE_DOUBLE;
/* Arrays of chunk dims that will be used for chunking the dataset */
const hsize_t chunk_dims[1] = {(hsize_t) dims[0]};
trexio_exit_code rc_write = TREXIO_FAILURE;
/* NOTE: chunk size is set upon creation of the HDF5 dataset and cannot be changed ! */
if ( H5LTfind_dataset(f->determinant_group, dset_coeff_name) != 1 ) {
/* If the file does not exist -> create it and write */
/* Create chunked dataset with det_dtype datatype and write indices into it */
rc_write = trexio_hdf5_create_write_dset_sparse(f->determinant_group, dset_coeff_name, det_dtype, chunk_dims, coeff);
if (rc_write != TREXIO_SUCCESS) return rc_write;
} else {
/* If the file exists -> open it and write */
hsize_t offset_data[1] = {(hsize_t) offset_file};
/* Create chunked dataset with det_dtype datatype and write indices into it */
rc_write = trexio_hdf5_open_write_dset_sparse(f->determinant_group, dset_coeff_name, det_dtype, chunk_dims, offset_data, coeff);
if (rc_write != TREXIO_SUCCESS) return rc_write;
}
return TREXIO_SUCCESS;
}
trexio_exit_code
trexio_hdf5_read_determinant_coefficient_size (trexio_t* const file, int64_t* const size_max)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (size_max == NULL) return TREXIO_INVALID_ARG_2;
char dset_coeff_name[128];
memset(dset_coeff_name, 0, sizeof(dset_coeff_name));
const int32_t trexio_state = file->state;
if (trexio_state != 0) {
sprintf(dset_coeff_name, "determinant_coefficient_state_%" PRId32, trexio_state);
} else {
strncpy(dset_coeff_name, "determinant_coefficient", 24);
}
const trexio_hdf5_t* f = (const trexio_hdf5_t*) file;
hid_t dset_id = H5Dopen(f->determinant_group, dset_coeff_name, H5P_DEFAULT);
if (dset_id <= 0) return TREXIO_INVALID_ID;
hid_t fspace_id = H5Dget_space(dset_id);
if (fspace_id < 0) {
H5Dclose(dset_id);
return TREXIO_INVALID_ID;
}
// allocate space for the dimensions to be read
hsize_t ddims[1] = {0};
// get the rank and dimensions of the dataset
H5Sget_simple_extent_dims(fspace_id, ddims, NULL);
H5Dclose(dset_id);
H5Sclose(fspace_id);
*size_max = (int64_t) ddims[0];
return TREXIO_SUCCESS;
}
#+end_src
#+begin_src c :tangle has_determinant_hdf5.c
@ -1261,34 +1270,6 @@ trexio_exit_code trexio_hdf5_has_determinant_list(trexio_t* const file)
return TREXIO_FAILURE;
}
}
trexio_exit_code trexio_hdf5_has_determinant_coefficient(trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
trexio_hdf5_t* f = (trexio_hdf5_t*) file;
if (f->determinant_group == (hsize_t) 0) return TREXIO_HAS_NOT;
char dset_coeff_name[128];
memset(dset_coeff_name, 0, sizeof(dset_coeff_name));
const int32_t trexio_state = file->state;
if (trexio_state != 0) {
sprintf(dset_coeff_name, "determinant_coefficient_state_%" PRId32, trexio_state);
} else {
strncpy(dset_coeff_name, "determinant_coefficient", 24);
}
herr_t status = H5LTfind_dataset(f->determinant_group, dset_coeff_name);
/* H5LTfind_dataset returns 1 if dataset exists, 0 otherwise */
if (status == 1){
return TREXIO_SUCCESS;
} else if (status == 0) {
return TREXIO_HAS_NOT;
} else {
return TREXIO_FAILURE;
}
}
#+end_src
* Helper functions

4
src/templates_text/build.sh
View File

@ -30,23 +30,27 @@ cat populated/pop_has_dset_str_text.c >> trexio_text.c
cat populated/pop_has_dset_sparse_text.c >> trexio_text.c
cat populated/pop_has_attr_num_text.c >> trexio_text.c
cat populated/pop_has_attr_str_text.c >> trexio_text.c
cat populated/pop_has_buffered_text.c >> trexio_text.c
cat populated/pop_read_dset_data_text.c >> trexio_text.c
cat populated/pop_read_dset_str_text.c >> trexio_text.c
cat populated/pop_read_dset_sparse_text.c >> trexio_text.c
cat populated/pop_read_attr_str_text.c >> trexio_text.c
cat populated/pop_read_attr_num_text.c >> trexio_text.c
cat populated/pop_read_buffered_text.c >> trexio_text.c
cat populated/pop_write_dset_data_text.c >> trexio_text.c
cat populated/pop_write_dset_str_text.c >> trexio_text.c
cat populated/pop_write_dset_sparse_text.c >> trexio_text.c
cat populated/pop_write_attr_str_text.c >> trexio_text.c
cat populated/pop_write_attr_num_text.c >> trexio_text.c
cat populated/pop_write_buffered_text.c >> trexio_text.c
cat populated/pop_hrw_dset_data_text.h >> trexio_text.h
cat populated/pop_hrw_dset_str_text.h >> trexio_text.h
cat populated/pop_hrw_dset_sparse_text.h >> trexio_text.h
cat populated/pop_hrw_attr_num_text.h >> trexio_text.h
cat populated/pop_hrw_attr_str_text.h >> trexio_text.h
cat populated/pop_hrw_buffered_text.h >> trexio_text.h
cat suffix_text.h >> trexio_text.h

482
src/templates_text/templator_text.org
View File

@ -1333,6 +1333,236 @@ trexio_exit_code trexio_text_has_$group_dset$(trexio_t* const file)
}
#+end_src
* Template for has/read/write a buffered vector
Each array is stored in a separate =.txt= file due to the fact that buffered I/O has to be decoupled
from conventional write/read/flush behaviour of the TEXT back end. Chunks are used to read/write the data
to prevent memory overflow. Chunks have a given ~int64_t size~.
Size specifies the number of vector elements to be written.
#+begin_src c :tangle hrw_buffered_text.h :exports none
trexio_exit_code trexio_text_has_$group_dset$(trexio_t* const file);
trexio_exit_code trexio_text_read_$group_dset$(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, int64_t* const eof_read_size, double* const dset);
trexio_exit_code trexio_text_write_$group_dset$(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, const double* dset);
trexio_exit_code trexio_text_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max);
#+end_src
#+begin_src c :tangle read_buffered_text.c
trexio_exit_code trexio_text_read_$group_dset$(trexio_t* const file,
const int64_t offset_file,
const uint32_t rank,
const uint64_t* dims,
int64_t* const eof_read_size,
double* const dset)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (eof_read_size == NULL) return TREXIO_INVALID_ARG_5;
if (dset == NULL) return TREXIO_INVALID_ARG_6;
const char file_name[256] = "/$group_dset$.txt";
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, file_name, TREXIO_MAX_FILENAME_LENGTH - sizeof(file_name));
/* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */
FILE* f = fopen(file_full_path, "r");
if (f == NULL) return TREXIO_FILE_ERROR;
/* Specify the line length in order to offset properly.
Each double value 24 elements + one newline char.
,*/
uint64_t line_length = 25UL;
/* Offset in the file according to the provided value of offset_file and optimal line_length */
fseek(f, (long) offset_file * line_length, SEEK_SET);
/* Read the data from the file and check the return code of fprintf to verify that > 0 bytes have been read or reached EOF */
int rc;
char buffer[64];
uint32_t buf_size = sizeof(buffer);
/* Counter for number of elements beind processed */
uint64_t count = 0UL;
for (uint64_t i=0UL; i < dims[0]; ++i) {
memset(buffer, 0, buf_size);
if (fgets(buffer, buf_size-1, f) == NULL){
fclose(f);
,*eof_read_size = count;
return TREXIO_END;
} else {
rc = sscanf(buffer, "%lf", dset + i);
if (rc <= 0) {
fclose(f);
return TREXIO_FAILURE;
}
count += 1UL;
}
}
/* Close the TXT file */
rc = fclose(f);
if (rc != 0) return TREXIO_FILE_ERROR;
return TREXIO_SUCCESS;
}
trexio_exit_code
trexio_text_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (size_max == NULL) return TREXIO_INVALID_ARG_2;
const char file_name[256] = "/$group_dset$.txt.size";
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, file_name, TREXIO_MAX_FILENAME_LENGTH - sizeof(file_name));
/* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */
FILE* f = fopen(file_full_path, "r");
if (f == NULL) return TREXIO_FILE_ERROR;
/* Read the data from the file and check the return code of fprintf to verify that > 0 bytes have been read or reached EOF */
int rc;
int64_t size_item, size_accum=0L;
/* Read the values from the file. BEWARE OF POSSIBLE MAX_INT64 OVERFLOW ! */
while(fscanf(f, "%" SCNd64, &size_item) != EOF) {
/* Check that summation will not overflow the int64_t value */
if (INT64_MAX - size_accum > size_item) {
size_accum += size_item;
} else {
fclose(f);
,*size_max = -1L;
return TREXIO_INT_SIZE_OVERFLOW;
}
}
/* Close the TXT file */
rc = fclose(f);
if (rc != 0) return TREXIO_FILE_ERROR;
/* Overwrite the value at the input address and return TREXIO_SUCCESS */
,*size_max = size_accum;
return TREXIO_SUCCESS;
}
#+end_src
#+begin_src c :tangle write_buffered_text.c
trexio_exit_code trexio_text_write_$group_dset$(trexio_t* const file,
const int64_t offset_file,
const uint32_t rank,
const uint64_t* dims,
const double* dset)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (dset == NULL) return TREXIO_INVALID_ARG_5;
const char file_name[256] = "/$group_dset$.txt";
const int append_str_len = 6;
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, file_name, TREXIO_MAX_FILENAME_LENGTH - sizeof(file_name));
/* Open the file in "a" (append) mode to guarantee that no truncation happens upon consecutive writes */
FILE* f = fopen(file_full_path, "a");
if (f == NULL) return TREXIO_FILE_ERROR;
/* Write the data in the file and check the return code of fprintf to verify that > 0 bytes have been written */
int rc;
for (uint64_t i=0UL; i < dims[0]; ++i) {
rc = fprintf(f, "%24.16e\n", *(dset+ i));
if (rc <= 0) {
fclose(f);
return TREXIO_FAILURE;
}
}
/* Close the TXT file */
rc = fclose(f);
if (rc != 0) return TREXIO_FILE_ERROR;
/* Append .size to the file_full_path in order to write additional info about the written buffer of data */
strncat(file_full_path, ".size", append_str_len);
/* Open the new file in "a" (append) mode to append info about the buffer that has been just written */
FILE *f_wSize = fopen(file_full_path, "a");
if (f_wSize == NULL) return TREXIO_FILE_ERROR;
/* Write the buffer_size */
rc = fprintf(f_wSize, "%" PRIu64 "\n", dims[0]);
if (rc <= 0) {
fclose(f_wSize);
return TREXIO_FAILURE;
}
/* Close the TXT file */
rc = fclose(f_wSize);
if (rc != 0) return TREXIO_FILE_ERROR;
/* Additional part for the trexio_text_has_group to work */
const char group_file_name[256] = "/$group$.txt";
memset (file_full_path, 0, TREXIO_MAX_FILENAME_LENGTH);
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, group_file_name, TREXIO_MAX_FILENAME_LENGTH - sizeof(group_file_name));
bool file_exists = trexio_text_file_exists(file_full_path);
/* Create an empty file for the trexio_text_has_group to work */
if (!file_exists) {
FILE *fp = fopen(file_full_path, "ab+");
fclose(fp);
}
/* Exit upon success */
return TREXIO_SUCCESS;
}
#+end_src
#+begin_src c :tangle has_buffered_text.c
trexio_exit_code trexio_text_has_$group_dset$(trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
const char file_name[256] = "/$group_dset$.txt";
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, file_name, TREXIO_MAX_FILENAME_LENGTH - sizeof(file_name));
/* Check the return code of access function to determine whether the file with data exists or not */
if (access(file_full_path, F_OK) == 0){
return TREXIO_SUCCESS;
} else {
return TREXIO_HAS_NOT;
}
}
#+end_src
* Template for text delete a group (UNSAFE mode)
#+begin_src c :tangle delete_group_text.h :exports none
@ -1374,10 +1604,6 @@ trexio_text_delete_$group$ (trexio_t* const file)
trexio_exit_code trexio_text_has_determinant_list(trexio_t* const file);
trexio_exit_code trexio_text_read_determinant_list(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, int64_t* const eof_read_size, int64_t* const list);
trexio_exit_code trexio_text_write_determinant_list(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, const int64_t* list);
trexio_exit_code trexio_text_has_determinant_coefficient(trexio_t* const file);
trexio_exit_code trexio_text_read_determinant_coefficient(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, int64_t* const eof_read_size, double* const coeff);
trexio_exit_code trexio_text_write_determinant_coefficient(trexio_t* const file, const int64_t offset_file, const uint32_t rank, const uint64_t* dims, const double* coeff);
trexio_exit_code trexio_text_read_determinant_coefficient_size(trexio_t* const file, int64_t* const size_max);
#+end_src
#+begin_src c :tangle read_determinant_text.c
@ -1462,136 +1688,6 @@ trexio_exit_code trexio_text_read_determinant_list(trexio_t* const file,
return TREXIO_SUCCESS;
}
trexio_exit_code trexio_text_read_determinant_coefficient(trexio_t* const file,
const int64_t offset_file,
const uint32_t rank,
const uint64_t* dims,
int64_t* const eof_read_size,
double* const coeff)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (eof_read_size == NULL) return TREXIO_INVALID_ARG_5;
if (coeff == NULL) return TREXIO_INVALID_ARG_6;
char coeff_file_name[256];
memset(coeff_file_name, 0, sizeof(coeff_file_name));
const int32_t trexio_state = file->state;
if (trexio_state != 0) {
sprintf(coeff_file_name, "/determinant_coefficient_state_%" PRId32 ".txt", trexio_state);
} else {
strncpy(coeff_file_name, "/determinant_coefficient.txt", 32);
}
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, coeff_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen(coeff_file_name));
/* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */
FILE* f = fopen(file_full_path, "r");
if (f == NULL) return TREXIO_FILE_ERROR;
/* Specify the line length in order to offset properly.
Each double value 24 elements + one newline char.
,*/
uint64_t line_length = 25UL;
/* Offset in the file according to the provided value of offset_file and optimal line_length */
fseek(f, (long) offset_file * line_length, SEEK_SET);
/* Read the data from the file and check the return code of fprintf to verify that > 0 bytes have been read or reached EOF */
int rc;
/* Declare fixed buffer which will be used to read the determinant string <a1 a2 ... a/\ b1 b2 ... b\/> */
char buffer[64];
uint32_t buf_size = sizeof(buffer);
/* Counter for number of elements beind processed */
uint64_t count = 0UL;
for (uint64_t i=0UL; i < dims[0]; ++i) {
memset(buffer, 0, buf_size);
if (fgets(buffer, buf_size-1, f) == NULL){
fclose(f);
,*eof_read_size = count;
return TREXIO_END;
} else {
rc = sscanf(buffer, "%lf", coeff + i);
if (rc <= 0) {
fclose(f);
return TREXIO_FAILURE;
}
count += 1UL;
}
}
/* Close the TXT file */
rc = fclose(f);
if (rc != 0) return TREXIO_FILE_ERROR;
return TREXIO_SUCCESS;
}
trexio_exit_code
trexio_text_read_determinant_coefficient_size(trexio_t* const file, int64_t* const size_max)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (size_max == NULL) return TREXIO_INVALID_ARG_2;
char coeff_file_name[256];
memset(coeff_file_name, 0, sizeof(coeff_file_name));
const int32_t trexio_state = file->state;
if (trexio_state != 0) {
sprintf(coeff_file_name, "/determinant_coefficient_state_%" PRId32 ".txt.size", trexio_state);
} else {
strncpy(coeff_file_name, "/determinant_coefficient.txt.size", 64);
}
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, coeff_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen(coeff_file_name));
/* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */
FILE* f = fopen(file_full_path, "r");
if (f == NULL) return TREXIO_FILE_ERROR;
/* Read the data from the file and check the return code of fprintf to verify that > 0 bytes have been read or reached EOF */
int rc;
int64_t size_item, size_accum=0L;
/* Read the values from the file. BEWARE OF POSSIBLE MAX_INT64 OVERFLOW ! */
while(fscanf(f, "%" SCNd64, &size_item) != EOF) {
/* Check that summation will not overflow the int64_t value */
if (INT64_MAX - size_accum > size_item) {
size_accum += size_item;
} else {
fclose(f);
,*size_max = -1L;
return TREXIO_INT_SIZE_OVERFLOW;
}
}
/* Close the TXT file */
rc = fclose(f);
if (rc != 0) return TREXIO_FILE_ERROR;
/* Overwrite the value at the input address and return TREXIO_SUCCESS */
,*size_max = size_accum;
return TREXIO_SUCCESS;
}
#+end_src
#+begin_src c :tangle write_determinant_text.c
@ -1659,94 +1755,6 @@ trexio_exit_code trexio_text_write_determinant_list(trexio_t* const file,
/* Exit upon success */
return TREXIO_SUCCESS;
}
trexio_exit_code trexio_text_write_determinant_coefficient(trexio_t* const file,
const int64_t offset_file,
const uint32_t rank,
const uint64_t* dims,
const double* coeff)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
if (coeff == NULL) return TREXIO_INVALID_ARG_5;
char coeff_file_name[256];
memset(coeff_file_name, 0, sizeof(coeff_file_name));
const int32_t trexio_state = file->state;
if (trexio_state != 0) {
sprintf(coeff_file_name, "/determinant_coefficient_state_%" PRId32 ".txt", trexio_state);
} else {
strncpy(coeff_file_name, "/determinant_coefficient.txt", 32);
}
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, coeff_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen(coeff_file_name));
/* Open the file in "a" (append) mode to guarantee that no truncation happens upon consecutive writes */
FILE* f = fopen(file_full_path, "a");
if (f == NULL) return TREXIO_FILE_ERROR;
/* Write the data in the file and check the return code of fprintf to verify that > 0 bytes have been written */
int rc;
for (uint64_t i=0UL; i < dims[0]; ++i) {
rc = fprintf(f, "%24.16e\n", *(coeff + i));
if (rc <= 0) {
fclose(f);
return TREXIO_FAILURE;
}
}
/* Close the TXT file */
rc = fclose(f);
if (rc != 0) return TREXIO_FILE_ERROR;
/* Append .size to the file_full_path in order to write additional info about the written buffer of data */
strncat(file_full_path, ".size", 6);
/* Open the new file in "a" (append) mode to append info about the buffer that has been just written */
FILE *f_wSize = fopen(file_full_path, "a");
if (f_wSize == NULL) return TREXIO_FILE_ERROR;
/* Write the buffer_size */
rc = fprintf(f_wSize, "%" PRIu64 "\n", dims[0]);
if (rc <= 0) {
fclose(f_wSize);
return TREXIO_FAILURE;
}
/* Close the TXT file */
rc = fclose(f_wSize);
if (rc != 0) return TREXIO_FILE_ERROR;
/* Additional part for the trexio_text_has_group to work */
const char det_file_name[256] = "/determinant.txt";
memset (file_full_path, 0, TREXIO_MAX_FILENAME_LENGTH);
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, det_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen(det_file_name));
bool file_exists = trexio_text_file_exists(file_full_path);
/* Create an empty file for the trexio_text_has_group to work */
if (!file_exists) {
FILE *fp = fopen(file_full_path, "ab+");
fclose(fp);
}
/* Exit upon success */
return TREXIO_SUCCESS;
}
#+end_src
#+begin_src c :tangle has_determinant_text.c
@ -1771,36 +1779,6 @@ trexio_exit_code trexio_text_has_determinant_list(trexio_t* const file)
return TREXIO_HAS_NOT;
}
}
trexio_exit_code trexio_text_has_determinant_coefficient(trexio_t* const file)
{
if (file == NULL) return TREXIO_INVALID_ARG_1;
char coeff_file_name[256];
memset(coeff_file_name, 0, sizeof(coeff_file_name));
const int32_t trexio_state = file->state;
if (trexio_state != 0) {
sprintf(coeff_file_name, "/determinant_coefficient_state_%" PRId32 ".txt", trexio_state);
} else {
strncpy(coeff_file_name, "/determinant_coefficient.txt", 32);
}
/* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
/* Copy directory name in file_full_path */
strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
/* Append name of the file with sparse data */
strncat (file_full_path, coeff_file_name,
TREXIO_MAX_FILENAME_LENGTH-strlen(coeff_file_name));
/* Check the return code of access function to determine whether the file with data exists or not */
if (access(file_full_path, F_OK) == 0){
return TREXIO_SUCCESS;
} else {
return TREXIO_HAS_NOT;
}
}
#+end_src
* Constant file suffixes (not used by the generator) :noexport:

2
tests/delete_group_text.c
View File

@ -5,7 +5,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_del.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_write_delete_group (const char* file_name, const back_end_t backend) {

4
tests/io_all.c
View File

@ -26,11 +26,11 @@ int main() {
assert (rc == 0);
}
rc = system("rm -rf test_all.dir");
rc = system("rm -f -- test_all.dir/*.txt test_all.dir/*.txt.size test_all.dir/.lock && rm -fd -- test_all.dir");
assert (rc == 0);
test_write("test_all.dir", TREXIO_TEXT);
test_read ("test_all.dir", TREXIO_TEXT);
rc = system("rm -rf test_all.dir");
rc = system("rm -f -- test_all.dir/*.txt test_all.dir/*.txt.size test_all.dir/.lock && rm -fd -- test_all.dir");
assert (rc == 0);
return 0;

49
tests/io_determinant_hdf5.c
View File

@ -6,7 +6,7 @@
#define TEST_BACKEND TREXIO_HDF5
#define TREXIO_FILE "test_determinant.h5"
#define RM_COMMAND "rm -f " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE
#define SIZE 100
#define N_CHUNKS 5
#define STATE_TEST 2
@ -64,6 +64,12 @@ static int test_write_determinant (const char* file_name, const back_end_t backe
uint64_t offset_d = 0UL;
if (offset != 0L) offset_f += offset;
// write the state_id of a given file: 0 is ground state
if (trexio_has_state_id(file) == TREXIO_HAS_NOT) {
rc = trexio_write_state_id(file, 0);
assert(rc == TREXIO_SUCCESS);
}
// write n_chunks times using write_sparse
for(int i=0; i<N_CHUNKS; ++i){
@ -73,13 +79,10 @@ static int test_write_determinant (const char* file_name, const back_end_t backe
rc = trexio_write_determinant_coefficient(file, offset_f, chunk_size, &det_coef[offset_d]);
assert(rc == TREXIO_SUCCESS);
// The block below will write the coefficients for STATE_TEST
// The block below will check the set_state
rc = trexio_set_state(file, STATE_TEST);
assert(rc == TREXIO_SUCCESS);
rc = trexio_write_determinant_coefficient(file, offset_f, chunk_size, &det_coef[offset_d]);
assert(rc == TREXIO_SUCCESS);
// set state back to the default 0 (ground state)
rc = trexio_set_state(file, 0);
assert(rc == TREXIO_SUCCESS);
@ -100,13 +103,6 @@ static int test_write_determinant (const char* file_name, const back_end_t backe
assert(rc == TREXIO_SUCCESS);
assert(determinant_num == coeff_size);
rc = trexio_set_state(file, STATE_TEST);
assert(rc == TREXIO_SUCCESS);
rc = trexio_read_determinant_coefficient_size(file, &coeff_size);
assert(rc == TREXIO_SUCCESS);
assert(determinant_num == coeff_size);
// close current session
rc = trexio_close(file);
assert (rc == TREXIO_SUCCESS);
@ -142,16 +138,6 @@ static int test_has_determinant(const char* file_name, const back_end_t backend)
rc = trexio_has_determinant_coefficient(file);
assert(rc==TREXIO_SUCCESS);
// also check for STATE_TEST
rc = trexio_set_state(file, STATE_TEST);
assert(rc == TREXIO_SUCCESS);
rc = trexio_has_determinant_coefficient(file);
assert(rc==TREXIO_SUCCESS);
rc = trexio_set_state(file, 0);
assert(rc == TREXIO_SUCCESS);
// close current session
rc = trexio_close(file);
assert (rc == TREXIO_SUCCESS);
@ -189,13 +175,11 @@ static int test_read_determinant (const char* file_name, const back_end_t backen
// define arrays to read into
int64_t* det_list_read;
double* det_coef_read;
double* det_coef_s2_read;
double check_diff;
uint64_t size_r = 40L;
det_list_read = (int64_t*) calloc(2*int_num*size_r,sizeof(int64_t));
det_coef_read = (double*) calloc(size_r,sizeof(double));
det_coef_s2_read = (double*) calloc(size_r,sizeof(double));
// specify the read parameters, here:
// 1 chunk of 10 elements using offset of 40 (i.e. lines No. 40--59) into elements of the array starting from 5
@ -225,19 +209,10 @@ static int test_read_determinant (const char* file_name, const back_end_t backen
//printf("%lf %lf\n", check_diff, det_coef_read[offset_data_read]);
assert(check_diff*check_diff < 1e-14);
// read one chuk of coefficients for a different state
rc = trexio_set_state(file, STATE_TEST);
assert(rc == TREXIO_SUCCESS);
rc = trexio_read_determinant_coefficient(file, offset_file_read, &chunk_read, &det_coef_s2_read[offset_data_read]);
assert(rc == TREXIO_SUCCESS);
assert(chunk_read == read_size_check);
check_diff = det_coef_s2_read[0] - 0.;
assert(check_diff*check_diff < 1e-14);
rc = trexio_set_state(file, 0);
int32_t read_state_id = 666;
rc = trexio_read_state_id(file, &read_state_id);
assert(rc == TREXIO_SUCCESS);
assert(read_state_id == 0);
// now attempt to read so that one encounters end of file during reading (i.e. offset_file_read + chunk_read > size_max)
offset_file_read = 97L;
@ -284,7 +259,6 @@ static int test_read_determinant (const char* file_name, const back_end_t backen
assert(rc == TREXIO_SUCCESS);
assert(det_num == size_check);
// check conversion of determinants into orbital lists
int64_t size_list = TREXIO_NORB_PER_INT * int_num;
int32_t* orb_list_up = (int32_t*) calloc(size_list, sizeof(int32_t));
@ -323,7 +297,6 @@ static int test_read_determinant (const char* file_name, const back_end_t backen
// free the memory
free(det_list_read);
free(det_coef_read);
free(det_coef_s2_read);
free(det_list_check);
free(orb_list_up);
free(orb_list_dn);

50
tests/io_determinant_text.c
View File

@ -6,7 +6,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_determinant.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
#define SIZE 100
#define N_CHUNKS 5
#define STATE_TEST 2
@ -64,6 +64,12 @@ static int test_write_determinant (const char* file_name, const back_end_t backe
uint64_t offset_d = 0UL;
if (offset != 0L) offset_f += offset;
// write the state_id of a given file: 0 is ground state
if (trexio_has_state_id(file) == TREXIO_HAS_NOT) {
rc = trexio_write_state_id(file, 0);
assert(rc == TREXIO_SUCCESS);
}
// write n_chunks times using write_sparse
for(int i=0; i<N_CHUNKS; ++i){
@ -73,13 +79,10 @@ static int test_write_determinant (const char* file_name, const back_end_t backe
rc = trexio_write_determinant_coefficient(file, offset_f, chunk_size, &det_coef[offset_d]);
assert(rc == TREXIO_SUCCESS);
// The block below will write the coefficients for STATE_TEST
// The block below will check the set_state
rc = trexio_set_state(file, STATE_TEST);
assert(rc == TREXIO_SUCCESS);
rc = trexio_write_determinant_coefficient(file, offset_f, chunk_size, &det_coef[offset_d]);
assert(rc == TREXIO_SUCCESS);
// set state back to the default 0 (ground state)
rc = trexio_set_state(file, 0);
assert(rc == TREXIO_SUCCESS);
@ -100,13 +103,6 @@ static int test_write_determinant (const char* file_name, const back_end_t backe
assert(rc == TREXIO_SUCCESS);
assert(determinant_num == coeff_size);
rc = trexio_set_state(file, STATE_TEST);
assert(rc == TREXIO_SUCCESS);
rc = trexio_read_determinant_coefficient_size(file, &coeff_size);
assert(rc == TREXIO_SUCCESS);
assert(determinant_num == coeff_size);
// close current session
rc = trexio_close(file);
assert (rc == TREXIO_SUCCESS);
@ -142,16 +138,6 @@ static int test_has_determinant(const char* file_name, const back_end_t backend)
rc = trexio_has_determinant_coefficient(file);
assert(rc==TREXIO_SUCCESS);
// also check for STATE_TEST
rc = trexio_set_state(file, STATE_TEST);
assert(rc == TREXIO_SUCCESS);
rc = trexio_has_determinant_coefficient(file);
assert(rc==TREXIO_SUCCESS);
rc = trexio_set_state(file, 0);
assert(rc == TREXIO_SUCCESS);
// close current session
rc = trexio_close(file);
assert (rc == TREXIO_SUCCESS);
@ -189,13 +175,11 @@ static int test_read_determinant (const char* file_name, const back_end_t backen
// define arrays to read into
int64_t* det_list_read;
double* det_coef_read;
double* det_coef_s2_read;
double check_diff;
uint64_t size_r = 40L;
det_list_read = (int64_t*) calloc(2*int_num*size_r,sizeof(int64_t));
det_coef_read = (double*) calloc(size_r,sizeof(double));
det_coef_s2_read = (double*) calloc(size_r,sizeof(double));
// specify the read parameters, here:
// 1 chunk of 10 elements using offset of 40 (i.e. lines No. 40--59) into elements of the array starting from 5
@ -225,19 +209,10 @@ static int test_read_determinant (const char* file_name, const back_end_t backen
//printf("%lf %lf\n", check_diff, det_coef_read[offset_data_read]);
assert(check_diff*check_diff < 1e-14);
// read one chuk of coefficients for a different state
rc = trexio_set_state(file, STATE_TEST);
assert(rc == TREXIO_SUCCESS);
rc = trexio_read_determinant_coefficient(file, offset_file_read, &chunk_read, &det_coef_s2_read[offset_data_read]);
assert(rc == TREXIO_SUCCESS);
assert(chunk_read == read_size_check);
check_diff = det_coef_s2_read[0] - 0.;
assert(check_diff*check_diff < 1e-14);
rc = trexio_set_state(file, 0);
int32_t read_state_id = 666;
rc = trexio_read_state_id(file, &read_state_id);
assert(rc == TREXIO_SUCCESS);
assert(read_state_id == 0);
// now attempt to read so that one encounters end of file during reading (i.e. offset_file_read + chunk_read > size_max)
offset_file_read = 97L;
@ -284,7 +259,6 @@ static int test_read_determinant (const char* file_name, const back_end_t backen
assert(rc == TREXIO_SUCCESS);
assert(det_num == size_check);
// check conversion of determinants into orbital lists
int64_t size_list = TREXIO_NORB_PER_INT * int_num;
int32_t* orb_list_up = (int32_t*) calloc(size_list, sizeof(int32_t));
@ -323,7 +297,6 @@ static int test_read_determinant (const char* file_name, const back_end_t backen
// free the memory
free(det_list_read);
free(det_coef_read);
free(det_coef_s2_read);
free(det_list_check);
free(orb_list_up);
free(orb_list_dn);
@ -339,6 +312,7 @@ int main(){
/*============== Test launcher ================*/
int rc;
rc = system(RM_COMMAND);
assert (rc == 0);

6
tests/io_dset_float_text.c
View File

@ -5,7 +5,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_dset_f.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_write_dset (const char* file_name, const back_end_t backend) {
@ -40,7 +40,7 @@ static int test_write_dset (const char* file_name, const back_end_t backend) {
// write numerical attribute in an empty file
rc = trexio_write_nucleus_num(file, num);
assert (rc == TREXIO_SUCCESS);
// write numerical dataset in a file
rc = trexio_write_nucleus_coord(file, coord);
assert (rc == TREXIO_SUCCESS);
@ -144,5 +144,3 @@ int main(void) {
return 0;
}

2
tests/io_dset_int_text.c
View File

@ -5,7 +5,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_dset_i.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_write_dset (const char* file_name, const back_end_t backend) {

2
tests/io_dset_sparse_text.c
View File

@ -6,7 +6,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_dset_sparse.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
#define SIZE 100
#define N_CHUNKS 5

6
tests/io_dset_str_text.c
View File

@ -6,7 +6,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_dset_s.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_write_dset_str (const char* file_name, const back_end_t backend) {
@ -39,7 +39,7 @@ static int test_write_dset_str (const char* file_name, const back_end_t backend)
// write numerical attribute in an empty file
rc = trexio_write_nucleus_num(file, num);
assert (rc == TREXIO_SUCCESS);
// write dataset of string in the file (including FAKE statements)
int max_str_len = 16;
rc = trexio_write_nucleus_label(file, labels, max_str_len);
@ -153,5 +153,3 @@ int main(void) {
return 0;
}

2
tests/io_num_text.c
View File

@ -5,7 +5,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_num.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_write_num (const char* file_name, const back_end_t backend) {

6
tests/io_safe_dset_float_text.c
View File

@ -6,7 +6,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_safe_dset_f.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_write_dset (const char* file_name, const back_end_t backend) {
@ -41,7 +41,7 @@ static int test_write_dset (const char* file_name, const back_end_t backend) {
// write numerical attribute in an empty file
rc = trexio_write_nucleus_num(file, num);
assert (rc == TREXIO_SUCCESS);
/* write numerical dataset with an unsafe dimension
* this should return TREXIO_UNSAFE_ARRAY_DIM indicating
* that access beyong allocated memory is likely to occur */
@ -161,5 +161,3 @@ int main(void) {
return 0;
}

4
tests/io_str_text.c
View File

@ -6,7 +6,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_str.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_write_str (const char* file_name, const back_end_t backend) {
@ -129,5 +129,3 @@ int main(void) {
return 0;
}

2
tests/open_text.c
View File

@ -6,7 +6,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_open.dir"
#define TREXIO_VOID "non_existing_" TREXIO_FILE
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_open_w (const char* file_name, const back_end_t backend) {

2
tests/overwrite_all_text.c
View File

@ -6,7 +6,7 @@
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_over.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_write (const char* file_name, const back_end_t backend) {

4
tests/pre_close.c
View File

@ -5,8 +5,8 @@
#include <stdint.h>
#define TEST_BACKEND TREXIO_TEXT
#define TREXIO_FILE "test_dset_sparse.dir"
#define RM_COMMAND "rm -rf " TREXIO_FILE
#define TREXIO_FILE "test_pre_close.dir"
#define RM_COMMAND "rm -f -- " TREXIO_FILE "/*.txt " TREXIO_FILE "/*.txt.size " TREXIO_FILE "/.lock && rm -fd -- " TREXIO_FILE
static int test_pre_close_1 (const char* file_name, const back_end_t backend)
{

4
tests/test_f.f90
View File

@ -14,12 +14,12 @@ program test_trexio
rc = trexio_info()
call system('rm -rf -- test_write_f.dir')
call system('rm -f -- test_write_f.dir/*.txt test_write_f.dir/*.txt.size test_write_f.dir/.lock && rm -fd -- test_write_f.dir')
print *, 'call test_write(''test_write_f.dir'', TREXIO_TEXT)'
call test_write('test_write_f.dir', TREXIO_TEXT)
print *, 'call test_read(''test_write_f.dir'', TREXIO_TEXT)'
call test_read('test_write_f.dir', TREXIO_TEXT)
call system('rm -rf -- test_write_f.dir')
call system('rm -f -- test_write_f.dir/*.txt test_write_f.dir/*.txt.size test_write_f.dir/.lock && rm -fd -- test_write_f.dir')
call test_read_void('test_write_f.dir', TREXIO_TEXT)

8
tools/generator.py
View File

@ -12,12 +12,12 @@ detailed_nums = get_detailed_num_dict(trex_config)
detailed_strs = get_detailed_str_dict(trex_config)
# helper dictionaries that contain names of groups, nums or dsets as keys
dsets = get_dset_dict(trex_config)
detailed_dsets_nostr, detailed_dsets_str, detailed_dsets_sparse = split_dset_dict_detailed(dsets)
detailed_dsets_nostr, detailed_dsets_str, detailed_dsets_sparse, detailed_dsets_buf = split_dset_dict_detailed(dsets)
detailed_dsets = detailed_dsets_nostr.copy()
detailed_dsets.update(detailed_dsets_str)
# build a big dictionary with all pre-processed data
detailed_all = {
'datasets' : dict(detailed_dsets_nostr, **detailed_dsets_str, **detailed_dsets_sparse),
'datasets' : dict(detailed_dsets_nostr, **detailed_dsets_str, **detailed_dsets_sparse, **detailed_dsets_buf),
'groups' : group_dict,
'numbers' : detailed_nums,
'strings' : detailed_strs
@ -62,6 +62,10 @@ for fname in files_todo['dset_str']:
for fname in files_todo['dset_sparse']:
recursive_populate_file(fname, template_paths, detailed_dsets_sparse)
# populate has/read/write_buffered functions with recursive scheme
for fname in files_todo['buffered']:
recursive_populate_file(fname, template_paths, detailed_dsets_buf)
# populate group-related functions with mixed scheme
for fname in files_todo['group']:
# recursive scheme for delete_group functions

51
tools/generator_tools.py
View File

@ -41,7 +41,7 @@ def get_files_todo(source_files: dict) -> dict:
f for f in all_files
if 'read' in f or 'write' in f or 'has' in f or 'flush' in f or 'free' in f or 'hrw' in f or 'delete' in f
]
for key in ['dset_data', 'dset_str', 'dset_sparse', 'attr_num', 'attr_str', 'group']:
for key in ['dset_data', 'dset_str', 'dset_sparse', 'attr_num', 'attr_str', 'group', 'buffered']:
files_todo[key] = list(filter(lambda x: key in x, files_todo['all']))
files_todo['group'].append('struct_text_group_dset.h')
@ -108,11 +108,10 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N
triggers = ['group_dset_dtype', 'group_dset_py_dtype', 'group_dset_h5_dtype', 'default_prec', 'is_index',
'group_dset_f_dtype_default', 'group_dset_f_dtype_double', 'group_dset_f_dtype_single',
'group_dset_dtype_default', 'group_dset_dtype_double', 'group_dset_dtype_single',
'group_dset_rank', 'group_dset_dim_list', 'group_dset_f_dims',
'group_dset_rank', 'group_dset_unique_rank', 'group_dset_dim_list', 'group_dset_f_dims',
'group_num_f_dtype_default', 'group_num_f_dtype_double', 'group_num_f_dtype_single',
'group_num_dtype_default', 'group_num_dtype_double', 'group_num_dtype_single',
'group_num_h5_dtype', 'group_num_py_dtype',
'group_dset_format_scanf', 'group_dset_format_printf', 'group_dset_sparse_dim',
'group_num_h5_dtype', 'group_num_py_dtype', 'group_dset_format_scanf', 'group_dset_format_printf',
'group_dset_sparse_indices_printf', 'group_dset_sparse_indices_scanf',
'sparse_format_printf_8', 'sparse_format_printf_16', 'sparse_format_printf_32',
'sparse_line_length_8', 'sparse_line_length_16', 'sparse_line_length_32',
@ -151,6 +150,11 @@ def recursive_populate_file(fname: str, paths: dict, detailed_source: dict) -> N
if 'dim' in detailed_source[item]['trex_json_int_type']:
templine = line.replace('//', '')
f_out.write(templine)
# special case to get the max dimension of sparse datasets with different dimensions
elif 'trexio_read_$group_dset_unique_dim$_64' in line:
for i in range(int(detailed_source[item]['group_dset_unique_rank'])):
templine = line.replace('$group_dset_unique_dim$', detailed_source[item]['unique_dims'][i]).replace('$dim_id$', str(i))
f_out.write(templine)
# general case of recursive replacement of inline triggers
else:
populated_line = recursive_replace_line(line, triggers, detailed_source[item])
@ -495,6 +499,21 @@ def get_dtype_dict (dtype: str, target: str, rank = None, int_len_printf = None)
f'group_{target}_format_scanf' : 'lf',
f'group_{target}_py_dtype' : 'float'
})
elif 'buffered' in dtype:
dtype_dict.update({
'default_prec' : '64',
f'group_{target}_dtype' : 'double',
f'group_{target}_h5_dtype' : 'native_double',
f'group_{target}_f_dtype_default' : 'real(c_double)',
f'group_{target}_f_dtype_double' : 'real(c_double)',
f'group_{target}_f_dtype_single' : 'real(c_float)',
f'group_{target}_dtype_default' : 'double',
f'group_{target}_dtype_double' : 'double',
f'group_{target}_dtype_single' : 'float',
f'group_{target}_format_printf' : '24.16e',
f'group_{target}_format_scanf' : 'lf',
f'group_{target}_py_dtype' : 'float'
})
elif dtype in ['int', 'dim', 'dim readonly', 'index']:
dtype_dict.update({
'default_prec' : '32',
@ -657,11 +676,12 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
configuration (dict) : configuration from `trex.json`
Returns:
dset_numeric_dict, dset_string_dict (tuple) : dictionaries corresponding to all numeric- and string-based datasets, respectively.
(tuple) : dictionaries corresponding to all types of datasets in trexio.
"""
dset_numeric_dict = {}
dset_string_dict = {}
dset_sparse_dict = {}
dset_string_dict = {}
dset_sparse_dict = {}
dset_buffer_dict = {}
for k,v in datasets.items():
# create a temp dictionary
@ -698,11 +718,18 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
else:
tmp_dict['is_index'] = 'false'
# add the list of dimensions
tmp_dict['dims'] = [dim.replace('.','_') for dim in v[1]]
# get a list of unique dimensions for sparse datasets
if is_sparse:
tmp_dict['unique_dims'] = list(set(tmp_dict['dims']))
tmp_dict['group_dset_unique_rank'] = str(len(tmp_dict['unique_dims']))
# add the rank
tmp_dict['rank'] = rank
tmp_dict['group_dset_rank'] = str(rank)
# add the list of dimensions
tmp_dict['dims'] = [dim.replace('.','_') for dim in v[1]]
# build a list of dimensions to be inserted in the dims array initialization, e.g. {ao_num, ao_num}
dim_list = tmp_dict['dims'][0]
if rank > 1:
@ -719,8 +746,6 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
tmp_dict['group_dset_f_dims'] = dim_f_list
if is_sparse:
# store the max possible dim of the sparse dset (e.g. mo_num)
tmp_dict['group_dset_sparse_dim'] = tmp_dict['dims'][0]
# build printf/scanf sequence and compute line length for n-index sparse quantity
index_printf = f'*(index_sparse + {str(rank)}*i'
index_scanf = f'index_sparse + {str(rank)}*i'
@ -755,12 +780,14 @@ def split_dset_dict_detailed (datasets: dict) -> tuple:
# split datasets in numeric- and string- based
if 'str' in datatype:
dset_string_dict[k] = tmp_dict
elif 'buffered' in datatype:
dset_buffer_dict[k] = tmp_dict
elif is_sparse:
dset_sparse_dict[k] = tmp_dict
else:
dset_numeric_dict[k] = tmp_dict
return (dset_numeric_dict, dset_string_dict, dset_sparse_dict)
return (dset_numeric_dict, dset_string_dict, dset_sparse_dict, dset_buffer_dict)
def check_dim_consistency(num: dict, dset: dict) -> None:

74
trex.org
View File

@ -35,10 +35,16 @@ For sparse data structures such as electron replusion integrals,
the data can be too large to fit in memory and the data needs to be
fetched using multiple function calls to perform I/O on buffers.
For more information on how to read/write sparse data structures, see
the [[./examples.html][examples]].
the [[./examples.html][examples]]. The ~sparse~ data representation implies the
[[https://en.wikipedia.org/wiki/Sparse_matrix#Coordinate_list_(COO)][coordinate list]] representation, namely the user has to write a list
of indices and values.
For determinants, the ~special~ attribute is present in the type. This
means that the source code is not produced by the generator, but hand-written.
For the Configuration Interfaction (CI) and Configuration State Function (CSF)
groups, the ~buffered~ data type is introduced, which allows similar incremental
I/O as for ~sparse~ data but without the need to write indices of the sparse values.
For determinant lists (integer bit fields), the ~special~ attribute is present in the type.
This means that the source code is not produced by the generator, but hand-written.
#+begin_src python :tangle trex.json :exports none
{
@ -696,11 +702,11 @@ prim_factor =
An illustration on how to read determinants is presented in the [[./examples.html][examples]].
#+NAME: determinant
| Variable | Type | Dimensions | Description |
|---------------+-----------------+-------------------------------+--------------------------------------------------------|
| ~num~ | ~dim readonly~ | | Number of determinants |
| ~list~ | ~int special~ | ~(determinant.num)~ | List of determinants as integer bit fields |
| ~coefficient~ | ~float special~ | ~(state.num,determinant.num)~ | Coefficients of the determinants from the CI expansion |
| Variable | Type | Dimensions | Description |
|---------------+------------------+---------------------+--------------------------------------------------------|
| ~num~ | ~dim readonly~ | | Number of determinants |
| ~list~ | ~int special~ | ~(determinant.num)~ | List of determinants as integer bit fields |
| ~coefficient~ | ~float buffered~ | ~(determinant.num)~ | Coefficients of the determinants from the CI expansion |
#+CALL: json(data=determinant, title="determinant")
@ -708,9 +714,9 @@ prim_factor =
:results:
#+begin_src python :tangle trex.json
"determinant": {
"num" : [ "dim readonly" , [] ]
, "list" : [ "int special" , [ "determinant.num" ] ]
, "coefficient" : [ "float special", [ "determinant.num", "state.num" ] ]
"num" : [ "dim readonly" , [] ]
, "list" : [ "int special" , [ "determinant.num" ] ]
, "coefficient" : [ "float buffered", [ "determinant.num" ] ]
} ,
#+end_src
:end:
@ -732,11 +738,11 @@ prim_factor =
the basis of Slater determinants.
#+NAME: csf
| Variable | Type | Dimensions | Description |
|-------------------+-----------------+-----------------------------+------------------------------------------------|
| ~num~ | ~dim readonly~ | | Number of CSFs |
| ~coefficient~ | ~float special~ | ~(state.num,csf.num)~ | Coefficients of the CSFs from the CI expansion |
| ~det_coefficient~ | ~float sparse~ | ~(determinant.num,csf.num)~ | Projection on the determinant basis |
| Variable | Type | Dimensions | Description |
|-------------------+------------------+-----------------------------+------------------------------------------------|
| ~num~ | ~dim readonly~ | | Number of CSFs |
| ~coefficient~ | ~float buffered~ | ~(csf.num)~ | Coefficients of the CSFs from the CI expansion |
| ~det_coefficient~ | ~float sparse~ | ~(determinant.num,csf.num)~ | Projection on the determinant basis |
#+CALL: json(data=csf, title="csf")
@ -744,25 +750,32 @@ prim_factor =
:results:
#+begin_src python :tangle trex.json
"csf": {
"num" : [ "dim readonly" , [] ]
, "coefficient" : [ "float special", [ "csf.num", "state.num" ] ]
, "det_coefficient" : [ "float sparse" , [ "csf.num", "determinant.num" ] ]
"num" : [ "dim readonly" , [] ]
, "coefficient" : [ "float buffered", [ "csf.num" ] ]
, "det_coefficient" : [ "float sparse" , [ "csf.num", "determinant.num" ] ]
} ,
#+end_src
:end:
* Excited states (state group)
By default, the ~determinant~ group corresponds to the ground state.
However, it should be also possible to store the coefficients that
correspond to excited state wave functions for the same set of
determinants. This is the goal of the present group
This group contains information about excited state. Since TREXIO version 2.3.0
the state-specific data (e.g. CI/CSF coeffcients, RDMs) is written in a separate
file in order to avoid over-complicated internal logics and global state switches.
The ~file_name~ and ~label~ arrays have to be written only for the master file,
e.g. the one containing the ground state wave function.
The ~id~ and ~current_label~ attributes have to be specified for each file
(containing both ground and excited state data).
#+NAME: state
| Variable | Type | Dimensions | Description |
|----------+-------+---------------+------------------------------------------------|
| ~num~ | ~dim~ | | Number of states (including the ground state) |
| ~label~ | ~str~ | ~(state.num)~ | Label of a given state (e.g. 'S' for singlets) |
| Variable | Type | Dimensions | Description |
|-----------------+-------+---------------+-----------------------------------------------------------------------------------------------|
| ~num~ | ~dim~ | | Number of states (including the ground state) |
| ~id~ | ~int~ | | Index of a current state (0 is ground state) |
| ~label~ | ~str~ | ~(state.num)~ | Labels of all states related to this file (e.g. 'S' for singlets) |
| ~current_label~ | ~str~ | | Labels of the current state that is in a file |
| ~file_name~ | ~str~ | ~(state.num)~ | Names of the TREXIO files linked to the current one (i.e. containing data for excited states) |
#+CALL: json(data=state, title="state")
@ -770,8 +783,11 @@ prim_factor =
:results:
#+begin_src python :tangle trex.json
"state": {
"num" : [ "dim", [] ]
, "label" : [ "str", [ "state.num" ] ]
"num" : [ "dim", [] ]
, "id" : [ "int", [] ]
, "label" : [ "str", [ "state.num" ] ]
, "current_label" : [ "str", [] ]
, "file_name" : [ "str", [ "state.num" ] ]
} ,
#+end_src
:end: