-
-two_e is a sparse data structure, which can be too large to fit
-in memory. So we provide functions to read and write it by
-chunks.
-In the text back end, the easiest way to do it is to create a
-file for each sparse float structure.
-
-
-
-
trexio_exit_code
-trexio_text_buffered_read_rdm_two_e(trexio_t* const file,
- const uint64_t offset,
- const uint64_t size,
- int64_t* const index,
- double* const value);
-
-trexio_exit_code
-trexio_text_buffered_write_rdm_two_e(trexio_t* const file,
- const uint64_t offset,
- const uint64_t size,
- const int64_t* index,
- const double* value);
-
-
-
-
-
trexio_exit_code
-trexio_text_buffered_read_rdm_two_e(trexio_t* const file,
- const uint64_t offset,
- const uint64_t size,
- int64_t* const index,
- double* const value)
-{
- if (file == NULL) return TREXIO_INVALID_ARG_1;
- if (index == NULL) return TREXIO_INVALID_ARG_4;
- if (value == NULL) return TREXIO_INVALID_ARG_5;
-
- rdm_t* const rdm = trexio_text_read_rdm((trexio_text_t*) file);
- if (rdm == NULL) return TREXIO_FAILURE;
-
- FILE* f = fopen(rdm->two_e_file_name, "r");
- if (f == NULL) return TREXIO_END;
-
- const uint64_t line_length = 64L;
- fseek(f, (long) offset * line_length, SEEK_SET);
-
- for (uint64_t i=0 ; i<size ; ++i) {
- int rc = fscanf(f, "%9" SCNd64 " %9" SCNd64 " %9" SCNd64 " %9" SCNd64 " %24le\n",
- &index[4*i],
- &index[4*i+1],
- &index[4*i+2],
- &index[4*i+3],
- &value[i]);
- if (rc == 5) {
- /* Do nothing */
- } else if (rc == EOF) {
- return TREXIO_END;
+ return TREXIO_FAILURE;
}
}
+ /* Close the TXT file */
+ rc = fclose(f);
+ if (rc != 0) return TREXIO_FILE_ERROR;
+
+ /* Append .size to the file_full_path in order to write additional info about the written buffer of data */
+ strncat(file_full_path, ".size", 6);
+
+ /* Open the new file in "a" (append) mode to append info about the buffer that has been just written */
+ FILE *f_wSize = fopen(file_full_path, "a");
+ if (f_wSize == NULL) return TREXIO_FILE_ERROR;
+
+ /* Write the buffer_size */
+ rc = fprintf(f_wSize, "%" PRId64 " %" PRId64 "\n", size, io_start_pos);
+ if (rc <= 0) {
+ fclose(f_wSize);
+ return TREXIO_FAILURE;
+ }
+
+ /* Close the TXT file */
+ rc = fclose(f_wSize);
+ if (rc != 0) return TREXIO_FILE_ERROR;
+
+ /* Exit upon success */
return TREXIO_SUCCESS;
}
+
+
-
trexio_exit_code
-
trexio_text_buffered_write_rdm_two_e(
trexio_t*
const file,
-
const uint64_t offset,
-
const uint64_t size,
-
const int64_t*
index,
-
const double*
value)
+
+
trexio_exit_code trexio_text_read_$group_dset$(trexio_t* const file,
+ const int64_t offset_file,
+ const int64_t size,
+ const int64_t size_max,
+ int64_t* const eof_read_size,
+ int32_t* const index_sparse,
+ double* const value_sparse)
{
- if (file == NULL) return TREXIO_INVALID_ARG_1;
- if (index == NULL) return TREXIO_INVALID_ARG_4;
- if (value == NULL) return TREXIO_INVALID_ARG_5;
- if (file->mode != 'r') return TREXIO_READONLY;
+ if (file == NULL) return TREXIO_INVALID_ARG_1;
+ if (eof_read_size == NULL) return TREXIO_INVALID_ARG_5;
- rdm_t* const rdm = trexio_text_read_rdm((trexio_text_t*) file);
- if (rdm == NULL) return TREXIO_FAILURE;
+ /* Build the name of the file with sparse data.
+ The $group_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed?
+ */
+ const char $group_dset$_file_name[256] = "/$group_dset$.txt";
+ /* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
+ char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
- FILE* f = fopen(rdm->two_e_file_name, "w");
- if (f == NULL) return TREXIO_FAILURE;
+ /* Copy directory name in file_full_path */
+ strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
+ /* Append name of the file with sparse data */
+ strncat (file_full_path, $group_dset$_file_name,
+ TREXIO_MAX_FILENAME_LENGTH-strlen($group_dset$_file_name));
- const uint64_t line_length = 64L;
- fseek(f, (long) offset * line_length, SEEK_SET);
+ /* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */
+ FILE* f = fopen(file_full_path, "r");
+ if(f == NULL) return TREXIO_FILE_ERROR;
- for (uint64_t i=0 ; i<size ; ++i) {
- int rc = fprintf(f, "%9" PRId64 " %9" PRId64 " %9" PRId64 " %9" PRId64 " %24le\n",
- index[4*i],
- index[4*i+1],
- index[4*i+2],
- index[4*i+3],
- value[i]);
- if (rc != 5) return TREXIO_FAILURE;
+ /* Specify the line length in order to offset properly. For example, for 4-index quantities
+ the line_length is 69 because 10 per index + 4 spaces + 24 for floating point value + 1 for the new line char
+ */
+ uint64_t line_length = 0UL;
+ /* Determine the line length depending on the size_max (usually mo_num or ao_num) */
+ if (size_max < UINT8_MAX) {
+ line_length = $sparse_line_length_8$; // 41 for 4 indices
+ } else if (size_max < UINT16_MAX) {
+ line_length = $sparse_line_length_16$; // 49 for 4 indices
+ } else {
+ line_length = $sparse_line_length_32$; //69 for 4 indices
}
+ /* Offset in the file according to the provided value of offset_file and optimal line_length */
+ fseek(f, (long) offset_file * line_length, SEEK_SET);
+
+ /* Read the data from the file and check the return code of fprintf to verify that > 0 bytes have been read or reached EOF */
+ int rc;
+ char buffer[1024];
+ uint64_t count = 0UL;
+ for (uint64_t i=0UL; i<size; ++i) {
+
+ memset(buffer,0,sizeof(buffer));
+
+ if(fgets(buffer, 1023, f) == NULL){
+
+ fclose(f);
+ *eof_read_size = count;
+ return TREXIO_END;
+
+ } else {
+
+ rc = sscanf(buffer, "$group_dset_format_scanf$",
+ $group_dset_sparse_indices_scanf$,
+ value_sparse + i);
+ if(rc <= 0) {
+ fclose(f);
+ return TREXIO_FAILURE;
+ }
+ count += 1UL;
+
+ }
+ }
+
+ /* Close the TXT file */
+ rc = fclose(f);
+ if(rc != 0) return TREXIO_FILE_ERROR;
+
return TREXIO_SUCCESS;
}
+
+
+
+
trexio_exit_code trexio_text_read_$group_dset$_size(trexio_t* const file, int64_t* const size_max)
+{
+ if (file == NULL) return TREXIO_INVALID_ARG_1;
+
+ /* Build the name of the file with sparse data.
+ The $group_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed?
+ */
+ const char $group_dset$_file_name[256] = "/$group_dset$.txt.size";
+ /* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
+ char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
+
+ /* Copy directory name in file_full_path */
+ strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
+ /* Append name of the file with sparse data */
+ strncat (file_full_path, $group_dset$_file_name,
+ TREXIO_MAX_FILENAME_LENGTH-strlen($group_dset$_file_name));
+
+ /* Open the file in "r" (read) mode to guarantee that no truncation happens upon consecutive reads */
+ FILE* f = fopen(file_full_path, "r");
+ if(f == NULL) return TREXIO_FILE_ERROR;
+
+
+ /* Read the data from the file and check the return code of fprintf to verify that > 0 bytes have been read or reached EOF */
+ int rc;
+ int64_t size_item, offset_item, size_accum=0L;
+
+ /* Read the values from the file. BEWARE OF POSSIBLE MAX_INT64 OVERFLOW ! */
+ while(fscanf(f, "%" SCNd64 " %" SCNd64 "", &size_item, &offset_item) != EOF) {
+ /* Check that summation will not overflow the int64_t value */
+ if (INT64_MAX - size_accum > size_item) {
+ size_accum += size_item;
+ } else {
+ fclose(f);
+ *size_max = -1L;
+ return TREXIO_INT_SIZE_OVERFLOW;
+ }
+ }
+
+ /* Close the TXT file */
+ rc = fclose(f);
+ if(rc != 0) return TREXIO_FILE_ERROR;
+
+ /* Overwrite the value at the input address and return TREXIO_SUCCESS */
+ *size_max = size_accum;
+ return TREXIO_SUCCESS;
+
+}
+
+
+
+
+
trexio_exit_code trexio_text_has_$group_dset$(trexio_t* const file)
+{
+ if (file == NULL) return TREXIO_INVALID_ARG_1;
+
+ /* Build the name of the file with sparse data.
+ The $group_dset$.txt is limited to 256 symbols for the moment. What are the chances that it will exceed?
+ */
+ const char $group_dset$_file_name[256] = "/$group_dset$.txt";
+ /* The full path to the destination TXT file with sparse data. This will include TREXIO directory name. */
+ char file_full_path[TREXIO_MAX_FILENAME_LENGTH];
+
+ /* Copy directory name in file_full_path */
+ strncpy (file_full_path, file->file_name, TREXIO_MAX_FILENAME_LENGTH);
+ /* Append name of the file with sparse data */
+ strncat (file_full_path, $group_dset$_file_name,
+ TREXIO_MAX_FILENAME_LENGTH-strlen($group_dset$_file_name));
+
+ /* Check the return code of access function to determine whether the file with sparse data exists or not */
+ if (access(file_full_path, F_OK) == 0){
+ return TREXIO_SUCCESS;
+ } else {
+ return TREXIO_HAS_NOT;
+ }
+}
+