TREX/trexio
1
0
Fork 0
mirror of https://github.com/TREX-CoE/trexio.git synced 2025-01-10 13:08:27 +01:00
Code Issues Projects Releases Wiki Activity

[WIP] refactor read_group set of functions to read data in arbitrary order

Browse Source 
This fix is required to fix backwards compatibility issue of TEXT back end. In the meantime, the custom garbage collector from generator_tools which was deallocating previously allocated memory (datasets and strings) is no longer valid here since the order of allocation can be atrbitrary. This requires a new custom garbage collector, maybe as a separate functions in TEXT back end
This commit is contained in:
q-posev 2022-02-07 15:57:38 +01:00
parent 999dd2065a
commit 5eb697abcc
1 changed files with 180 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
src/templates_text/templator_text.org
View File

@ -318,16 +318,19 @@ trexio_text_read_$group$ (trexio_text_t* const file)
return NULL; return NULL;
} }
/* Read the dimensioning variables */
int rc = 0; int rc = 0;
/* workaround for the case of missing blocks in the file */
// START REPEAT GROUP_DSET_ALL // START REPEAT GROUP_DSET_ALL
rc = fscanf(f, "%1023s", buffer); uint64_t size_$group_dset$ = 0;
if ((rc != 1) || (strcmp(buffer, "rank_$group_dset$") != 0)) { // END REPEAT GROUP_DSET_ALL
FREE(buffer);
fclose(f); while(fscanf(f, "%1023s", buffer) != EOF) {
FREE($group$);
return NULL; if (strcmp(buffer, "EXIT") == 0) {
} break;
// START REPEAT GROUP_DSET_ALL
} else if (strcmp(buffer, "rank_$group_dset$") == 0) {
rc = fscanf(f, "%u", &($group$->rank_$group_dset$)); rc = fscanf(f, "%u", &($group$->rank_$group_dset$));
if (rc != 1) { if (rc != 1) {
@ -337,14 +340,11 @@ trexio_text_read_$group$ (trexio_text_t* const file)
return NULL; return NULL;
} }
/* workaround for the case of missing blocks in the file */ if ($group$->rank_$group_dset$ != 0) size_$group_dset$ = 1UL;
uint64_t size_$group_dset$ = 0;
if ($group$->rank_$group_dset$ != 0) size_$group_dset$ = 1;
for (uint32_t i=0; i<$group$->rank_$group_dset$; ++i){ for (uint32_t i=0; i<$group$->rank_$group_dset$; ++i){
uint32_t j=0; uint32_t j=0;
rc = fscanf(f, "%1023s %u", buffer, &j); rc = fscanf(f, "%1023s %u", buffer, &j);
if ((rc != 2) || (strcmp(buffer, "dims_$group_dset$") != 0) || (j!=i)) { if ((rc != 2) || (strcmp(buffer, "dims_$group_dset$") != 0) || (j!=i)) {
FREE(buffer); FREE(buffer);
@ -365,19 +365,72 @@ trexio_text_read_$group$ (trexio_text_t* const file)
size_$group_dset$ *= $group$->dims_$group_dset$[i]; size_$group_dset$ *= $group$->dims_$group_dset$[i];
} }
// END REPEAT GROUP_DSET_ALL // END REPEAT GROUP_DSET_ALL
// START REPEAT GROUP_DSET_NUM
} else if (strcmp(buffer, "$group_dset$") == 0) {
// START REPEAT GROUP_NUM /* Allocate arrays */
/* Read data */ $group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$);
unsigned int $group_num$_isSet; if ($group$->$group_dset$ == NULL) {
rc = fscanf(f, "%1023s", buffer);
assert(!((rc != 1) || (strcmp(buffer, "$group_num$_isSet") != 0)));
if ((rc != 1) || (strcmp(buffer, "$group_num$_isSet") != 0)) {
FREE(buffer); FREE(buffer);
fclose(f); fclose(f);
FREE($group$); FREE($group$);
return NULL; return NULL;
} }
for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
rc = fscanf(f, "%$group_dset_format_scanf$", &($group$->$group_dset$[i]));
if (rc != 1) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
}
// END REPEAT GROUP_DSET_NUM
// START REPEAT GROUP_DSET_STR
} else if (strcmp(buffer, "$group_dset$") == 0) {
if(size_$group_dset$ != 0) {
/* Allocate arrays */
$group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$);
if ($group$->$group_dset$ == NULL) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
/* WARNING: this tmp array allows to avoid allocation of space for each element of array of string
BUT it's size has to be number_of_str*max_len_str where max_len_str is somewhat arbitrary, e.g. 32.
,*/
char* tmp_$group_dset$;
tmp_$group_dset$ = CALLOC(size_$group_dset$*32, char);
for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
$group$->$group_dset$[i] = tmp_$group_dset$;
/* conventional fcanf with "%s" only return the string before the first space character
,* to read string with spaces use "%[^\n]" possible with space before or after, i.e. " %[^\n]"
,* Q: depending on what ? */
rc = fscanf(f, " %1023[^\n]", tmp_$group_dset$);
assert(!(rc != 1));
if (rc != 1) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
size_t tmp_$group_dset$_len = strlen($group$->$group_dset$[i]);
tmp_$group_dset$ += tmp_$group_dset$_len + 1;
}
}
// END REPEAT GROUP_DSET_STR
// START REPEAT GROUP_NUM
} else if (strcmp(buffer, "$group_num$_isSet") == 0) {
unsigned int $group_num$_isSet;
/* additional parameter $group_num$_isSet is needed to suppress warning when fscanf into bool variable using %u or %d */ /* additional parameter $group_num$_isSet is needed to suppress warning when fscanf into bool variable using %u or %d */
rc = fscanf(f, "%u", &($group_num$_isSet)); rc = fscanf(f, "%u", &($group_num$_isSet));
$group$->$group_num$_isSet = (bool) $group_num$_isSet; $group$->$group_num$_isSet = (bool) $group_num$_isSet;
@ -408,17 +461,10 @@ trexio_text_read_$group$ (trexio_text_t* const file)
return NULL; return NULL;
} }
} }
// END REPEAT GROUP_NUM
// END REPEAT GROUP_NUM
// START REPEAT GROUP_ATTR_STR // START REPEAT GROUP_ATTR_STR
rc = fscanf(f, "%1023s", buffer); } else if (strcmp(buffer, "len_$group_str$") == 0) {
assert(!((rc != 1) || (strcmp(buffer, "len_$group_str$") != 0)));
if ((rc != 1) || (strcmp(buffer, "len_$group_str$") != 0)) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
rc = fscanf(f, "%" SCNu64 "", &($group$->len_$group_str$)); rc = fscanf(f, "%" SCNu64 "", &($group$->len_$group_str$));
assert(!(rc != 1)); assert(!(rc != 1));
@ -462,91 +508,14 @@ trexio_text_read_$group$ (trexio_text_t* const file)
} }
} }
// END REPEAT GROUP_ATTR_STR // END REPEAT GROUP_ATTR_STR
} else {
// START REPEAT GROUP_DSET_NUM continue;
/* Allocate arrays */
$group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$);
assert (!($group$->$group_dset$ == NULL));
if ($group$->$group_dset$ == NULL) {
FREE(buffer);
fclose(f);
FREE($group$->$group_dset$);
FREE($group$);
return NULL;
} }
rc = fscanf(f, "%1023s", buffer);
assert(!((rc != 1) || (strcmp(buffer, "$group_dset$") != 0)));
if ((rc != 1) || (strcmp(buffer, "$group_dset$") != 0)) {
FREE(buffer);
fclose(f);
FREE($group$->$group_dset$);
FREE($group$);
return NULL;
} }
for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
rc = fscanf(f, "%$group_dset_format_scanf$", &($group$->$group_dset$[i]));
assert(!(rc != 1));
if (rc != 1) {
FREE(buffer);
fclose(f);
FREE($group$->$group_dset$);
FREE($group$);
return NULL;
}
}
// END REPEAT GROUP_DSET_NUM
// START REPEAT GROUP_DSET_STR
rc = fscanf(f, "%1023s", buffer);
assert(!((rc != 1) || (strcmp(buffer, "$group_dset$") != 0)));
if ((rc != 1) || (strcmp(buffer, "$group_dset$") != 0)) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
/* Allocate arrays */
if(size_$group_dset$ != 0) {
$group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$);
if ($group$->$group_dset$ == NULL) {
FREE(buffer);
fclose(f);
FREE($group$->$group_dset$);
FREE($group$);
return NULL;
}
/* WARNING: this tmp array allows to avoid allocation of space for each element of array of string
BUT it's size has to be number_of_str*max_len_str where max_len_str is somewhat arbitrary, e.g. 32.
,*/
char* tmp_$group_dset$;
tmp_$group_dset$ = CALLOC(size_$group_dset$*32, char);
for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
$group$->$group_dset$[i] = tmp_$group_dset$;
/* conventional fcanf with "%s" only return the string before the first space character
,* to read string with spaces use "%[^\n]" possible with space before or after, i.e. " %[^\n]"
,* Q: depending on what ? */
rc = fscanf(f, " %1023[^\n]", tmp_$group_dset$);
assert(!(rc != 1));
if (rc != 1) {
FREE(buffer);
fclose(f);
FREE($group$->$group_dset$);
FREE($group$);
return NULL;
}
size_t tmp_$group_dset$_len = strlen($group$->$group_dset$[i]);
tmp_$group_dset$ += tmp_$group_dset$_len + 1;
}
}
// END REPEAT GROUP_DSET_STR
FREE(buffer); FREE(buffer);
fclose(f); fclose(f);
f = NULL; f = NULL;