TREX/trexio
1
0
Fork 0
mirror of https://github.com/TREX-CoE/trexio.git synced 2024-11-03 20:54:07 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #82 from TREX-CoE/fix-text-compatibility

Browse Source 
refactor `trexio_text_read_group` set of functions to read data in an arbitrary order
This commit is contained in:
Evgeny Posenitskiy 2022-02-16 14:51:14 +01:00 committed by GitHub
commit 9482ff357d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 220 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ChangeLog
View File

@ -4,6 +4,7 @@ CHANGES
2.2 2.2
--- ---
- Fixed backwards incompatibility of the `TREXIO_TEXT` back end in #82
- Added `TREXIO_AUTO` back end for read-only mode (`r`) in PR #80 - Added `TREXIO_AUTO` back end for read-only mode (`r`) in PR #80
- Added unsafe mode (`u`) to `trexio_open`. When enabled, data can be - Added unsafe mode (`u`) to `trexio_open`. When enabled, data can be
overwritten (using conventional `trexio_write`) and groups can be deleted overwritten (using conventional `trexio_write`) and groups can be deleted
@ -59,4 +60,3 @@ access to both local and non-local quantities
- Using libtool/automake - Using libtool/automake
- Rewrote generator - Rewrote generator

409
src/templates_text/templator_text.org
View File

@ -80,12 +80,12 @@
#+begin_src c :tangle struct_text_group_dset.h #+begin_src c :tangle struct_text_group_dset.h
typedef struct $group$_s { typedef struct $group$_s {
$group_num_dtype_double$ $group_num$; $group_num_dtype_double$ $group_num$;
bool $group_num$_isSet;
$group_dset_dtype$* $group_dset$; $group_dset_dtype$* $group_dset$;
uint32_t rank_$group_dset$;
uint32_t to_flush;
uint64_t dims_$group_dset$[16]; uint64_t dims_$group_dset$[16];
uint64_t len_$group_str$; uint64_t len_$group_str$;
uint32_t rank_$group_dset$;
uint32_t to_flush;
bool $group_num$_isSet;
char* $group_str$; char* $group_str$;
char file_name[TREXIO_MAX_FILENAME_LENGTH]; char file_name[TREXIO_MAX_FILENAME_LENGTH];
} $group$_t; } $group$_t;
@ -318,234 +318,243 @@ trexio_text_read_$group$ (trexio_text_t* const file)
return NULL; return NULL;
} }
/* Read the dimensioning variables */
int rc = 0; int rc = 0;
// START REPEAT GROUP_DSET_ALL trexio_exit_code rc_free = TREXIO_FAILURE;
rc = fscanf(f, "%1023s", buffer);
if ((rc != 1) || (strcmp(buffer, "rank_$group_dset$") != 0)) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
rc = fscanf(f, "%u", &($group$->rank_$group_dset$));
if (rc != 1) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
/* workaround for the case of missing blocks in the file */ /* workaround for the case of missing blocks in the file */
// START REPEAT GROUP_DSET_ALL
uint64_t size_$group_dset$ = 0; uint64_t size_$group_dset$ = 0;
if ($group$->rank_$group_dset$ != 0) size_$group_dset$ = 1;
for (uint32_t i=0; i<$group$->rank_$group_dset$; ++i){
uint32_t j=0;
rc = fscanf(f, "%1023s %u", buffer, &j);
if ((rc != 2) || (strcmp(buffer, "dims_$group_dset$") != 0) || (j!=i)) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
rc = fscanf(f, "%" SCNu64 "\n", &($group$->dims_$group_dset$[i]));
assert(!(rc != 1));
if (rc != 1) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
size_$group_dset$ *= $group$->dims_$group_dset$[i];
}
// END REPEAT GROUP_DSET_ALL // END REPEAT GROUP_DSET_ALL
// START REPEAT GROUP_NUM while(fscanf(f, "%1023s", buffer) != EOF) {
/* Read data */
unsigned int $group_num$_isSet;
rc = fscanf(f, "%1023s", buffer);
assert(!((rc != 1) || (strcmp(buffer, "$group_num$_isSet") != 0)));
if ((rc != 1) || (strcmp(buffer, "$group_num$_isSet") != 0)) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
/* additional parameter $group_num$_isSet is needed to suppress warning when fscanf into bool variable using %u or %d */ if (strcmp(buffer, "EXIT") == 0) {
rc = fscanf(f, "%u", &($group_num$_isSet)); break;
$group$->$group_num$_isSet = (bool) $group_num$_isSet; // START REPEAT GROUP_DSET_ALL
assert(!(rc != 1)); } else if (strcmp(buffer, "rank_$group_dset$") == 0) {
if (rc != 1) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
if ($group$->$group_num$_isSet == true) { rc = fscanf(f, "%u", &($group$->rank_$group_dset$));
rc = fscanf(f, "%1023s", buffer); if (rc != 1) {
assert(!((rc != 1) || (strcmp(buffer, "$group_num$") != 0))); FREE(buffer);
if ((rc != 1) || (strcmp(buffer, "$group_num$") != 0)) { fclose(f);
FREE(buffer); /* Set pointer to the struct so that the garbage collector can do the job on file handle */
fclose(f); file->$group$ = $group$;
FREE($group$); rc_free = trexio_text_free_$group$(file);
return NULL; assert(rc_free == TREXIO_SUCCESS);
} return NULL;
}
rc = fscanf(f, "%$group_num_format_scanf$", &($group$->$group_num$)); if ($group$->rank_$group_dset$ != 0) size_$group_dset$ = 1UL;
assert(!(rc != 1));
if (rc != 1) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
}
// END REPEAT GROUP_NUM
// START REPEAT GROUP_ATTR_STR for (uint32_t i=0; i<$group$->rank_$group_dset$; ++i){
rc = fscanf(f, "%1023s", buffer);
assert(!((rc != 1) || (strcmp(buffer, "len_$group_str$") != 0)));
if ((rc != 1) || (strcmp(buffer, "len_$group_str$") != 0)) {
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
rc = fscanf(f, "%" SCNu64 "", &($group$->len_$group_str$)); uint32_t j=0;
assert(!(rc != 1)); rc = fscanf(f, "%1023s %u", buffer, &j);
if (rc != 1) { if ((rc != 2) || (strcmp(buffer, "dims_$group_dset$") != 0) || (j!=i)) {
FREE(buffer); FREE(buffer);
fclose(f); fclose(f);
FREE($group$); /* Set pointer to the struct so that the garbage collector can do the job on file handle */
return NULL; file->$group$ = $group$;
} rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
rc = fscanf(f, "%1023s", buffer); rc = fscanf(f, "%" SCNu64 "\n", &($group$->dims_$group_dset$[i]));
assert(!((rc != 1) || (strcmp(buffer, "$group_str$") != 0))); assert(!(rc != 1));
if ((rc != 1) || (strcmp(buffer, "$group_str$") != 0)) { if (rc != 1) {
FREE(buffer); FREE(buffer);
fclose(f); fclose(f);
FREE($group$->$group_str$); /* Set pointer to the struct so that the garbage collector can do the job on file handle */
FREE($group$); file->$group$ = $group$;
return NULL; rc_free = trexio_text_free_$group$(file);
} assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
if ($group$->len_$group_str$ != 0) { size_$group_dset$ *= $group$->dims_$group_dset$[i];
}
// END REPEAT GROUP_DSET_ALL
// START REPEAT GROUP_DSET_NUM
} else if (strcmp(buffer, "$group_dset$") == 0) {
$group$->$group_str$ = CALLOC($group$->len_$group_str$, char); /* Allocate arrays */
assert (!($group$->$group_str$ == NULL)); $group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$);
if ($group$->$group_str$ == NULL) { if ($group$->$group_dset$ == NULL) {
FREE(buffer); FREE(buffer);
fclose(f); fclose(f);
FREE($group$->$group_str$); /* Set pointer to the struct so that the garbage collector can do the job on file handle */
FREE($group$); file->$group$ = $group$;
return NULL; rc_free = trexio_text_free_$group$(file);
} assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
rc = fscanf(f, " %1023[^\n]", $group$->$group_str$); for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
assert(!(rc != 1)); rc = fscanf(f, "%$group_dset_format_scanf$", &($group$->$group_dset$[i]));
if (rc != 1) { if (rc != 1) {
FREE(buffer); FREE(buffer);
fclose(f); fclose(f);
FREE($group$->$group_str$); /* Set pointer to the struct so that the garbage collector can do the job on file handle */
FREE($group$); file->$group$ = $group$;
return NULL; rc_free = trexio_text_free_$group$(file);
} assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
}
} // END REPEAT GROUP_DSET_NUM
// END REPEAT GROUP_ATTR_STR // START REPEAT GROUP_DSET_STR
} else if (strcmp(buffer, "$group_dset$") == 0) {
// START REPEAT GROUP_DSET_NUM if(size_$group_dset$ != 0) {
/* Allocate arrays */ /* Allocate arrays */
$group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$); $group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$);
assert (!($group$->$group_dset$ == NULL)); if ($group$->$group_dset$ == NULL) {
if ($group$->$group_dset$ == NULL) { FREE(buffer);
FREE(buffer); fclose(f);
fclose(f); /* Set pointer to the struct so that the garbage collector can do the job on file handle */
FREE($group$->$group_dset$); file->$group$ = $group$;
FREE($group$); rc_free = trexio_text_free_$group$(file);
return NULL; assert(rc_free == TREXIO_SUCCESS);
} return NULL;
}
rc = fscanf(f, "%1023s", buffer); /* WARNING: this tmp array allows to avoid allocation of space for each element of array of string
assert(!((rc != 1) || (strcmp(buffer, "$group_dset$") != 0))); BUT it's size has to be number_of_str*max_len_str where max_len_str is somewhat arbitrary, e.g. 32.
if ((rc != 1) || (strcmp(buffer, "$group_dset$") != 0)) { ,*/
FREE(buffer); char* tmp_$group_dset$;
fclose(f); tmp_$group_dset$ = CALLOC(size_$group_dset$*32, char);
FREE($group$->$group_dset$);
FREE($group$);
return NULL;
}
for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) { for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
rc = fscanf(f, "%$group_dset_format_scanf$", &($group$->$group_dset$[i])); $group$->$group_dset$[i] = tmp_$group_dset$;
assert(!(rc != 1)); /* conventional fcanf with "%s" only return the string before the first space character
if (rc != 1) { ,* to read string with spaces use "%[^\n]" possible with space before or after, i.e. " %[^\n]"
FREE(buffer); ,* Q: depending on what ? */
fclose(f); rc = fscanf(f, " %1023[^\n]", tmp_$group_dset$);
FREE($group$->$group_dset$); assert(!(rc != 1));
FREE($group$); if (rc != 1) {
return NULL; FREE(buffer);
} fclose(f);
} /* Set pointer to the struct so that the garbage collector can do the job on file handle */
// END REPEAT GROUP_DSET_NUM file->$group$ = $group$;
rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
// START REPEAT GROUP_DSET_STR size_t tmp_$group_dset$_len = strlen($group$->$group_dset$[i]);
rc = fscanf(f, "%1023s", buffer); tmp_$group_dset$ += tmp_$group_dset$_len + 1;
assert(!((rc != 1) || (strcmp(buffer, "$group_dset$") != 0))); }
if ((rc != 1) || (strcmp(buffer, "$group_dset$") != 0)) { }
FREE(buffer);
fclose(f);
FREE($group$);
return NULL;
}
/* Allocate arrays */ // END REPEAT GROUP_DSET_STR
if(size_$group_dset$ != 0) { // START REPEAT GROUP_NUM
$group$->$group_dset$ = CALLOC(size_$group_dset$, $group_dset_dtype$); } else if (strcmp(buffer, "$group_num$_isSet") == 0) {
if ($group$->$group_dset$ == NULL) {
FREE(buffer);
fclose(f);
FREE($group$->$group_dset$);
FREE($group$);
return NULL;
}
/* WARNING: this tmp array allows to avoid allocation of space for each element of array of string unsigned int $group_num$_isSet;
BUT it's size has to be number_of_str*max_len_str where max_len_str is somewhat arbitrary, e.g. 32. /* additional parameter $group_num$_isSet is needed to suppress warning when fscanf into bool variable using %u or %d */
,*/ rc = fscanf(f, "%u", &($group_num$_isSet));
char* tmp_$group_dset$; $group$->$group_num$_isSet = (bool) $group_num$_isSet;
tmp_$group_dset$ = CALLOC(size_$group_dset$*32, char);
for (uint64_t i=0 ; i<size_$group_dset$ ; ++i) {
$group$->$group_dset$[i] = tmp_$group_dset$;
/* conventional fcanf with "%s" only return the string before the first space character
,* to read string with spaces use "%[^\n]" possible with space before or after, i.e. " %[^\n]"
,* Q: depending on what ? */
rc = fscanf(f, " %1023[^\n]", tmp_$group_dset$);
assert(!(rc != 1)); assert(!(rc != 1));
if (rc != 1) { if (rc != 1) {
FREE(buffer); FREE(buffer);
fclose(f); fclose(f);
FREE($group$->$group_dset$); /* Set pointer to the struct so that the garbage collector can do the job on file handle */
FREE($group$); file->$group$ = $group$;
rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL; return NULL;
} }
size_t tmp_$group_dset$_len = strlen($group$->$group_dset$[i]); if ($group$->$group_num$_isSet == true) {
tmp_$group_dset$ += tmp_$group_dset$_len + 1; rc = fscanf(f, "%1023s", buffer);
assert(!((rc != 1) || (strcmp(buffer, "$group_num$") != 0)));
if ((rc != 1) || (strcmp(buffer, "$group_num$") != 0)) {
FREE(buffer);
fclose(f);
/* Set pointer to the struct so that the garbage collector can do the job on file handle */
file->$group$ = $group$;
rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
rc = fscanf(f, "%$group_num_format_scanf$", &($group$->$group_num$));
assert(!(rc != 1));
if (rc != 1) {
FREE(buffer);
fclose(f);
/* Set pointer to the struct so that the garbage collector can do the job on file handle */
file->$group$ = $group$;
rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
}
// END REPEAT GROUP_NUM
// START REPEAT GROUP_ATTR_STR
} else if (strcmp(buffer, "len_$group_str$") == 0) {
rc = fscanf(f, "%" SCNu64 "", &($group$->len_$group_str$));
assert(!(rc != 1));
if (rc != 1) {
FREE(buffer);
fclose(f);
/* Set pointer to the struct so that the garbage collector can do the job on file handle */
file->$group$ = $group$;
rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
rc = fscanf(f, "%1023s", buffer);
assert(!((rc != 1) || (strcmp(buffer, "$group_str$") != 0)));
if ((rc != 1) || (strcmp(buffer, "$group_str$") != 0)) {
FREE(buffer);
fclose(f);
/* Set pointer to the struct so that the garbage collector can do the job on file handle */
file->$group$ = $group$;
rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
if ($group$->len_$group_str$ != 0) {
$group$->$group_str$ = CALLOC($group$->len_$group_str$, char);
assert (!($group$->$group_str$ == NULL));
if ($group$->$group_str$ == NULL) {
FREE(buffer);
fclose(f);
/* Set pointer to the struct so that the garbage collector can do the job on file handle */
file->$group$ = $group$;
rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
rc = fscanf(f, " %1023[^\n]", $group$->$group_str$);
assert(!(rc != 1));
if (rc != 1) {
FREE(buffer);
fclose(f);
/* Set pointer to the struct so that the garbage collector can do the job on file handle */
file->$group$ = $group$;
rc_free = trexio_text_free_$group$(file);
assert(rc_free == TREXIO_SUCCESS);
return NULL;
}
}
// END REPEAT GROUP_ATTR_STR
} else {
continue;
} }
} }
// END REPEAT GROUP_DSET_STR
FREE(buffer); FREE(buffer);
fclose(f); fclose(f);

41
tools/generator_tools.py
View File

@ -311,8 +311,6 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai
subloop_dset = False subloop_dset = False
subloop_num = False subloop_num = False
loop_body = '' loop_body = ''
dset_allocated = []
str_allocated = []
for line in f_in : for line in f_in :
@ -335,25 +333,12 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai
if ('REPEAT GROUP_DSET_NUM' in line) and (detailed_dset[dset]['group_dset_dtype'] == 'char*'): if ('REPEAT GROUP_DSET_NUM' in line) and (detailed_dset[dset]['group_dset_dtype'] == 'char*'):
continue continue
dset_allocated.append(dset) save_body = loop_body
populated_body = recursive_replace_line(save_body, triggers, detailed_dset[dset])
if 'FREE($group$->$group_dset$)' in loop_body: f_out.write(populated_body)
tmp_string = ''
for dset_alloc in dset_allocated:
tmp_string += f'FREE({group}->{dset_alloc});\n '
tmp_body = loop_body.replace('FREE($group$->$group_dset$);', tmp_string)
populated_body = recursive_replace_line(tmp_body, triggers, detailed_dset[dset])
f_out.write(populated_body)
else:
save_body = loop_body
populated_body = recursive_replace_line(save_body, triggers, detailed_dset[dset])
f_out.write(populated_body)
subloop_dset = False subloop_dset = False
loop_body = '' loop_body = ''
dset_allocated = []
continue continue
elif 'END REPEAT GROUP_NUM' in line: elif 'END REPEAT GROUP_NUM' in line:
@ -374,26 +359,12 @@ def special_populate_text_group(fname: str, paths: dict, group_dict: dict, detai
if group != detailed_strings[str]['group']: if group != detailed_strings[str]['group']:
continue continue
str_allocated.append(str) save_body = loop_body
populated_body = recursive_replace_line(save_body, triggers, detailed_strings[str])
if 'FREE($group$->$group_str$)' in loop_body: f_out.write(populated_body)
tmp_string = ''
for str_alloc in str_allocated:
tmp_string += f'FREE({group}->{str_alloc});\n '
tmp_body = loop_body.replace('FREE($group$->$group_str$);', tmp_string)
populated_body = recursive_replace_line(tmp_body, triggers, detailed_strings[str])
f_out.write(populated_body)
else:
save_body = loop_body
populated_body = recursive_replace_line(save_body, triggers, detailed_strings[str])
f_out.write(populated_body)
subloop_num = False subloop_num = False
loop_body = '' loop_body = ''
str_allocated = []
continue continue
if not subloop_num and not subloop_dset: if not subloop_num and not subloop_dset: