From a65b22eebb3edcce4adb8a9d2d2cd85524dc11bd Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 25 Jan 2022 14:59:51 +0100 Subject: [PATCH 1/2] add TREXIO_AUTO back end for READONLY (r) mode --- src/templates_front/templator_front.org | 72 ++++++++++++++++++++----- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/src/templates_front/templator_front.org b/src/templates_front/templator_front.org index 8b7ea89..1af3c5d 100644 --- a/src/templates_front/templator_front.org +++ b/src/templates_front/templator_front.org @@ -179,7 +179,7 @@ __trexio_path__ = None | ~TREXIO_OPEN_ERROR~ | 15 | 'Error opening file' | | ~TREXIO_LOCK_ERROR~ | 16 | 'Error locking file' | | ~TREXIO_UNLOCK_ERROR~ | 17 | 'Error unlocking file' | - | ~TREXIO_FILE_ERROR~ | 18 | 'Invalid file handle' | + | ~TREXIO_FILE_ERROR~ | 18 | 'Invalid file' | | ~TREXIO_GROUP_READ_ERROR~ | 19 | 'Error reading group' | | ~TREXIO_GROUP_WRITE_ERROR~ | 20 | 'Error writing group' | | ~TREXIO_ELEM_READ_ERROR~ | 21 | 'Error reading element' | @@ -432,7 +432,7 @@ return '\n'.join(result) return "Error unlocking file"; break; case TREXIO_FILE_ERROR: - return "Invalid file handle"; + return "Invalid file"; break; case TREXIO_GROUP_READ_ERROR: return "Error reading group"; @@ -540,12 +540,15 @@ def string_of_error(return_code: int) -> str: 2) ~TREXIO_TEXT~ relies on basic file I/O in C, namely ~fopen, fclose, fprintf, fscanf~ etc. from ~stdio.h~ library. This back end is not optimized for performance. It is supposed to be used for debug purposes or, for example, when the user wants to modify some data manually within the file. This back end is supposed to work "out-of-the-box" since there are no external dependencies, which might be useful for users that do not have access to HDF5 library. The produced files usually have ~.txt~ extension. + ~TREXIO_AUTO~ can be provided as a back end when opening an existing TREXIO file in read-only ~'r'~ mode. + In this case, TREXIO will try to automatically detect the back end, which should be used to open the file. + Additional back ends can be implemented thanks to the modular nature of the front end. This can be achieved by adding a new ~case~ (corresponding to the desired back end) in the front-end ~switch~. Then the corresponding back-end ~has/read/write~ functions has to be implemented. For example, see the commented lines that correspond to the ~TREXIO_JSON~ back end (not implemented yet). - _Note_: It is important to increment the value of TREXIO_INVALID_BACK_END when a new back end is added. Otherwise, it will not be available. + _Note_: It is important to increment the value of ~TREXIO_INVALID_BACK_END~ when a new back end is added. Otherwise, it will not be available. *** C @@ -555,6 +558,7 @@ typedef int32_t back_end_t; #define TREXIO_HDF5 ( (back_end_t) 0 ) #define TREXIO_TEXT ( (back_end_t) 1 ) #define TREXIO_INVALID_BACK_END ( (back_end_t) 2 ) +#define TREXIO_AUTO TREXIO_INVALID_BACK_END /*#define TREXIO_JSON ( (back_end_t) 2 )*/ #define TREXIO_DELIM "\n" @@ -595,6 +599,7 @@ bool trexio_has_backend(back_end_t back_end) { integer(trexio_back_end_t), parameter :: TREXIO_TEXT = 1 ! integer(trexio_back_end_t), parameter :: TREXIO_JSON = 2 integer(trexio_back_end_t), parameter :: TREXIO_INVALID_BACK_END = 2 + integer(trexio_back_end_t), parameter :: TREXIO_AUTO = TREXIO_INVALID_BACK_END #+end_src The function below is a Fortran interface for the aforementioned C-compatible ~trexio_has_back_end~ function. @@ -628,6 +633,7 @@ TREXIO_HDF5 = 0 TREXIO_TEXT = 1 #TREXIO_JSON = 2 TREXIO_INVALID_BACK_END = 2 +TREXIO_AUTO = TREXIO_INVALID_BACK_END #+end_src ** Read/write behavior @@ -779,6 +785,7 @@ struct trexio_back_end_s { 3) ~back_end~ - integer number (or the corresponding global parameter) specifying the back end - ~TREXIO_HDF5~ - for HDF5 back end (integer alternative: 0) - ~TREXIO_TEXT~ - for TEXT back end (integer alternative: 1) + - ~TREXIO_AUTO~ - to automatically detect the applicable back end output: ~trexio_t~ file handle @@ -810,21 +817,56 @@ trexio_open(const char* file_name, const char mode, } /* Check overflow in file_name */ - if (back_end < 0 || back_end >= TREXIO_INVALID_BACK_END) { + /* Check that the mode is valid */ + if (mode != 'r' && mode != 'w') { + if (rc_open != NULL) *rc_open = TREXIO_INVALID_ARG_2; + return NULL; + } + + /* Check that the back end is valid in non-read mode */ + if ((back_end < 0 || back_end >= TREXIO_INVALID_BACK_END) && mode != 'r') { if (rc_open != NULL) *rc_open = TREXIO_INVALID_ARG_3; return NULL; } - if (mode != 'r' && mode != 'w') { - if (rc_open != NULL) *rc_open = TREXIO_INVALID_ARG_2; + /* Check that the back end is valid in read-only mode */ + if ((back_end < 0 || back_end > TREXIO_INVALID_BACK_END) && mode == 'r') { + if (rc_open != NULL) *rc_open = TREXIO_INVALID_ARG_3; return NULL; } + back_end_t back_end_local = back_end; + /* Try to determine the applicable backend if the back_end argument is TREXIO_AUTO */ + if (back_end == TREXIO_AUTO && mode == 'r') { +#ifdef HAVE_HDF5 + trexio_exit_code rc_text, rc_hdf5; + /* Check if the TREXIO file exists and if it is a directory */ + rc_text = trexio_text_inquire(file_name); + if (rc_text == TREXIO_SUCCESS) { + back_end_local = TREXIO_TEXT; + } else { + /* If not, check if it is an HDF5 file */ + rc_hdf5 = trexio_hdf5_inquire(file_name); + if (rc_hdf5 == TREXIO_SUCCESS) { + back_end_local = TREXIO_HDF5; + } else { + /* File is neither a directory nor an HDF5 file -> return an error */ + back_end_local = -1; + if (rc_open != NULL) *rc_open = TREXIO_FILE_ERROR; + return NULL; + } + } +#else + /* In the current implementation if HDF5 back end is not available - then there is only back end left */ + back_end_local = TREXIO_TEXT; +#endif + } + trexio_t* result = NULL; void* result_tmp = NULL; /* Allocate data structures */ - switch (back_end) { + switch (back_end_local) { case TREXIO_TEXT: result_tmp = malloc(sizeof(trexio_text_t)); @@ -838,6 +880,11 @@ trexio_open(const char* file_name, const char mode, if (rc_open != NULL) *rc_open = TREXIO_BACK_END_MISSING; return NULL; #endif + + default: + if (rc_open != NULL) *rc_open = TREXIO_FILE_ERROR; + return NULL; + /* case TREXIO_JSON: result = (trexio_t*) malloc (sizeof(trexio_json_t)); @@ -864,7 +911,7 @@ trexio_open(const char* file_name, const char mode, return NULL; } - result->back_end = back_end; + result->back_end = back_end_local; result->mode = mode; result->one_based = false; // Need to be flipped in Fortran interface int irc = pthread_mutex_init ( &(result->thread_lock), NULL); @@ -881,7 +928,7 @@ trexio_open(const char* file_name, const char mode, rc = TREXIO_OPEN_ERROR; - switch (back_end) { + switch (back_end_local) { case TREXIO_TEXT: rc = trexio_text_init(result); @@ -913,7 +960,7 @@ trexio_open(const char* file_name, const char mode, rc = TREXIO_LOCK_ERROR; - switch (back_end) { + switch (back_end_local) { case TREXIO_TEXT: rc = trexio_text_lock(result); @@ -949,7 +996,7 @@ trexio_open(const char* file_name, const char mode, } if (rc == TREXIO_HAS_NOT) { - switch (back_end) { + switch (back_end_local) { case TREXIO_TEXT: rc = trexio_text_write_metadata_package_version(result, TREXIO_PACKAGE_VERSION); @@ -1051,8 +1098,7 @@ trexio_exit_code trexio_set_one_based(trexio_t* file); #+begin_src c :tangle prefix_front.c trexio_exit_code trexio_set_one_based(trexio_t* file) { - if (file == NULL) - return TREXIO_FILE_ERROR; + if (file == NULL) return TREXIO_FILE_ERROR; file->one_based = true; From d9309f5ab2254ea30250f7003b7eb01a164f8a6a Mon Sep 17 00:00:00 2001 From: q-posev Date: Tue, 25 Jan 2022 15:00:55 +0100 Subject: [PATCH 2/2] modify some tests to use TREXIO_AUTO back end --- python/test/test_api.py | 2 +- tests/io_all.c | 2 +- tests/open_hdf5.c | 25 +++++++++++++++++++++++++ tests/open_text.c | 25 +++++++++++++++++++++++++ tests/test_f.f90 | 2 +- 5 files changed, 53 insertions(+), 3 deletions(-) diff --git a/python/test/test_api.py b/python/test/test_api.py index 86eb619..a600983 100644 --- a/python/test/test_api.py +++ b/python/test/test_api.py @@ -155,7 +155,7 @@ del test_file #==========================================================# # open previously created TREXIO file, now in 'read' mode -test_file2 = trexio.File(output_filename, 'r', TEST_TREXIO_BACKEND) +test_file2 = trexio.File(output_filename, 'r', trexio.TREXIO_AUTO) assert test_file2.exists # check for existence of some of the previously written variables diff --git a/tests/io_all.c b/tests/io_all.c index d2d1ed2..e476efd 100644 --- a/tests/io_all.c +++ b/tests/io_all.c @@ -163,7 +163,7 @@ int test_read(const char* file_name, const back_end_t backend) { /*================= START OF TEST ==================*/ // open existing file on 'read' mode [created by test_write] - file = trexio_open(file_name, 'r', backend, &rc); + file = trexio_open(file_name, 'r', TREXIO_AUTO, &rc); assert (file != NULL); // read nucleus_num diff --git a/tests/open_hdf5.c b/tests/open_hdf5.c index 7f4ec62..d0a6581 100644 --- a/tests/open_hdf5.c +++ b/tests/open_hdf5.c @@ -57,6 +57,30 @@ static int test_open_r (const char* file_name, const back_end_t backend) { } +static int test_open_auto (const char* file_name) { + +/* Try to open the TREXIO file in 'read' mode */ + + trexio_t* file = NULL; + trexio_exit_code rc; + +/*================= START OF TEST ==================*/ + + // open file in 'write' mode + file = trexio_open(file_name, 'r', TREXIO_AUTO, &rc); + assert (file != NULL); + assert (rc == TREXIO_SUCCESS); + + // close current session + rc = trexio_close(file); + assert (rc == TREXIO_SUCCESS); + +/*================= END OF TEST ==================*/ + + return 0; +} + + static int test_open_errors (const back_end_t backend) { /* Try to call trexio_open with bad arguments */ @@ -128,6 +152,7 @@ int main(void) { test_open_w (TREXIO_FILE, TEST_BACKEND); test_open_r (TREXIO_FILE, TEST_BACKEND); + test_open_auto (TREXIO_FILE); test_open_errors(TEST_BACKEND); test_inquire (TEST_BACKEND); diff --git a/tests/open_text.c b/tests/open_text.c index 4ec7810..130ba91 100644 --- a/tests/open_text.c +++ b/tests/open_text.c @@ -57,6 +57,30 @@ static int test_open_r (const char* file_name, const back_end_t backend) { } +static int test_open_auto (const char* file_name) { + +/* Try to open the TREXIO file in 'read' mode */ + + trexio_t* file = NULL; + trexio_exit_code rc; + +/*================= START OF TEST ==================*/ + + // open file in 'write' mode + file = trexio_open(file_name, 'r', TREXIO_AUTO, &rc); + assert (file != NULL); + assert (rc == TREXIO_SUCCESS); + + // close current session + rc = trexio_close(file); + assert (rc == TREXIO_SUCCESS); + +/*================= END OF TEST ==================*/ + + return 0; +} + + static int test_open_errors (const back_end_t backend) { /* Try to call trexio_open with bad arguments */ @@ -128,6 +152,7 @@ int main(void) { test_open_w (TREXIO_FILE, TEST_BACKEND); test_open_r (TREXIO_FILE, TEST_BACKEND); + test_open_auto (TREXIO_FILE); test_open_errors(TEST_BACKEND); test_inquire (TEST_BACKEND); diff --git a/tests/test_f.f90 b/tests/test_f.f90 index e7d4ee1..bc227ad 100644 --- a/tests/test_f.f90 +++ b/tests/test_f.f90 @@ -226,7 +226,7 @@ subroutine test_read(file_name, back_end) rc = trexio_inquire(file_name) call trexio_assert(rc, TREXIO_SUCCESS) - trex_file = trexio_open(file_name, 'r', back_end, rc) + trex_file = trexio_open(file_name, 'r', TREXIO_AUTO, rc) call trexio_assert(rc, TREXIO_SUCCESS) rc = trexio_read_nucleus_num(trex_file, num_read)