#+Title: Templator for TEXT backend * Constant file prefixes (not used by generator) :noxport: #+begin_src emacs-lisp (setq-local org-babel-default-header-args:c '((:comments . "both"))) org-babel-default-header-args:c #+end_src #+RESULTS: : ((:comments . both)) # Local Variables: # eval: (setq-local org-babel-default-header-args:Python '((:session . "foo"))) # End: #+NAME:header #+begin_src c /* This file was generated from the trexio.org org-mode file. To generate it, open trexio.org in Emacs and execute M-x org-babel-tangle */ #+end_src #+begin_src c :tangle prefix_text.h :noweb yes <
> #ifndef _TREXIO_TEXT_H #define _TREXIO_TEXT_H #include "trexio.h" #include "trexio_private.h" #include "trexio_s.h" #include #include #include #include #include #include #include #include #include #+end_src #+begin_src c :tangle prefix_text.c :noweb yes /* This file was generated from the trexio.org org-mode file. To generate it, open trexio.org in Emacs and execute M-x org-babel-tangle */ #include "trexio_text.h" #define DEBUG printf("%s : line %d\n", __FILE__, __LINE__); #+end_src * TEXT Back end The "file" produced by the text back end is a directory with one file per group. When the file is open, it is locked by the current process. No other process can read/write the same file. This guarantees that the representation in memory is consistent with the file and avoid re-reading the file before writing. To lock the file, we lock the =.lock= file which is present in the directory. The file is written when closed, or when the flush function is called. *** Template for group-related structures in text back end #+begin_src c :tangle struct_text_group_dset.h typedef struct $group$_s { FILE* file; $group_dset_dtype$* $group_dset$; uint64_t $group_num$; uint64_t dims_$group_dset$[16]; uint32_t rank_$group_dset$; int to_flush; } $group$_t; #+end_src *** Template for general structure in text back end #+begin_src c :tangle struct_text_group.h typedef struct rdm_s { FILE* file; uint64_t dim_one_e; double* one_e; char* two_e_file_name; int to_flush; } rdm_t; typedef struct trexio_text_s { trexio_t parent ; int lock_file; $group$_t* $group$; rdm_t* rdm; } trexio_text_t; #+end_src *** Init/deinit functions (constant part) #+begin_src c :tangle basic_text.h trexio_exit_code trexio_text_init(trexio_t* file); #+end_src #+begin_src c :tangle basic_text.c trexio_exit_code trexio_text_init(trexio_t* file) { if (file == NULL) return TREXIO_INVALID_ARG_1; trexio_text_t* f = (trexio_text_t*) file; /* Put all pointers to NULL but leave parent untouched */ memset(&(f->parent)+1,0,sizeof(trexio_text_t)-sizeof(trexio_t)); /* If directory doesn't exist, create it in write mode */ struct stat st; if (stat(file->file_name, &st) == 0 && S_ISDIR(st.st_mode)) { /* Do nothing */ } else { if (file->mode == 'r') return TREXIO_READONLY; if (mkdir(file->file_name, 0777) != 0) { return TREXIO_FAILURE; } } /* Create the lock file in the directory */ const char* lock_file_name = "/.lock"; char* file_name = CALLOC(strlen(file->file_name) + strlen(lock_file_name) + 1, char); if (file_name == NULL) { return TREXIO_ALLOCATION_FAILED; } strcpy (file_name, file->file_name); strcat (file_name, lock_file_name); f->lock_file = open(file_name,O_WRONLY|O_CREAT|O_TRUNC, 0644); FREE(file_name); if (f->lock_file <= 0) { return TREXIO_FAILURE; } return TREXIO_SUCCESS; } #+end_src #+begin_src c :tangle basic_text.h trexio_exit_code trexio_text_lock(trexio_t* file); #+end_src #+begin_src c :tangle basic_text.c trexio_exit_code trexio_text_lock(trexio_t* file) { if (file == NULL) return TREXIO_INVALID_ARG_1; trexio_text_t* f = (trexio_text_t*) file; struct flock fl; fl.l_type = F_WRLCK; fl.l_whence = SEEK_SET; fl.l_start = 0; fl.l_len = 0; fl.l_pid = getpid(); int rc = fcntl(f->lock_file, F_SETLKW, &fl); if (rc == -1) return TREXIO_FAILURE; return TREXIO_SUCCESS; } #+end_src #+begin_src c :tangle basic_text.h trexio_exit_code trexio_text_finalize(trexio_t* file); #+end_src #+begin_src c :tangle basic_text.h trexio_exit_code trexio_text_unlock(trexio_t* file); #+end_src #+begin_src c :tangle basic_text.c trexio_exit_code trexio_text_unlock(trexio_t* file) { if (file == NULL) return TREXIO_INVALID_ARG_1; trexio_text_t* f = (trexio_text_t*) file; struct flock fl; fl.l_type = F_WRLCK; fl.l_whence = SEEK_SET; fl.l_start = 0; fl.l_len = 0; fl.l_pid = getpid(); fl.l_type = F_UNLCK; fcntl(f->lock_file, F_SETLK, &fl); close(f->lock_file); return TREXIO_SUCCESS; } #+end_src *** Init/deinit functions (templated part) #+begin_src c :tangle basic_text_group.c trexio_exit_code trexio_text_finalize(trexio_t* file) { if (file == NULL) return TREXIO_INVALID_ARG_1; trexio_exit_code rc; //rc = trexio_text_free_$group$( (trexio_text_t*) file); //assert (rc == TREXIO_SUCCESS); assert (trexio_text_free_$group$( (trexio_text_t*) file) == TREXIO_SUCCESS); rc = trexio_text_free_rdm( (trexio_text_t*) file); assert (rc == TREXIO_SUCCESS); return TREXIO_SUCCESS; } #+end_src *** Template for text read struct #+begin_src c :tangle read_group_text.h $group$_t* trexio_text_read_$group$(trexio_text_t* file); #+end_src #+begin_src c :tangle read_group_text.c $group$_t* trexio_text_read_$group$(trexio_text_t* file) { if (file == NULL) return NULL; /* If the data structure exists, return it */ if (file->$group$ != NULL) { return file->$group$; } /* Allocate the data structure */ $group$_t* $group$ = MALLOC($group$_t); if ($group$ == NULL) return NULL; memset($group$,0,sizeof($group$_t)); /* Build the file name */ const char* $group$_file_name = "/$group$.txt"; char * file_name = (char*) calloc( strlen(file->parent.file_name) + strlen($group$_file_name) + 1, sizeof(char)); if (file_name == NULL) { FREE($group$); DEBUG return NULL; } strcpy (file_name, file->parent.file_name); strcat (file_name, $group$_file_name); /* If the file exists, read it */ FILE* f = fopen(file_name,"r"); if (f != NULL) { /* Find size of file to allocate the max size of the string buffer */ fseek(f, 0L, SEEK_END); size_t sz = ftell(f); fseek(f, 0L, SEEK_SET); char* buffer = CALLOC(sz,char); if (buffer == NULL) { FREE(file_name); FREE($group$); DEBUG return NULL; } /* Read the dimensioning variables */ int rc; // START REPEAT GROUP_DSET rc = fscanf(f, "%s", buffer); if ((rc != 1) || (strcmp(buffer, "rank_$group_dset$") != 0)) { FREE(buffer); FREE(file_name); FREE($group$); DEBUG return NULL; } rc = fscanf(f, "%u", &($group$->rank_$group_dset$)); if (rc != 1) { FREE(buffer); FREE(file_name); FREE($group$); DEBUG return NULL; } // workaround for the case of missing blocks in the file uint64_t size_$group_dset$ = 0; if ($group$->rank_$group_dset$ != 0) size_$group_dset$ = 1; for (unsigned int i=0; i<$group$->rank_$group_dset$; i++){ unsigned int j=-1; rc = fscanf(f, "%s %u", buffer, &j); if ((rc != 2) || (strcmp(buffer, "dims_$group_dset$") != 0) || (j!=i)) { FREE(buffer); FREE(file_name); FREE($group$); DEBUG return NULL; } rc = fscanf(f, "%lu\n", &($group$->dims_$group_dset$[i])); assert(!(rc != 1)); if (rc != 1) { FREE(buffer); FREE(file_name); FREE($group$); DEBUG return NULL; } size_$group_dset$ *= $group$->dims_$group_dset$[i]; } // END REPEAT GROUP_DSET // START REPEAT GROUP_NUM /* Read data */ rc = fscanf(f, "%s", buffer); assert(!((rc != 1) || (strcmp(buffer, "$group_num$") != 0))); if ((rc != 1) || (strcmp(buffer, "$group_num$") != 0)) { FREE(buffer); FREE(file_name); FREE($group$); DEBUG return NULL; } rc = fscanf(f, "%lu", &($group$->$group_num$)); assert(!(rc != 1)); if (rc != 1) { FREE(buffer); FREE(file_name); FREE($group$); DEBUG return NULL; } // END REPEAT GROUP_NUM // START REPEAT GROUP_DSET /* Allocate arrays */ $group$->$group_dset$ = ($group_dset_dtype$*) calloc(size_$group_dset$, sizeof($group_dset_dtype$)); assert (!($group$->$group_dset$ == NULL)); if ($group$->$group_dset$ == NULL) { FREE(buffer); FREE(file_name); FREE($group$); DEBUG return NULL; } rc = fscanf(f, "%s", buffer); assert(!((rc != 1) || (strcmp(buffer, "$group_dset$") != 0))); if ((rc != 1) || (strcmp(buffer, "$group_dset$") != 0)) { FREE(buffer); FREE(file_name); // TODO: free all dsets FREE($group$->$group_dset$); FREE($group$); DEBUG return NULL; } for (uint64_t i=0 ; i$group_dset$[i])); assert(!(rc != 1)); if (rc != 1) { FREE(buffer); FREE(file_name); // TODO: free all dsets FREE($group$->$group_dset$); FREE($group$); DEBUG return NULL; } } // END REPEAT GROUP_DSET FREE(buffer); fclose(f); f = NULL; } if (file->parent.mode == 'w') { $group$->file = fopen(file_name,"a"); } else { $group$->file = fopen(file_name,"r"); } FREE(file_name); assert (!($group$->file == NULL)); if ($group$->file == NULL) { // TODO: free all dsets FREE($group$->$group_dset$); FREE($group$); DEBUG return NULL; } fseek($group$->file, 0L, SEEK_SET); file->$group$ = $group$; return $group$; } #+end_src *** Template for text flush struct #+begin_src c :tangle flush_group_text.h trexio_exit_code trexio_text_flush_$group$(const trexio_text_t* file); #+end_src #+begin_src c :tangle flush_group_text.c trexio_exit_code trexio_text_flush_$group$(const trexio_text_t* file) { if (file == NULL) return TREXIO_INVALID_ARG_1; if (file->parent.mode == 'r') return TREXIO_READONLY; $group$_t* $group$ = file->$group$; if ($group$ == NULL) return TREXIO_SUCCESS; if ($group$->to_flush == 0) return TREXIO_SUCCESS; FILE* f = $group$->file; assert (f != NULL); rewind(f); /* Write the dimensioning variables */ // START REPEAT GROUP_DSET fprintf(f, "rank_$group_dset$ %d\n", $group$->rank_$group_dset$); // workaround for the case of missing blocks in the file uint64_t size_$group_dset$ = 0; if ($group$->rank_$group_dset$ != 0) size_$group_dset$ = 1; for (unsigned int i=0; i<$group$->rank_$group_dset$; i++){ fprintf(f, "dims_$group_dset$ %d %ld\n", i, $group$->dims_$group_dset$[i]); size_$group_dset$ *= $group$->dims_$group_dset$[i]; } // END REPEAT GROUP_DSET // START REPEAT GROUP_NUM fprintf(f, "$group_num$ %ld\n", $group$->$group_num$); // END REPEAT GROUP_NUM // START REPEAT GROUP_DSET /* Write arrays */ fprintf(f, "$group_dset$\n"); for (uint64_t i=0 ; i$group_dset$[i]); } // END REPEAT GROUP_DSET fflush(f); file->$group$->to_flush = 0; return TREXIO_SUCCESS; } #+end_src *** Template for text free memory Memory is allocated when reading. The following function frees memory. #+begin_src c :tangle free_group_text.h trexio_exit_code trexio_text_free_$group$(trexio_text_t* file); #+end_src #+begin_src c :tangle free_group_text.c trexio_exit_code trexio_text_free_$group$(trexio_text_t* file) { if (file == NULL) return TREXIO_INVALID_ARG_1; trexio_exit_code rc; if (file->parent.mode != 'r') { rc = trexio_text_flush_$group$(file); if (rc != TREXIO_SUCCESS) return TREXIO_FAILURE; } $group$_t* $group$ = file->$group$; if ($group$ == NULL) return TREXIO_SUCCESS; if ($group$->file != NULL) { fclose($group$->file); $group$->file = NULL; } // START REPEAT GROUP_DSET if ($group$->$group_dset$ != NULL) { FREE ($group$->$group_dset$); } // END REPEAT GROUP_DSET FREE ($group$); return TREXIO_SUCCESS; } #+end_src *** Template for read/write the $group_num$ attribute #+begin_src c :tangle rw_num_text.h trexio_exit_code trexio_text_read_$group_num$(const trexio_t* file, uint64_t* num); trexio_exit_code trexio_text_write_$group_num$(const trexio_t* file, const uint64_t num); #+end_src #+begin_src c :tangle read_num_text.c trexio_exit_code trexio_text_read_$group_num$(const trexio_t* file, uint64_t* num) { if (file == NULL) return TREXIO_INVALID_ARG_1; if (num == NULL) return TREXIO_INVALID_ARG_2; $group$_t* $group$ = trexio_text_read_$group$((trexio_text_t*) file); if ($group$ == NULL) return TREXIO_FAILURE; /**/ *num = $group$->$group_num$; return TREXIO_SUCCESS; } #+end_src #+begin_src c :tangle write_num_text.c trexio_exit_code trexio_text_write_$group_num$(const trexio_t* file, const uint64_t num) { if (file == NULL) return TREXIO_INVALID_ARG_1; if (file->mode == 'r') return TREXIO_READONLY; $group$_t* $group$ = trexio_text_read_$group$((trexio_text_t*) file); if ($group$ == NULL) return TREXIO_FAILURE; $group$->$group_num$ = num; $group$->to_flush = 1; return TREXIO_SUCCESS; } #+end_src *** Template for read/write the $group_dset$ dataset The ~dset~ array is assumed allocated with the appropriate size. #+begin_src c :tangle rw_dset_text.h trexio_exit_code trexio_text_read_$group_dset$(const trexio_t* file, $group_dset_dtype$* $group_dset$, const uint32_t rank, const uint64_t* dims); trexio_exit_code trexio_text_write_$group_dset$(const trexio_t* file, const $group_dset_dtype$* $group_dset$, const uint32_t rank, const uint64_t* dims); #+end_src #+begin_src c :tangle read_dset_text.c trexio_exit_code trexio_text_read_$group_dset$(const trexio_t* file, $group_dset_dtype$* $group_dset$, const uint32_t rank, const uint64_t* dims) { if (file == NULL) return TREXIO_INVALID_ARG_1; if ($group_dset$ == NULL) return TREXIO_INVALID_ARG_2; $group$_t* $group$ = trexio_text_read_$group$((trexio_text_t*) file); if ($group$ == NULL) return TREXIO_FAILURE; if (rank != $group$->rank_$group_dset$) return TREXIO_INVALID_ARG_3; uint64_t dim_size = 1; for (unsigned int i=0; idims_$group_dset$[i]) return TREXIO_INVALID_ARG_4; dim_size *= dims[i]; } for (uint64_t i=0 ; i$group_dset$[i]; } return TREXIO_SUCCESS; } #+end_src #+begin_src c :tangle write_dset_text.c trexio_exit_code trexio_text_write_$group_dset$(const trexio_t* file, const $group_dset_dtype$* $group_dset$, const uint32_t rank, const uint64_t* dims) { if (file == NULL) return TREXIO_INVALID_ARG_1; if ($group_dset$ == NULL) return TREXIO_INVALID_ARG_2; if (file->mode == 'r') return TREXIO_READONLY; $group$_t* $group$ = trexio_text_read_$group$((trexio_text_t*) file); if ($group$ == NULL) return TREXIO_FAILURE; if ($group$->$group_dset$ != NULL) { FREE($group$->$group_dset$); } $group$->rank_$group_dset$ = rank; uint64_t dim_size = 1; for (unsigned int i=0; i<$group$->rank_$group_dset$; i++){ $group$->dims_$group_dset$[i] = dims[i]; dim_size *= dims[i]; } $group$->$group_dset$ = ($group_dset_dtype$*) calloc(dim_size, sizeof($group_dset_dtype$)); for (uint64_t i=0 ; i$group_dset$[i] = $group_dset$[i]; } $group$->to_flush = 1; return TREXIO_SUCCESS; } #+end_src *** RDM struct **** Read the complete struct #+begin_src c :tangle rdm_text.h rdm_t* trexio_text_read_rdm(trexio_text_t* file); #+end_src #+begin_src c :tangle rdm_text.c rdm_t* trexio_text_read_rdm(trexio_text_t* file) { if (file == NULL) return NULL; if (file->rdm != NULL) return file->rdm; /* Allocate the data structure */ rdm_t* rdm = MALLOC(rdm_t); assert (rdm != NULL); rdm->one_e = NULL; rdm->two_e_file_name = NULL; rdm->file = NULL; rdm->to_flush = 0; /* Try to open the file. If the file does not exist, return */ const char* rdm_file_name = "/rdm.txt"; char * file_name = (char*) calloc( strlen(file->parent.file_name) + strlen(rdm_file_name) + 1, sizeof(char)); assert (file_name != NULL); strcpy (file_name, file->parent.file_name); strcat (file_name, rdm_file_name); /* If the file exists, read it */ FILE* f = fopen(file_name,"r"); if (f != NULL) { /* Find size of file to allocate the max size of the string buffer */ fseek(f, 0L, SEEK_END); size_t sz = ftell(f); fseek(f, 0L, SEEK_SET); char* buffer = CALLOC(sz,char); /* Read the dimensioning variables */ int rc; rc = fscanf(f, "%s", buffer); assert (rc == 1); assert (strcmp(buffer, "dim_one_e") == 0); rc = fscanf(f, "%lu", &(rdm->dim_one_e)); assert (rc == 1); /* Allocate arrays */ rdm->one_e = (double*) calloc(rdm->dim_one_e, sizeof(double)); assert (rdm->one_e != NULL); /* Read one_e */ rc = fscanf(f, "%s", buffer); assert (rc == 1); assert (strcmp(buffer, "one_e") == 0); for (uint64_t i=0 ; idim_one_e; i++) { rc = fscanf(f, "%lf", &(rdm->one_e[i])); assert (rc == 1); } /* Read two_e */ rc = fscanf(f, "%s", buffer); assert (rc == 1); assert (strcmp(buffer, "two_e_file_name") == 0); rc = fscanf(f, "%s", buffer); assert (rc == 1); rdm->two_e_file_name = CALLOC (strlen(buffer),char); strcpy(rdm->two_e_file_name, buffer); FREE(buffer); fclose(f); f = NULL; } if (file->parent.mode == 'w') { rdm->file = fopen(file_name,"a"); } else { rdm->file = fopen(file_name,"r"); } FREE(file_name); file->rdm = rdm ; return rdm; } #+end_src **** Flush the complete struct #+begin_src c :tangle rdm_text.h trexio_exit_code trexio_text_flush_rdm(const trexio_text_t* file); #+end_src #+begin_src c :tangle rdm_text.c trexio_exit_code trexio_text_flush_rdm(const trexio_text_t* file) { if (file == NULL) return TREXIO_INVALID_ARG_1; if (file->parent.mode == 'r') return TREXIO_READONLY; rdm_t* rdm = file->rdm; if (rdm == NULL) return TREXIO_SUCCESS; if (rdm->to_flush == 0) return TREXIO_SUCCESS; FILE* f = rdm->file; assert (f != NULL); rewind(f); /* Write the dimensioning variables */ fprintf(f, "num %ld\n", rdm->dim_one_e); /* Write arrays */ fprintf(f, "one_e\n"); for (uint64_t i=0 ; i< rdm->dim_one_e; i++) { fprintf(f, "%lf\n", rdm->one_e[i]); } fprintf(f, "two_e_file_name\n"); fprintf(f, "%s\n", rdm->two_e_file_name); fflush(f); file->rdm->to_flush = 0; return TREXIO_SUCCESS; } #+end_src **** Free memory Memory is allocated when reading. The followig function frees memory. #+begin_src c :tangle rdm_text.h trexio_exit_code trexio_text_free_rdm(trexio_text_t* file); #+end_src #+begin_src c :tangle rdm_text.c trexio_exit_code trexio_text_free_rdm(trexio_text_t* file) { if (file == NULL) return TREXIO_INVALID_ARG_1; trexio_exit_code rc; if (file->parent.mode != 'r') { rc = trexio_text_flush_rdm(file); if (rc != TREXIO_SUCCESS) return TREXIO_FAILURE; } rdm_t* rdm = file->rdm; if (rdm == NULL) return TREXIO_SUCCESS; if (rdm->file != NULL) { fclose(rdm->file); rdm->file = NULL; } if (rdm->one_e != NULL) { FREE (rdm->one_e); } if (rdm->two_e_file_name != NULL) { FREE (rdm->two_e_file_name); } free (rdm); file->rdm = NULL; return TREXIO_SUCCESS; } #+end_src **** Read/Write the one_e attribute The ~one_e~ array is assumed allocated with the appropriate size. #+begin_src c :tangle rdm_text.h trexio_exit_code trexio_text_read_rdm_one_e(const trexio_t* file, double* one_e, const uint64_t dim_one_e); trexio_exit_code trexio_text_write_rdm_one_e(const trexio_t* file, const double* one_e, const uint64_t dim_one_e); #+end_src #+begin_src c :tangle rdm_text.c trexio_exit_code trexio_text_read_rdm_one_e(const trexio_t* file, double* one_e, const uint64_t dim_one_e) { if (file == NULL) return TREXIO_INVALID_ARG_1; if (one_e == NULL) return TREXIO_INVALID_ARG_2; rdm_t* rdm = trexio_text_read_rdm((trexio_text_t*) file); if (rdm == NULL) return TREXIO_FAILURE; if (dim_one_e != rdm->dim_one_e) return TREXIO_INVALID_ARG_3; for (uint64_t i=0 ; ione_e[i]; } return TREXIO_SUCCESS; } trexio_exit_code trexio_text_write_rdm_one_e(const trexio_t* file, const double* one_e, const uint64_t dim_one_e) { if (file == NULL) return TREXIO_INVALID_ARG_1; if (one_e == NULL) return TREXIO_INVALID_ARG_2; if (file->mode != 'r') return TREXIO_READONLY; rdm_t* rdm = trexio_text_read_rdm((trexio_text_t*) file); if (rdm == NULL) return TREXIO_FAILURE; rdm->dim_one_e = dim_one_e; for (uint64_t i=0 ; ione_e[i] = one_e[i]; } rdm->to_flush = 1; return TREXIO_SUCCESS; } #+end_src **** Read/Write the two_e attribute ~two_e~ is a sparse data structure, which can be too large to fit in memory. So we provide functions to read and write it by chunks. In the text back end, the easiest way to do it is to create a file for each sparse float structure. #+begin_src c :tangle rdm_text.h trexio_exit_code trexio_text_buffered_read_rdm_two_e(const trexio_t* file, const uint64_t offset, const uint64_t size, int64_t* index, double* value); trexio_exit_code trexio_text_buffered_write_rdm_two_e(const trexio_t* file, const uint64_t offset, const uint64_t size, const int64_t* index, const double* value); #+end_src #+begin_src c :tangle rdm_text.c trexio_exit_code trexio_text_buffered_read_rdm_two_e(const trexio_t* file, const uint64_t offset, const uint64_t size, int64_t* index, double* value) { if (file == NULL) return TREXIO_INVALID_ARG_1; if (index == NULL) return TREXIO_INVALID_ARG_4; if (value == NULL) return TREXIO_INVALID_ARG_5; rdm_t* rdm = trexio_text_read_rdm((trexio_text_t*) file); if (rdm == NULL) return TREXIO_FAILURE; FILE* f = fopen(rdm->two_e_file_name, "r"); if (f == NULL) return TREXIO_END; const uint64_t line_length = 64; fseek(f, (long) offset * line_length, SEEK_SET); int rc; for (uint64_t i=0 ; imode != 'r') return TREXIO_READONLY; rdm_t* rdm = trexio_text_read_rdm((trexio_text_t*) file); if (rdm == NULL) return TREXIO_FAILURE; FILE* f = fopen(rdm->two_e_file_name, "w"); if (f == NULL) return TREXIO_FAILURE; const uint64_t line_length = 64; fseek(f, (long) offset * line_length, SEEK_SET); int rc; for (uint64_t i=0 ; i