From 8234a2aea4d45c75d73d9cf85f1e13a3fc68ff74 Mon Sep 17 00:00:00 2001 From: Anthony Scemama Date: Wed, 12 Jul 2017 00:11:34 +0200 Subject: [PATCH 1/3] Revert "Add hostname in temporary filename (problem on clusters)" --- Bash/ezfio.sh | 22 +++++------ C/.empty | 0 Makefile | 2 +- bin/archive_ezfio.py | 2 +- configure.py | 7 ++-- src/Makefile | 87 ++++++++++++++++++----------------------- src/f_types.py | 2 +- src/groups.py | 9 ----- src/groups_templates.py | 68 ++++++++++++-------------------- src/libezfio_file.irp.f | 10 ++--- src/libezfio_util.irp.f | 17 +++----- 11 files changed, 91 insertions(+), 135 deletions(-) delete mode 100644 C/.empty diff --git a/Bash/ezfio.sh b/Bash/ezfio.sh index 991688d..9d36360 100644 --- a/Bash/ezfio.sh +++ b/Bash/ezfio.sh @@ -4,7 +4,7 @@ EZFIO_ROOT=$( cd $(dirname "${BASH_SOURCE}")/.. ; pwd -P ) function _ezfio_py() { - python "${EZFIO_ROOT}/Python/ezfio.py" $@ + python ${EZFIO_ROOT}/Python/ezfio.py $@ } @@ -84,11 +84,11 @@ function _ezfio_set_file() { _require_first_argument $@ || return 1 - if [[ ! -d "$1" ]] + if [[ ! -d $1 ]] then - mkdir -p "$1" || return 1 + mkdir -p $1 || return 1 fi - export EZFIO_FILE="$1" + export EZFIO_FILE=$1 _ezfio_info "Set file ${EZFIO_FILE}" return 0 } @@ -106,12 +106,12 @@ function _ezfio_has() _require_ezfio_file || return 1 _require_first_argument $@ || return 1 - if [[ ! -d "${EZFIO_FILE}/${1,,}" ]] + if [[ ! -d ${EZFIO_FILE}/${1,,} ]] then return 1 fi - if [[ -z "$2" ]] + if [[ -z $2 ]] then return 0 fi @@ -123,14 +123,14 @@ function _ezfio_get() { _require_ezfio_file || return 1 - if [[ -z "$1" ]] + if [[ -z $1 ]] then ls ${EZFIO_FILE} && return 0 || return 1 fi - if [[ -z "$2" ]] + if [[ -z $2 ]] then - ls "${EZFIO_FILE}/${1,,}" && return 0 || return 1 + ls ${EZFIO_FILE}/${1,,} && return 0 || return 1 fi _ezfio_py get $@ && return 0 || return 1 @@ -142,11 +142,11 @@ function _ezfio_set() _require_first_argument $@ || return 1 _require_second_argument $@ || return 2 - if [[ -z "$3" ]] + if [[ -z $3 ]] then _ezfio_py set $@ || return 1 else - echo "$3" | _ezfio_py set "$1" "$2" || return 1 + echo $3 | _ezfio_py set $1 $2 || return 1 fi return 0 } diff --git a/C/.empty b/C/.empty deleted file mode 100644 index e69de29..0000000 diff --git a/Makefile b/Makefile index 88b078f..5ac6182 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ include make.config .PHONY: default clean veryclean archive default: make.config - cd src && BUILD_SYSTEM=make $(MAKE) + cd src && $(MAKE) clean: - bash -c "[[ -f lib/libezfio.a ]] && rm $$PWD/lib/*" diff --git a/bin/archive_ezfio.py b/bin/archive_ezfio.py index cb09c9f..82a5140 100755 --- a/bin/archive_ezfio.py +++ b/bin/archive_ezfio.py @@ -37,7 +37,7 @@ def main(): f_name = f[4:] try: exec """d['%s'] = ezfio.%s"""%(f_name,f_name) - except IOError: + except: if do_verbose: print "%-40s [%5s]"%(f_name, "Empty") else: diff --git a/configure.py b/configure.py index 0f86bb3..86e8773 100755 --- a/configure.py +++ b/configure.py @@ -26,7 +26,7 @@ d_default = { "FCFLAGS" : '-O2', "RANLIB" : 'ranlib', "AR" : 'ar', - "BUILD_SYSTEM" : 'make' + "NINJA" : 'ninja', } CONFIG_FILES=' '.join([ os.path.join("config",x) for x in os.listdir('config') if x != '.empty']) @@ -112,7 +112,7 @@ rule build_libezfio_a description = Building libezfio.a rule build_libezfio_irp_a - command = cp lib/libezfio.a lib/libezfio_irp.a ; {RANLIB} lib/libezfio_irp.a + command = cp lib/libezfio.a lib/libezfio_irp.a ; {AR} dv lib/libezfio_irp.a irp_stack.irp.o > /dev/null ; {RANLIB} lib/libezfio_irp.a description = Building libezfio_irp.a rule build_python @@ -141,4 +141,5 @@ build Ocaml/ezfio.ml: build_ocaml | lib/libezfio.a if __name__ == '__main__': - create_build_ninja() + + create_build_ninja() diff --git a/src/Makefile b/src/Makefile index 66cdeb5..91f9ae6 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,71 +1,60 @@ -.EXPORT_ALL_VARIABLES: +# EZFIO is an automatic generator of I/O libraries +# Copyright (C) 2009 Anthony SCEMAMA, CNRS +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Anthony Scemama +# LCPQ - IRSAMC - CNRS +# Universite Paul Sabatier +# 118, route de Narbonne +# 31062 Toulouse Cedex 4 +# scemama@irsamc.ups-tlse.fr + +DEFAULT_TARGET: all + include ../make.config +export -# Dark magic below modify with caution! -# "You are Not Expected to Understand This" -# . -# /^\ . -# /\ "V", -# /__\ I O o -# //..\\ I . -# \].`[/ I -# /l\/j\ (] . O -# /. ~~ ,\/I . -# \\L__j^\/I o -# \/--v} I o . -# | | I _________ -# | | I c(` ')o -# | l I \. ,/ -# _/j L l\_! _//^---^\\_ +include irpf90.make +irpf90.make: $(wildcard *.irp.f) ../config/* + $(IRPF90) -ifeq ($(BUILD_SYSTEM),ninja) - BUILD_FILE=IRPF90_temp/build.ninja - IRPF90FLAGS += -j -else ifeq ($(BUILD_SYSTEM),make) - BUILD_FILE=IRPF90_temp/build.make - BUILD_SYSTEM += -j -else -DUMMY: - $(error 'Wrong BUILD_SYSTEM: $(BUILD_SYSTEM)') -endif - -define run_and_touch - $(BUILD_SYSTEM) -C $(dir $(1) ) -f $(notdir $(1) ) $(addprefix $(CURDIR)/, $(2)) && touch $(2) -endef - -export PYTHONPATH:=$(CURDIR):$(PYTHONPATH) - +include ../version .PHONY: static all all: static ../Python/ezfio.py ../Ocaml/ezfio.ml + static: ../lib/libezfio.a ../lib/libezfio_irp.a ../lib/libezfio.a: IRPF90_temp/irpf90.a rm -f ../lib/libezfio.a - cp IRPF90_temp/irpf90.a ../lib/libezfio.a + $(AR) cru ../lib/libezfio.a $(OBJ1) $(RANLIB) ../lib/libezfio.a -#No more irp_stack ../lib/libezfio_irp.a: ../lib/libezfio.a + rm -f ../lib/libezfio_irp.a cp ../lib/libezfio.a ../lib/libezfio_irp.a + $(AR) dv ../lib/libezfio_irp.a irp_stack.irp.o + $(RANLIB) ../lib/libezfio_irp.a -../Python/ezfio.py: ../lib/libezfio.a +../Python/ezfio.py: python create_python.py -../Ocaml/ezfio.ml: ../lib/libezfio.a +../Ocaml/ezfio.ml: python create_ocaml.py -IRPF90_temp/irpf90.a: $(BUILD_FILE) - $(call run_and_touch, $<, $@) -$(BUILD_FILE): $(shell find . -maxdepth 2 -path ./IRPF90_temp -prune -o -name '*.irp.f' -print) ../config/* - $(IRPF90) $(IRPF90FLAGS) - -clean: - rm -f -- $(BUILD_FILE) $(shell find IRPF90_temp -type f \( -name "*.o" -o -name "*.mod" -name "*.a" \) -delete;) - rm -f ../lib/libezfio.a ../lib/libezfio_irp.a - -veryclean: clean - rm -rf IRPF90_temp/ IRPF90_man/ irpf90_entities dist tags diff --git a/src/f_types.py b/src/f_types.py index b70bfd2..d1c5d43 100644 --- a/src/f_types.py +++ b/src/f_types.py @@ -27,7 +27,7 @@ format= { 'integer' : ["'(I20)'", "%20d"], 'real' : ["'(E24.15)'","%24.15E"], 'double precision': ["'(E24.15)'","%24.15E"], - 'logical' : ["'(L1)'","%c"], + 'logical' : ["'(L)'","%c"], 'character*(*)' : ["'(A)'","%s"] } diff --git a/src/groups.py b/src/groups.py index 27f69c8..83dd175 100644 --- a/src/groups.py +++ b/src/groups.py @@ -40,9 +40,6 @@ print >>file_py, """ version = property(fset=None,fget=get_version) """%(v) -file_c = open("../C/ezfio.h","w") -print >>file_c, c_header - import sys for group in groups.keys(): @@ -80,7 +77,6 @@ for group in groups.keys(): 'group': group, 'dims': strdims} print attributes%d - print >>file_c, attributes_c%d print >>file_py, attributes_py%d else: d = { 'type': type, @@ -100,7 +96,6 @@ for group in groups.keys(): 'dims_py': dims_py, 'dims': dims } print buffered%d - print >>file_c, buffered_c%d print >>file_py, buffered_py%d else: dims_loop = '' @@ -147,9 +142,5 @@ for group in groups.keys(): 'declar_loop' : declar_loop} print attributes_arr%d print >>file_py, attributes_arr_py%d - print >>file_c, attributes_arr_c%d -print >>file_c, c_footer file_py.close() -file_c.close() - diff --git a/src/groups_templates.py b/src/groups_templates.py index 1150088..4e14c24 100644 --- a/src/groups_templates.py +++ b/src/groups_templates.py @@ -239,48 +239,38 @@ end #--------- attributes_c = """ -void ezfio_set_%(group)s_%(var)s(%(ctype)s %(var)s) +void ezfio_set_%(group)s_%(var)s(%(type)s %(var)s) { ezfio_set_%(group)s_%(var)s_(&%(var)s); } - -void ezfio_get_%(group)s_%(var)s(%(ctype)s *%(var)s) -{ - ezfio_get_%(group)s_%(var)s_(%(var)s); -} - -int ezfio_has_%(group)s_%(var)s() -{ - ezfio_has_%(group)s_%(var)s_(); -} """ -attributes_arr_c = """ -void ezfio_set_%(group)s_%(var)s(%(ctype)s %(var)s) -{ - ezfio_set_%(group)s_%(var)s_(&%(var)s); -} - -void ezfio_get_%(group)s_%(var)s(%(ctype)s *%(var)s) -{ - ezfio_get_%(group)s_%(var)s_(%(var)s); -} - -int ezfio_has_%(group)s_%(var)s() -{ - ezfio_has_%(group)s_%(var)s_(); -} -""" - -calculated_c = attributes_c - -c_header = """#ifndef EZFIO_H\n#define EZFIO_H - -#include +c_header = """ +#ifndef EZFIO_H +#define EZFIO_H #ifdef __cplusplus extern "C" { #endif +""" + +c_footer = """ +#ifdef __cplusplus +} +#endif +#endif +""" + + +file_h = [c_header] +file_h +=[ """ +void ezfio_set_file_(char* filename, long filename_size); + +void ezfio_read_buffer_(int* indices,double* values, int* isize); +void ezfio_write_buffer_(int* indices,double* values, int* isize); +"""] + +file_c = [ """ typedef struct { @@ -292,7 +282,7 @@ void ezfio_set_file(char* filename) { long len; len = strlen(filename); - ezfio_set_file_(filename,len); + ezfio_set_file_(filename,mlen); } sparse ezfio_read_buffer(int isize) @@ -307,15 +297,7 @@ void ezfio_write_buffer(sparse data, int isize) ezfio_write_buffer_(data.indices,data.values,&isize); } -""" - -c_footer = """ -#ifdef __cplusplus -} -#endif -#endif -""" - +""" ] diff --git a/src/libezfio_file.irp.f b/src/libezfio_file.irp.f index 9334a2a..860eb8e 100644 --- a/src/libezfio_file.irp.f +++ b/src/libezfio_file.irp.f @@ -128,16 +128,16 @@ logical function exists(path) character*(*) :: path character*(32) :: V inquire(file=trim(path)//'/.version',exist=exists) -! if (exists) then -! open(libezfio_iunit,file=trim(path)//'/.version') -! read(libezfio_iunit,*) V -! close(libezfio_iunit) + if (exists) then + open(libezfio_iunit,file=trim(path)//'/.version') + read(libezfio_iunit,*) V + close(libezfio_iunit) ! integer :: char_to_version ! if (char_to_version(V) > char_to_version(libezfio_version)) then ! call ezfio_error(irp_here, 'This file was generated with version '//trim(V)//& ! ' but the current installed version is '//trim(libezfio_version)//'') ! endif -! endif + endif end function subroutine ezfio_set_file(filename_in) diff --git a/src/libezfio_util.irp.f b/src/libezfio_util.irp.f index 1ab88b4..3d6a9b0 100644 --- a/src/libezfio_util.irp.f +++ b/src/libezfio_util.irp.f @@ -23,21 +23,14 @@ ! scemama@irsamc.ups-tlse.fr BEGIN_PROVIDER [ integer, PID ] -&BEGIN_PROVIDER [ character*(256), PID_str ] +&BEGIN_PROVIDER [ character*(8), PID_str ] implicit none BEGIN_DOC ! Current process ID END_DOC integer :: getpid - character*(240) :: hostname PID = getpid() write(PID_str,'(I8.8)') PID - call HOSTNM(hostname) - write(PID_str,'(A,''.'',I8.8,X)') trim(hostname), PID - PID_str = trim(PID_str) - print *, PID_str - call sleep(1) - END_PROVIDER logical function ezfio_exists(path) @@ -82,7 +75,7 @@ subroutine libezfio_openz(filename,mode,err) character*(*) :: filename, mode character*(1024) :: fifo integer :: err - fifo = trim(filename)//'.'//trim(PID_str) + fifo = trim(filename)//'.'//PID_str err=1 if (mode(1:1) == 'r') then @@ -104,7 +97,7 @@ subroutine libezfio_closez(filename,mode) END_DOC character*(*) :: filename, mode character*(1024) :: fifo - fifo = trim(filename)//'.'//trim(PID_str) + fifo = trim(filename)//'.'//PID_str if (mode(1:1) == 'w') then close(unit=libezfio_iunit) call system('gzip -c < '//trim(fifo)//' > '//trim(filename)) @@ -148,7 +141,7 @@ subroutine ezfio_write_%(type_short)s(dir,fil,dat) if (libezfio_read_only) then call ezfio_error(irp_here,'Read-only file.') endif - l_filename(1)=trim(dir)//'/.'//fil//'.'//trim(PID_str) + l_filename(1)=trim(dir)//'/.'//fil//'.'//PID_str l_filename(2)=trim(dir)//'/'//fil open(unit=libezfio_iunit,file=l_filename(1),form='FORMATTED',action='WRITE') write(libezfio_iunit,%(fmt)s) dat @@ -223,7 +216,7 @@ subroutine ezfio_write_array_%(type_short)s(dir,fil,rank,dims,dim_max,dat) if (libezfio_read_only) then call ezfio_error(irp_here,'Read-only file.') endif - l_filename(1)=trim(dir)//'/.'//fil//trim(PID_str)//'.gz' + l_filename(1)=trim(dir)//'/.'//fil//PID_str//'.gz' l_filename(2)=trim(dir)//'/'//fil//'.gz' err = 0 From c826c47c13abd188fd73843bfdafd576fd55b0bc Mon Sep 17 00:00:00 2001 From: Anthony Scemama Date: Wed, 12 Jul 2017 00:18:23 +0200 Subject: [PATCH 2/3] Add hostname in temporary filename Problem on KNL clusters --- src/libezfio_util.irp.f | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libezfio_util.irp.f b/src/libezfio_util.irp.f index 3d6a9b0..a23ca35 100644 --- a/src/libezfio_util.irp.f +++ b/src/libezfio_util.irp.f @@ -29,8 +29,12 @@ ! Current process ID END_DOC integer :: getpid + character*(240) :: hostname PID = getpid() write(PID_str,'(I8.8)') PID + call HOSTNM(hostname) + write(PID_str,'(A,''.'',I8.8,X)') trim(hostname), PID + PID_str = trim(PID_str) END_PROVIDER logical function ezfio_exists(path) @@ -75,7 +79,7 @@ subroutine libezfio_openz(filename,mode,err) character*(*) :: filename, mode character*(1024) :: fifo integer :: err - fifo = trim(filename)//'.'//PID_str + fifo = trim(filename)//'.'//trim(PID_str) err=1 if (mode(1:1) == 'r') then @@ -97,7 +101,7 @@ subroutine libezfio_closez(filename,mode) END_DOC character*(*) :: filename, mode character*(1024) :: fifo - fifo = trim(filename)//'.'//PID_str + fifo = trim(filename)//'.'//trim(PID_str) if (mode(1:1) == 'w') then close(unit=libezfio_iunit) call system('gzip -c < '//trim(fifo)//' > '//trim(filename)) @@ -141,7 +145,7 @@ subroutine ezfio_write_%(type_short)s(dir,fil,dat) if (libezfio_read_only) then call ezfio_error(irp_here,'Read-only file.') endif - l_filename(1)=trim(dir)//'/.'//fil//'.'//PID_str + l_filename(1)=trim(dir)//'/.'//fil//'.'//trim(PID_str) l_filename(2)=trim(dir)//'/'//fil open(unit=libezfio_iunit,file=l_filename(1),form='FORMATTED',action='WRITE') write(libezfio_iunit,%(fmt)s) dat @@ -216,7 +220,7 @@ subroutine ezfio_write_array_%(type_short)s(dir,fil,rank,dims,dim_max,dat) if (libezfio_read_only) then call ezfio_error(irp_here,'Read-only file.') endif - l_filename(1)=trim(dir)//'/.'//fil//PID_str//'.gz' + l_filename(1)=trim(dir)//'/.'//fil//trim(PID_str)//'.gz' l_filename(2)=trim(dir)//'/'//fil//'.gz' err = 0 From fc447ed41e7502ac4fdd8bc3fb5fe757c626ac51 Mon Sep 17 00:00:00 2001 From: Anthony Scemama Date: Wed, 12 Jul 2017 01:49:13 +0200 Subject: [PATCH 3/3] 8 - 256 --- src/libezfio_util.irp.f | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libezfio_util.irp.f b/src/libezfio_util.irp.f index a23ca35..3a3a573 100644 --- a/src/libezfio_util.irp.f +++ b/src/libezfio_util.irp.f @@ -23,7 +23,7 @@ ! scemama@irsamc.ups-tlse.fr BEGIN_PROVIDER [ integer, PID ] -&BEGIN_PROVIDER [ character*(8), PID_str ] +&BEGIN_PROVIDER [ character*(256), PID_str ] implicit none BEGIN_DOC ! Current process ID