diff --git a/example/Makefile b/example/Makefile index c026e2a..08bfe3a 100644 --- a/example/Makefile +++ b/example/Makefile @@ -1,21 +1,27 @@ -IRPF90= irpf90 -IRPF90FLAGS= -I ./ -I input/ -BUILD_SYSTEM= make +# make | ninja +############## + +BUILD_SYSTEM= ninja + +####### + +IRPF90= ../bin/irpf90 +IRPF90FLAGS= -I input --codelet=v:100 .EXPORT_ALL_VARIABLES: -AR = ar -CC = gcc -CFLAGS = -O2 -CXX = g++ -CXXFLAGS = -O2 -FC = gfortran -FCFLAGS = -O2 LIB = -RANLIB = ranlib - -OBJ = SRC = +OBJ = + +# Compiler ! Will be overwriten by the ENV one if avalaible. +FC =ifort +FCFLAGS ?= -O2 + +CC ?= gcc +CFLAGS ?= -O2 +CXX ?= g++ +CXXFLAGS ?= -O2 # Dark magic below modify with caution! # "You are Not Expected to Understand This" @@ -32,38 +38,57 @@ SRC = # | | I _________ # | | I c(` ')o # | l I \. ,/ -# _/j L l\_! _//^---^\\_ +# _/j L l\_! _// \\_ +#Misc +AR ?= +RANLIB ?= -ifeq ($(BUILD_SYSTEM),ninja) - BUILD_FILE=IRPF90_temp/build.ninja - IRPF90FLAGS += -j -else ifeq ($(BUILD_SYSTEM),make) - BUILD_FILE=IRPF90_temp/build.make - BUILD_SYSTEM += -j +# Variable need by IRPF90 +BUILD_SYSTEM_stripped=$(strip $(BUILD_SYSTEM)) +ifeq ($(BUILD_SYSTEM_stripped),ninja) + BUILD_FILE=IRPF90_temp/build.ninja + IRPF90FLAGS += -j +else ifeq ($(BUILD_SYSTEM_stripped),make) + BUILD_FILE=IRPF90_temp/build.make + BUILD_SYSTEM += -j else DUMMY: $(error 'Wrong BUILD_SYSTEM: $(BUILD_SYSTEM)') endif -define run_and_touch - $(BUILD_SYSTEM) -C $(dir $(1) ) -f $(notdir $(1) ) $(addprefix $(CURDIR)/, $(2)) && touch $(2) +# Actual Rule +EXE := $(shell egrep -ri '^\s*program' *.irp.f | cut -d'.' -f1) +ARCH = $(addprefix $(CURDIR)/,IRPF90_temp/irpf90.a) + +.PHONY: clean all + +all: $(EXE) + +define run + $(BUILD_SYSTEM_stripped) -C $(dir $(BUILD_FILE) ) -f $(notdir $(BUILD_FILE) ) $(1) endef -EXE := $(shell egrep -r '^\s*program' *.irp.f | awk '{print $$2}') +#We allow for the user to ask for 'relative' path +#But the IRPF90 build system use absolute path. +$(EXE): $(ARCH) + @printf "%b" "\033[0;32m Build $@...\033[m\n" + @$(call run, $(addprefix $(CURDIR)/, $@)) && touch $@ -.PHONY: all -all: $(BUILD_FILE) - $(call run_and_touch, $<, $(EXE)) +.NOTPARALLEL: $(BUILD_FILE) $(ARCH) + +$(ARCH): $(BUILD_FILE) + @printf "%b" "\033[0;32m Creating the archive...\033[m\n" + @$(call run, $@) && touch $@ -.NOTPARALLEL: $(EXE) -$(EXE): $(BUILD_FILE) - $(call run_and_touch, $<, $(EXE)) $(BUILD_FILE): $(shell find . -maxdepth 2 -path ./IRPF90_temp -prune -o -name '*.irp.f' -print) - $(IRPF90) $(IRPF90FLAGS) + @printf "%b" "\033[0;32m Running the IRPF90-compiler...\033[m\n" + @$(IRPF90) $(IRPF90FLAGS) clean: - rm -f -- $(BUILD_FILE) $(EXE) $(shell find IRPF90_temp -type f \( -name "*.o" -o -name "*.mod" -name "*.a" \) -delete;) + rm -f -- $(BUILD_FILE) $(EXE) + veryclean: clean rm -rf IRPF90_temp/ IRPF90_man/ irpf90_entities dist tags + diff --git a/example/irp_example2.irp.f b/example/irp_example2.irp.f index 2d0d23b..a5299b6 100644 --- a/example/irp_example2.irp.f +++ b/example/irp_example2.irp.f @@ -1,6 +1,6 @@ program irp_example2 print *, "Example 2" - print *, 't = ', t + print *, 'v = ', v IRP_WRITE t diff --git a/example/uvwt.irp.f b/example/uvwt.irp.f index 082f229..365a577 100644 --- a/example/uvwt.irp.f +++ b/example/uvwt.irp.f @@ -1,28 +1,31 @@ BEGIN_PROVIDER [integer, t ] + IMPLICIT NONE t = u1+v+4 END_PROVIDER BEGIN_PROVIDER [integer,w] + IMPLICIT NONE w = d5+3 END_PROVIDER BEGIN_PROVIDER [ integer, v ] - implicit none + IMPLICIT NONE v = u2+w+2 END_PROVIDER BEGIN_PROVIDER [ integer, u1 ] + IMPLICIT NONE integer :: fu u1 = fu(d1,d2) END_PROVIDER BEGIN_PROVIDER [ integer, u2 ] + IMPLICIT NONE integer :: fu u2 = fu(d3,d4) - !df END_PROVIDER -integer function fu(x,y) - integer :: x,y +INTEGER function fu(x,y) + INTEGER, INTENT(in) :: x,y fu = x+y+1 -end function +END FUNCTION diff --git a/src/build_file.py b/src/build_file.py index b3aad23..25949b5 100644 --- a/src/build_file.py +++ b/src/build_file.py @@ -36,6 +36,7 @@ irp_id = irpf90_t.irp_id cwd = os.getcwd() + def dress(f, in_root=False): #(str,bool) -> str """ Transfoms the filename into $PWD/IRPF90_temp/f @@ -65,9 +66,7 @@ def create_build_touches(l_irp_m, ninja): result_ninja = '\n'.join([ "build {target_o}: compile_fortran_{irp_id} {target_F90} | {list_of_modules_irp}", - " short_in = {short_target_F90}", - " short_out = {short_target_o}", - "" + " short_in = {short_target_F90}", " short_out = {short_target_o}", "" ]) result_make = '\n'.join([ @@ -96,15 +95,12 @@ def create_build_archive(l_irp_o, l_usr_o_wo_main, l_ext_o, l_irp_sup_o, ninja=T list_of_object = ' '.join(l_irp_o + l_usr_o_wo_main + l_ext_o + l_irp_sup_o) - result_ninja = '\n'.join([ - "build {lib}: archive_{irp_id} {list_of_object}", - " short_out = {short_lib}", - ""]) + result_ninja = '\n'.join( + ["build {lib}: archive_{irp_id} {list_of_object}", " short_out = {short_lib}", ""]) result_make = '\n'.join([ - "{lib}: {list_of_object}", - '\t@printf "Archive: {short_lib}\\n"', - "\t@$(AR) cr $@ $^", ""]) + "{lib}: {list_of_object}", '\t@printf "Archive: {short_lib}\\n"', "\t@$(AR) cr $@ $^", "" + ]) result = result_ninja if ninja else result_make return result.format(**locals()) @@ -125,7 +121,8 @@ def create_build_link(t, l_irp_m, l_usr_m, l_ext_m, ninja=True): basename = os.path.basename(filename) if basename != progname: from util import logger - logger.info('program-name `{0}` != file-name `{1}` (using file-name for now...)'.format(progname,basename)) + logger.info('program-name `{0}` != file-name `{1}` (using file-name for now...)'.format( + progname, basename)) target = dress(filename, in_root=True) short_target = filename @@ -138,14 +135,13 @@ def create_build_link(t, l_irp_m, l_usr_m, l_ext_m, ninja=True): result_ninja = '\n'.join([ "build {target}: link_{irp_id} {target_o} {irp_lib} | {list_of_module}", - " short_out = {short_target}", - ""]) + " short_out = {short_target}", "" + ]) result_make = '\n'.join([ - "{target}:{target_o} {irp_lib} | {list_of_module}", - '\t@printf "Link: {short_target}\\n"', - "\t@$(FC) $^ $(LIB) -o $@", - ""]) + "{target}:{target_o} {irp_lib} | {list_of_module}", '\t@printf "Link: {short_target}\\n"', + "\t@$(FC) $^ $(LIB) -o $@", "" + ]) result = result_ninja if ninja else result_make @@ -186,10 +182,10 @@ def create_build_compile(t, l_module, l_ext_modfile=[], ninja=True): # Expensive and stupid. We can create a dict to do the loockup only once for m in t.needed_modules_usr: - # m is name - for x in l_module: - if m in x.gen_mod and x.filename != t.filename: - needed_modules.append("%s.irp.o" % x.filename) + # m is name + for x in l_module: + if m in x.gen_mod and x.filename != t.filename: + needed_modules.append("%s.irp.o" % x.filename) from util import uniquify needed_modules = uniquify(needed_modules) @@ -206,7 +202,7 @@ def create_build_compile(t, l_module, l_ext_modfile=[], ninja=True): target_module_F90 = dress(short_target_module_F90) needed_modules_irp += [target_module_o] - list_of_modules = ' '.join(map(dress, needed_modules)) + list_of_modules = ' '.join(map(dress, needed_modules)) list_of_modules_irp = ' '.join(map(dress, needed_modules_irp)) inline_include = True @@ -217,27 +213,23 @@ def create_build_compile(t, l_module, l_ext_modfile=[], ninja=True): else: #The include have already by included list_of_includes = ' ' - + l_build = [ "build {target_o}: compile_fortran_{irp_id} {target_F90} | {list_of_includes} {list_of_modules} {list_of_modules_irp}", - " short_in = {short_target_F90}", - " short_out = {short_target}", - "" + " short_in = {short_target_F90}", " short_out = {short_target}", "" ] l_build_make = [ "{target_o}: {target_F90} | {list_of_includes} {list_of_modules} {list_of_modules_irp}", - '\t@printf "F: {short_target_F90} -> {short_target}\\n"', - "\t@$(FC) $(FCFLAGS) -c $^ -o $@", "" + '\t@printf "F: {short_target_F90} -> {short_target}\\n"', "\t@$(FC) $(FCFLAGS) -c $^ -o $@", + "" ] # No need of module when compiling the irp_module. if t.has_irp_module: l_build += [ "build {target_module_o}: compile_fortran_{irp_id} {target_module_F90} | {list_of_includes} {list_of_modules} ", - " short_in = {short_target_module_F90}", - " short_out = {short_target_module_o}", - "" + " short_in = {short_target_module_F90}", " short_out = {short_target_module_o}", "" ] l_build_make += [ @@ -250,7 +242,7 @@ def create_build_compile(t, l_module, l_ext_modfile=[], ninja=True): return '\n'.join(l_cur).format(**locals()) -def create_build_remaining(f,ninja): +def create_build_remaining(f, ninja): """ Create the build command for the remaining file f. f is a file name (str). """ @@ -271,108 +263,55 @@ def create_build_remaining(f,ninja): if extension.lower() in ['f', 'f90']: result = ["build {target_o}: compile_fortran_{irp_id} {target_i}"] + result_make = [ + '{target_o}: {target_i}', '\t@printf "F: {short_target_o} -> {short_target_i}\\n"', + "\t@$(FC) $(FCFLAGS) -c $^ -o $@", "" + ] + elif extension.lower() in ['c']: result = ["build {target_o}: compile_c_{irp_id} {target_i}"] elif extension.lower() in ['cxx', 'cpp']: result = ["build {target_o}: compile_cxx_{irp_id} {target_i}"] result += [" short_in = {short_target_i}", " short_out = {short_target_o}", ""] - return '\n'.join(result).format(**locals()) + + result_final = result if ninja else result_make + + return '\n'.join(result_final).format(**locals()) -def create_makefile(d_flags,d_var,irpf90_flags,ninja=True): +def create_makefile(d_flags, d_var, irpf90_flags, ninja=True): - result = ["IRPF90= irpf90", - "IRPF90FLAGS= %s" % irpf90_flags, - "BUILD_SYSTEM= %s" % ('ninja' if ninja else 'make'), - ""] - - # Export all the env variable used by irpf90 - result += ['.EXPORT_ALL_VARIABLES:', - '', - '\n'.join("{0} = {1}".format(k, v) for k, v in sorted(d_flags.iteritems())), - '', - '\n'.join("{0} = {1}".format(k, ' '.join(v)) for k, v in sorted(d_var.iteritems())), - ''] - - result += [ r'# Dark magic below modify with caution!', - r'# "You are Not Expected to Understand This"', - r"# .", - r"# /^\ .", - r'# /\ "V",', - r"# /__\ I O o", - r"# //..\\ I .", - r"# \].`[/ I", - r"# /l\/j\ (] . O", - r"# /. ~~ ,\/I .", - r"# \\L__j^\/I o", - r"# \/--v} I o .", - r"# | | I _________", - r"# | | I c(` ')o", - r"# | l I \. ,/", - r"# _/j L l\_! _//^---^\\_", - r""] - - result += ["", - "ifeq ($(BUILD_SYSTEM),ninja)", - "\tBUILD_FILE=IRPF90_temp/build.ninja", - "\tIRPF90FLAGS += -j", - "else ifeq ($(BUILD_SYSTEM),make)", - "\tBUILD_FILE=IRPF90_temp/build.make", - "else", - "DUMMY:", - "\t$(error 'Wrong BUILD_SYSTEM: $(BUILD_SYSTEM)')", - "endif"] - - result += ["", - "define run_and_touch", - " $(BUILD_SYSTEM) -C $(dir $(1) ) -f $(notdir $(1) ) $(addprefix $(CURDIR)/, $(2)) && touch $(2)", - "endef", - "", - "EXE := $(shell egrep -ri '^\s*program' *.irp.f | cut -d'.' -f1)", - "", - ".PHONY: all", - "", - "all: $(BUILD_FILE)", - "\t$(call run_and_touch, $<, $(EXE))", - "", - ".NOTPARALLEL: $(EXE)", - "$(EXE): $(BUILD_FILE)", - "\t$(call run_and_touch, $<, $(EXE))", - - "$(BUILD_FILE): $(shell find . -maxdepth 2 -path ./IRPF90_temp -prune -o -name '*.irp.f' -print)", - "\t$(IRPF90) $(IRPF90FLAGS)", - "", - "clean:", - '\trm -f -- $(BUILD_FILE) $(EXE)' - '\t$(shell find IRPF90_temp -type f \\( -name "*.o" -o -name "*.mod" -name "*.a" \\) -delete;)', - "veryclean: clean", - "\trm -rf IRPF90_temp/ IRPF90_man/ irpf90_entities dist tags"] + d = {'BUILD_SYSTEM': 'ninja' if ninja else 'make', + 'irpf90_flags': irpf90_flags} + d.update(d_flags) + d.update(d_var) + import util - data = '%s\n' % '\n'.join(result) - util.lazy_write_file('Makefile',data,conservative=True) + str_ = util.ashes_env.render('general.make', d) + util.lazy_write_file('Makefile', str_, conservative=True) + def create_make_all_clean(l_main): - # - '''Create the ALL and CLEAN target of Makefile + # + '''Create the ALL and CLEAN target of Makefile Note: Portability doesn't mater. -delete is maybe not posix but -exec rm {} + is far more ugly! ''' - l_executable =' '.join(dress( t.filename, in_root=True) for t in l_main) + l_executable = ' '.join(dress(t.filename, in_root=True) for t in l_main) - output = [".PHONY : all", - "all: {l_executable}", - "", - ".PHONY: clean", - "clean:", - '\tfind . -type f \( -name "*.o" -o -name "*.mod" \) -delete; rm -f {l_executable} --' - ""] + output = [ + ".PHONY : all", "all: {l_executable}", "", ".PHONY: clean", "clean:", + '\tfind . -type f \( -name "*.o" -o -name "*.mod" \) -delete; rm -f {l_executable} --' + "" + ] + + return ['\n'.join(output).format(**locals())] - return [ '\n'.join(output).format(**locals())] def create_var_and_rule(d_flags, ninja): @@ -383,59 +322,49 @@ def create_var_and_rule(d_flags, ninja): # Rules t = [ - "rule compile_fortran_{irp_id}", - " command = $FC $FCFLAGS -c $in -o $out", - " description = F : $short_in -> $short_out", - "", - "rule compile_c_{irp_id}", + "rule compile_fortran_{irp_id}", " command = $FC $FCFLAGS -c $in -o $out", + " description = F : $short_in -> $short_out", "", "rule compile_c_{irp_id}", " command = $CC $CFLAGS -c $in -o $out", - " description = C : $short_in -> $short_out", - "", - "rule compile_cxx_{irp_id}", + " description = C : $short_in -> $short_out", "", "rule compile_cxx_{irp_id}", " command = $CXX $CXXFLAGS -c $in -o $out", - " description = C++ : $short_in -> $short_out", - "", - "rule archive_{irp_id}", - " command = $AR cr $out $in", - " description = Archive: $short_out", - "", - "rule link_{irp_id}", - " command = $FC $FCFLAGS $in $LIB -o $out", - " description = Link: $short_out", - "" + " description = C++ : $short_in -> $short_out", "", "rule archive_{irp_id}", + " command = $AR cr $out $in", " description = Archive: $short_out", "", + "rule link_{irp_id}", " command = $FC $FCFLAGS $in $LIB -o $out", + " description = Link: $short_out", "" ] output += ['\n'.join(t).format(irp_id=irpf90_t.irp_id, **d_flags)] return output - # Environment variables d_default = { - "FC": "gfortran", - "FCFLAGS": "-O2", - "AR": "ar", - "RANLIB": " ranlib", - "CC": "gcc", - "CFLAGS": "-O2", - "CXX": "g++", - "CXXFLAGS": "-O2", - "LIB": ""} + "FC": "gfortran", + "FCFLAGS": "-O2", + "AR": "ar", + "RANLIB": " ranlib", + "CC": "gcc", + "CFLAGS": "-O2", + "CXX": "g++", + "CXXFLAGS": "-O2", + "LIB": "" +} d_flags = dict() for k, v in d_default.iteritems(): - d_flags[k] = os.environ[k] if k in os.environ else v + d_flags[k] = os.environ[k] if k in os.environ else v include_dir = ' ' + ' '.join(["-I %s" % (i) for i in command_line.include_dir]) d_var = dict() for k in ['SRC', 'OBJ']: - d_var[k] = os.environ[k].split() if k in os.environ else [] + d_var[k] = os.environ[k].split() if k in os.environ else [] def create_generalmakefile(ninja): - create_makefile(d_flags,d_var, include_dir,ninja) + create_makefile(d_flags, d_var, include_dir, ninja) + def run(d_module, ninja): #(Dict[str,Module],bool) -> str @@ -476,7 +405,7 @@ def run(d_module, ninja): l_irp_sup_o = ["irp_touches.irp.o"] l_irp_sup_s = ["irp_touches.irp.F90"] - if command_line.do_assert: + if command_line.do_debug or command_line.do_assert: l_irp_sup_o += ["irp_stack.irp.o"] l_irp_sup_s += ["irp_stack.irp.F90"] @@ -484,9 +413,13 @@ def run(d_module, ninja): l_irp_sup_o += ["irp_locks.irp.o"] l_irp_sup_s += ["irp_locks.irp.F90"] + if command_line.do_profile or command_line.do_codelet: + l_irp_sup_o += ["irp_rdtsc.o"] + l_irp_sup_s += ["irp_rdtsc.c"] + if command_line.do_profile: - l_irp_sup_o += ["irp_profile.irp.o", "irp_rdtsc.o"] - l_irp_sup_s += ["irp_profile.irp.F90", "irp_rdtsc.c"] + l_irp_sup_o += ["irp_profile.irp.o"] + l_irp_sup_s += ["irp_profile.irp.F90"] l_irp_sup_o = map(dress, l_irp_sup_o) l_irp_sup_s = map(dress, l_irp_sup_s) @@ -515,7 +448,7 @@ def run(d_module, ninja): for m in l_mod: output.append(create_build_compile(m, l_mod, l_ext_m, ninja)) - output.append(create_build_touches(l_irp_m,ninja)) + output.append(create_build_touches(l_irp_m, ninja)) # All the objects. Kind of, only need usr without main for the static library output.append(create_build_archive(l_irp_o, l_usr_o_wo_main, l_ext_o, l_irp_sup_o, ninja)) @@ -524,13 +457,13 @@ def run(d_module, ninja): output.append(create_build_link(i, l_irp_m, l_usr_m, l_ext_m, ninja)) # Remaining files - for i in l_irp_sup_s[1:]+l_ext_s: + for i in l_irp_sup_s[1:] + l_ext_s: output.append(create_build_remaining(i, ninja)) filename = os.path.join(irpdir, "build.ninja" if ninja else "build.make") data = '%s\n' % '\n\n'.join(output) import util - util.lazy_write_file(filename,data,touch=True) + util.lazy_write_file(filename, data, touch=True) return diff --git a/src/codelet.py b/src/codelet.py index d5e6f25..2007c28 100644 --- a/src/codelet.py +++ b/src/codelet.py @@ -27,8 +27,9 @@ from command_line import command_line import irpf90_t + def run(): - template = """ + template = """ program codelet_%(name)s implicit none integer :: i @@ -58,12 +59,11 @@ end """ - name, NMAX, precondition, filename = command_line.codelet - if precondition is None: - precondition = "" - else: - precondition = "PROVIDE "+precondition - - from util import lazy_write_file - lazy_write_file(filename,template%locals()) + name, NMAX, precondition, filename = command_line.codelet + if precondition is None: + precondition = "" + else: + precondition = "PROVIDE " + precondition + from util import lazy_write_file + lazy_write_file(filename, template % locals()) diff --git a/src/command_line.py b/src/command_line.py index e8e15bc..5a1c3ee 100644 --- a/src/command_line.py +++ b/src/command_line.py @@ -24,11 +24,7 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr -try: - import irpy -except: - import lib_irpy as irpy - +from lib.manager import irpy import getopt, sys from version import version @@ -36,124 +32,150 @@ import re description = "IRPF90 Fortran preprocessor." options = {} -options['a'] = [ 'assert' , 'Activates ASSERT statements. If absent, remove ASSERT statements.', 0 ] -options['c'] = [ 'codelet' , 'entity:NMAX or entity:precondition:NMAX : Generate a codelet to profile a provider running NMAX times', 1 ] -options['C'] = [ 'coarray' , 'All providers are coarrays', 0 ] -options['d'] = [ 'debug' , 'Activates debug. The name of the current subroutine/function/provider will be printed on the standard output when entering or exiting a routine, as well as the CPU time passed inside the routine.', 0 ] -options['D'] = [ 'define' , 'Defines a variable identified by the IRP_IF statements.', 1 ] -options['g'] = [ 'profile' , 'Activates profiling of the code.', 0 ] -options['h'] = [ 'help' , 'Print this help', 0 ] -options['I'] = [ 'include' , 'Include directory', 1 ] -options['j'] = [ 'ninja' , 'Use Ninja instead of make', 0 ] -options['i'] = [ 'init' , 'Initialize current directory. Creates a default Makefile and the temporary working directories.', 0 ] -options['l'] = [ 'align' , 'Align arrays using compiler directives and sets the $IRP_ALIGN variable. For example, --align=32 aligns all arrays on a 32 byte boundary.', 1 ] -options['m'] = [ 'memory' , 'Print memory allocations/deallocations.', 0 ] -options['n'] = [ 'inline' , ' : Force inlining of providers or builders', 1 ] -options['o'] = [ 'checkopt' , 'Shows where optimization may be required', 0 ] -options['p'] = [ 'preprocess' , 'Prints a preprocessed file to standard output. Useful for debugging files containing shell scripts.', 1 ] -options['r'] = [ 'no_directives', 'Ignore all compiler directives !DEC$ and !DIR$', 0 ] -options['s'] = [ 'substitute' , 'Substitute values in do loops for generating specific optimized code.', 1 ] -options['t'] = [ 'touch' , 'Display which entities are touched when touching the variable given as an argument.', 1 ] -options['v'] = [ 'version' , 'Prints version of irpf90', 0 ] -options['w'] = [ 'warnings' , 'Activate Warnings', 0 ] -options['z'] = [ 'openmp' , 'Activate for OpenMP code', 0 ] -options['G'] = [ 'graph' , 'Print the dependecy-graph of the entities (dots format)', 0 ] +options['a'] = ['assert', 'Activates ASSERT statements. If absent, remove ASSERT statements.', 0] +options['c'] = [ + 'codelet', + 'entity:NMAX or entity:precondition:NMAX : Generate a codelet to profile a provider running NMAX times', + 1 +] +options['C'] = ['coarray', 'All providers are coarrays', 0] +options['d'] = [ + 'debug', + 'Activates debug. The name of the current subroutine/function/provider will be printed on the standard output when entering or exiting a routine, as well as the CPU time passed inside the routine.', + 0 +] +options['D'] = ['define', 'Defines a variable identified by the IRP_IF statements.', 1] +options['g'] = ['profile', 'Activates profiling of the code.', 0] +options['h'] = ['help', 'Print this help', 0] +options['I'] = ['include', 'Include directory', 1] +options['j'] = ['ninja', 'Use Ninja instead of make', 0] +options['i'] = [ + 'init', + 'Initialize current directory. Creates a default Makefile and the temporary working directories.', + 0 +] +options['l'] = [ + 'align', + 'Align arrays using compiler directives and sets the $IRP_ALIGN variable. For example, --align=32 aligns all arrays on a 32 byte boundary.', + 1 +] +options['m'] = ['memory', 'Print memory allocations/deallocations.', 0] +options['n'] = ['inline', ' : Force inlining of providers or builders', 1] +options['o'] = ['checkopt', 'Shows where optimization may be required', 0] +options['p'] = [ + 'preprocess', + 'Prints a preprocessed file to standard output. Useful for debugging files containing shell scripts.', + 1 +] +options['r'] = ['no_directives', 'Ignore all compiler directives !DEC$ and !DIR$', 0] +options['s'] = [ + 'substitute', 'Substitute values in do loops for generating specific optimized code.', 1 +] +options['t'] = [ + 'touch', 'Display which entities are touched when touching the variable given as an argument.', + 1 +] +options['v'] = ['version', 'Prints version of irpf90', 0] +options['w'] = ['warnings', 'Activate Warnings', 0] +options['z'] = ['openmp', 'Activate for OpenMP code', 0] +options['G'] = ['graph', 'Print the dependecy-graph of the entities (dots format)', 0] +options['T'] = ['Task', 'Auto-parallelism ', 0] + class CommandLine(object): + def __init__(self): + global options + self._opts = None + self.argv = list(sys.argv) + self.executable_name = self.argv[0] - def __init__(self): - global options - self._opts = None - self.argv = list(sys.argv) - self.executable_name = self.argv[0] + @irpy.lazy_property + def defined(self): + return [a for o, a in self.opts if o in ["-D", '--' + options['D'][0]]] - @irpy.lazy_property - def defined(self): - return [ a for o,a in self.opts if o in [ "-D", '--'+options['D'][0] ] ] + @irpy.lazy_property + def graph(self): + return next((a.split() for o, a in self.opts if o in ["-G", '--' + options['G'][0]]), []) - @irpy.lazy_property - def graph(self): - return next((a.split() for o,a in self.opts if o in ["-G", '--'+options['G'][0] ]),[]) - - @irpy.lazy_property - def include_dir(self): - self._include_dir = [] - for o,a in self.opts: - if o in [ "-I", '--'+options['I'][0] ]: - if len(a) < 1: - print "Error: -I option needs a directory" - if a[-1] != '/': - a = a+'/' - self._include_dir.append(a) - return self._include_dir - - @irpy.lazy_property - def inline(self): - return next( (a for o,a in self.opts if o in [ "-n", '--'+options['n'][0] ]),'') + @irpy.lazy_property + def include_dir(self): + l = [] + for o, a in self.opts: + if o in ["-I", '--' + options['I'][0]]: + if len(a) < 1: + print "Error: -I option needs a directory" + if a[-1] != '/': + a = a + '/' + l.append(a) + return l - @irpy.lazy_property - def substituted(self): - self._substituted = {} - for o,a in self.opts: - if o in [ "-s", '--'+options['s'][0] ]: - k, v = a.split(':') - v_re = re.compile(r"(\W)(%s)(\W.*$|$)"%k.strip()) - self._substituted[k] = [v, v_re] - return self._substituted + @irpy.lazy_property + def inline(self): + return next((a for o, a in self.opts if o in ["-n", '--' + options['n'][0]]), '') - @irpy.lazy_property - def codelet(self): - for o,a in self.opts: - if o in [ "-c", '--'+options['c'][0] ]: - buffer = a.split(':') - filename = 'codelet_'+buffer[0]+'.irp.f' - if len(buffer) == 2: - return [buffer[0], int(buffer[1]), None, filename] - elif len(buffer) == 3: - return [buffer[0], int(buffer[2]), buffer[1], filename] - else: - print """ + @irpy.lazy_property + def substituted(self): + self._substituted = {} + for o, a in self.opts: + if o in ["-s", '--' + options['s'][0]]: + k, v = a.split(':') + v_re = re.compile(r"(\W)(%s)(\W.*$|$)" % k.strip()) + self._substituted[k] = [v, v_re] + return self._substituted + + @irpy.lazy_property + def codelet(self): + for o, a in self.opts: + if o in ["-c", '--' + options['c'][0]]: + buffer = a.split(':') + filename = 'codelet_' + buffer[0] + '.irp.f' + if len(buffer) == 2: + return [buffer[0], int(buffer[1]), None, filename] + elif len(buffer) == 3: + return [buffer[0], int(buffer[2]), buffer[1], filename] + else: + print """ Error in codelet definition. Use: --codelet=provider:NMAX or --codelet=provider:precondition:NMAX """ - sys.exit(1) + sys.exit(1) - @irpy.lazy_property - def preprocessed(self): - return [a for o,a in self.ops if o in [ "-p", '--'+options['p'][0] ] ] + @irpy.lazy_property + def preprocessed(self): + return [a for o, a in self.ops if o in ["-p", '--' + options['p'][0]]] - @irpy.lazy_property - def touched(self): - return [a for o,a in self.ops if o in [ "-t", '--'+options['t'][0] ] ] + @irpy.lazy_property + def touched(self): + return [a for o, a in self.ops if o in ["-t", '--' + options['t'][0]]] - @irpy.lazy_property - def align(self): - return next( (a for o,a in self.opts if o in [ "-l", '--'+options['l'][0] ]),'1') + @irpy.lazy_property + def align(self): + return next((a for o, a in self.opts if o in ["-l", '--' + options['l'][0]]), '1') - @irpy.lazy_property - def coarray(self): - return any(o for o,a in self.opts if o in [ "-C", '--'+options['C'][0] ]) + @irpy.lazy_property + def coarray(self): + return any(o for o, a in self.opts if o in ["-C", '--' + options['C'][0]]) - @irpy.lazy_property - def warnings(self): - return any(o for o,a in self.opts if o in [ "-W", '--'+options['W'][0] ]) + @irpy.lazy_property + def warnings(self): + return any(o for o, a in self.opts if o in ["-W", '--' + options['W'][0]]) - @irpy.lazy_property - def openmp(self): - return any(o for o,a in self.opts if o in [ "-z", '--'+options['z'][0] ]) + @irpy.lazy_property + def openmp(self): + return any(o for o, a in self.opts if o in ["-z", '--' + options['z'][0]]) - @irpy.lazy_property - def ninja(self): - return any(o for o,a in self.opts if o in [ "-j", '--'+options['j'][0] ]) + @irpy.lazy_property + def ninja(self): + return any(o for o, a in self.opts if o in ["-j", '--' + options['j'][0]]) - @irpy.lazy_property - def directives(self): - return not(any(o for o,a in self.opts if o in [ "-r", '--'+options['r'][0] ])) + @irpy.lazy_property + def directives(self): + return not (any(o for o, a in self.opts if o in ["-r", '--' + options['r'][0]])) - def usage(self): - t = """ + def usage(self): + t = """ $EXE - $DESCR Usage: @@ -161,37 +183,38 @@ Usage: Options: """ - t = t.replace("$EXE",self.executable_name) - t = t.replace("$DESCR",description) - print t - print_options() - print "" - print "Version : ", version - print "" + t = t.replace("$EXE", self.executable_name) + t = t.replace("$DESCR", description) + print t + print_options() + print "" + print "Version : ", version + print "" - def opts(self): - if self._opts is None: - optlist = ["",[]] - for o in options: - b = [o]+options[o] - if b[3] == 1: - b[0] = b[0]+":" - b[1] = b[1]+"=" - optlist[0] += b[0] - optlist[1] += [b[1]] - - try: - self._opts, args = getopt.getopt(self.argv[1:], optlist[0], optlist[1]) - except getopt.GetoptError, err: - # print help information and exit: - self.usage() - print str(err) # will print something like "option -a not recognized" - sys.exit(2) - - return self._opts - opts = property(fget=opts) - - t = """ + def opts(self): + if self._opts is None: + optlist = ["", []] + for o in options: + b = [o] + options[o] + if b[3] == 1: + b[0] = b[0] + ":" + b[1] = b[1] + "=" + optlist[0] += b[0] + optlist[1] += [b[1]] + + try: + self._opts, args = getopt.getopt(self.argv[1:], optlist[0], optlist[1]) + except getopt.GetoptError, err: + # print help information and exit: + self.usage() + print str(err) # will print something like "option -a not recognized" + sys.exit(2) + + return self._opts + + opts = property(fget=opts) + + t = """ def do_$LONG(self): if '_do_$LONG' not in self.__dict__: self._do_$LONG = False @@ -202,28 +225,35 @@ def do_$LONG(self): return self._do_$LONG do_$LONG = property(fget=do_$LONG) """ - for short in options: - long = options[short][0] - exec t.replace("$LONG",long).replace("$SHORT",short) #in locals() - - @irpy.lazy_property - def do_run(self): - return not(any( (self.do_version, self.do_help, self.do_preprocess, self.do_touch, self.do_init))) + for short in options: + long = options[short][0] + exec t.replace("$LONG", long).replace("$SHORT", short) #in locals() + @irpy.lazy_property + def do_run(self): + return not (any( + (self.do_version, self.do_help, self.do_preprocess, self.do_touch, self.do_init))) + +# @irpy.lazy_property +# def do_Task(self): +# return True command_line = CommandLine() + def print_options(): - keys = options.keys() - keys.sort() - import subprocess - for k in keys: - description = options[k][1] - p1 = subprocess.Popen(["fold", "-s", "-w", "40"],stdout=subprocess.PIPE,stdin=subprocess.PIPE) - description = p1.communicate(description)[0] - description = description.replace('\n','\n'.ljust(27)) - print ("-%s, --%s"%(k,options[k][0])).ljust(25), description+'\n' - print "\n" + keys = options.keys() + keys.sort() + import subprocess + for k in keys: + description = options[k][1] + p1 = subprocess.Popen( + ["fold", "-s", "-w", "40"], stdout=subprocess.PIPE, stdin=subprocess.PIPE) + description = p1.communicate(description)[0] + description = description.replace('\n', '\n'.ljust(27)) + print("-%s, --%s" % (k, options[k][0])).ljust(25), description + '\n' + print "\n" + if __name__ == '__main__': - print_options() + print_options() diff --git a/src/create_man.py b/src/create_man.py index 5d51efc..447f367 100644 --- a/src/create_man.py +++ b/src/create_man.py @@ -27,9 +27,10 @@ from entity import Entity from routine import Routine from irpf90_t import mandir -from util import parmap, build_dim,lazy_write_file +from util import parmap, build_dim, lazy_write_file import os + def do_print_short(entity): assert type(entity) == Entity str_ = "{0:<35} : {1:<30} :: {2:<25} {3}".format(entity.prototype.filename[0], @@ -56,7 +57,7 @@ def process_deps(l): def process_types(entity_input, d_entity): assert type(entity_input) == Entity - l_name = [entity_input.name] + entity_input.others_entity_name + l_name = entity_input.l_name l_entity = [d_entity[name] for name in l_name] l = [ "{0}\t:: {1}\t{2}".format(entity.type, name, build_dim(entity.dim) ) @@ -107,7 +108,6 @@ def do_print(entity, d_entity): lazy_write_file("%s%s.l" % (mandir, name), '%s\n' % str_) - ###################################################################### def do_print_subroutines(sub): assert type(sub) == Routine @@ -164,8 +164,8 @@ def run(d_entity, d_routine): l_subs = d_routine.values() - l_data_to_write = [("%s.l" % os.path.join(mandir, s.name), do_print_subroutines(s)) for s in l_subs] - + l_data_to_write = [("%s.l" % os.path.join(mandir, s.name), do_print_subroutines(s)) + for s in l_subs] def worker(l): filename, text = l diff --git a/src/entity.py b/src/entity.py index 5b9f3aa..59dd2a2 100644 --- a/src/entity.py +++ b/src/entity.py @@ -28,10 +28,7 @@ from irpf90_t import * from util import * from command_line import command_line import sys -try: - import irpy -except: - import lib_irpy as irpy +from lib.manager import irpy class Entity(object): @@ -64,7 +61,6 @@ class Entity(object): self.comm_world = comm_world - # ~ # ~ # ~ # G l o b a l P r o p e r t y # ~ # ~ # ~ @@ -97,7 +93,6 @@ class Entity(object): d[type(line)] += [(i, line)] return d - # ~ # ~ # ~ # M u l t i p l e P r o v i d e r H a n d l e r # ~ # ~ # ~ @@ -105,16 +100,15 @@ class Entity(object): def is_main(self): # () -> bool '''Check if this Entity is the main one - - Exemple: - BEGIN_PROVIDER [pi, double precision] & + + Exemple: + BEGIN_PROVIDER [pi, double precision] & BEGIN_PROVIDER [e, double preision] return True for 'pi' and False for 'e' ''' return self.name == self.same_as - @irpy.lazy_property def prototype(self): # () -> Line @@ -128,21 +122,25 @@ class Entity(object): ''' d = self.d_type_lines - return next(line for _,line in d[Begin_provider]+d[Cont_provider] if line.filename[1] == self.name) - + return next(line for _, line in d[Begin_provider] + d[Cont_provider] + if line.filename[1] == self.name) @irpy.lazy_property - def others_entity_name(self): + def l_name(self): + # () -> List[str] + d = self.d_type_lines + return [line.filename[1] for _, line in d[Begin_provider] + d[Cont_provider]] + + @irpy.lazy_property + def l_others_name(self): # () -> List[str] '''Extract the other entity-name defined''' - d = self.d_type_lines - return [line.filename[1] for _,line in d[Begin_provider]+d[Cont_provider] if not line.filename[1] == self.name] - + return [name for name in self.l_name if not name == self.name] @irpy.lazy_property def doc(self): # () -> List[str] - doc = [line.text.lstrip()[1:] for _,line in self.d_type_lines[Doc]] + doc = [line.text.lstrip()[1:] for _, line in self.d_type_lines[Doc]] if not doc: logger.warning("Entity '%s' is not documented" % (self.name)) return doc @@ -163,45 +161,36 @@ class Entity(object): return any(self.d_entity[i].is_written for i in self.parents) @irpy.lazy_property - def writer(self): + def io_er(self): if not self.is_main: result = [] - else: - name = self.name - result = [ \ - "subroutine writer_%s(irp_num)"%(name), - " use %s"%(self.fmodule), - " implicit none", - " character*(*), intent(in) :: irp_num", - " logical :: irp_is_open", - " integer :: irp_iunit" ] - if command_line.do_debug: - length = len("writer_%s" % (self.name)) - result += [\ - " character*(%d) :: irp_here = 'writer_%s'"%(length,name), - " call irp_enter(irp_here)" ] - result += [ \ - " if (.not.%s_is_built) then"%(self.same_as), - " call provide_%s"%(self.same_as), - " endif" ] - result += map(lambda x: " call writer_%s(irp_num)" % (x), self.needs) - result += [ \ - " irp_is_open = .True.", - " irp_iunit = 9", - " do while (irp_is_open)", - " irp_iunit = irp_iunit+1", - " inquire(unit=irp_iunit,opened=irp_is_open)", - " enddo" ] - for n in [name] + self.others_entity_name: - result += [\ - " open(unit=irp_iunit,file='irpf90_%s_'//trim(irp_num),form='FORMATTED',status='UNKNOWN',action='WRITE')"%(n), - " write(irp_iunit,*) %s%s"%(n,build_dim(self.d_entity[n].dim,colons=True)), - " close(irp_iunit)" ] - if command_line.do_debug: - result.append(" call irp_leave(irp_here)") - result.append("end subroutine writer_%s" % (name)) - result.append("") - return result + + from util import mangled + from util import ashes_env + name = self.name + + d_template = { + 'name': name, + 'fmodule': self.fmodule, + 'same_as': self.same_as, + 'do_debug': command_line.do_debug, + 'children': mangled(self.needs, self.d_entity), + 'group_entity': [{ + 'name': n, + 'dim': build_dim( + self.d_entity[n].dim, colons=True) + } for n in self.l_name] + } + + return ashes_env.render('ioer.f90', d_template).split('!TOKEN_SPLIT') + + @irpy.lazy_property + def reader(self): + return self.io_er[1].split('\n') + + @irpy.lazy_property + def writer(self): + return self.io_er[0].split('\n') @irpy.lazy_property_mutable def is_read(self): @@ -209,43 +198,8 @@ class Entity(object): return any(self.d_entity[i].is_read for i in self.parents) @irpy.lazy_property - def reader(self): - if not self.is_main: - result = [] - else: - name = self.name - result = [ \ - "subroutine reader_%s(irp_num)"%(name), - " use %s"%(self.fmodule), - " implicit none", - " character*(*), intent(in) :: irp_num", - " logical :: irp_is_open", - " integer :: irp_iunit" ] - if command_line.do_debug: - length = len("reader_%s" % (self.name)) - result += [\ - " character*(%d) :: irp_here = 'reader_%s'"%(length,name), - " call irp_enter(irp_here)" ] - result += map(lambda x: " call reader_%s(irp_num)" % (x), self.needs) - result += [ \ - " irp_is_open = .True.", - " irp_iunit = 9", - " do while (irp_is_open)", - " inquire(unit=irp_iunit,opened=irp_is_open)", - " enddo"] - for n in [name] + self.others: - result += [\ - " open(unit=irp_iunit,file='irpf90_%s_'//trim(irp_num),form='FORMATTED',status='OLD',action='READ')"%(n), - " read(irp_iunit,*) %s%s"%(n,build_dim(self.cm_d_variable[n].dim,colons=True)), - " close(irp_iunit)" ] - result += [ \ - " call touch_%s"%(name), - " %s_is_built = .True."%(name) ] - if command_line.do_debug: - result.append(" call irp_leave(irp_here)") - result.append("end subroutine reader_%s" % (name)) - result.append("") - return result + def is_source_touch(self): + return (Touch in self.d_type_lines or SoftTouch in self.d_type_lines) @irpy.lazy_property_mutable def is_self_touched(self): @@ -257,7 +211,7 @@ class Entity(object): '''If any of the children is touched, the entity is touched''' if self.is_self_touched or any(self.d_entity[i].is_touched for i in self.children): return True - + return False # ~ # ~ # ~ @@ -268,21 +222,21 @@ class Entity(object): def includes(self): # () -> str '''Extract the name of include who need be to be include in this Entity''' - return [line.filename for _,line in self.d_type_lines[Include]] + return [line.filename for _, line in self.d_type_lines[Include]] @irpy.lazy_property def uses(self): - '''Extract the name of module who are used in this Entity''' - return [line.filename for _,line in self.d_type_lines[Use]] + '''Extract the name of module who are used in this Entity''' + return [line.filename for _, line in self.d_type_lines[Use]] @irpy.lazy_property def calls(self): '''Extract the name ofthe function called by the entity''' def extract_name(line): - return line.text.split('(', 1)[0].split()[1].lower() + return line.text.split('(', 1)[0].split()[1].lower() - return [extract_name(line) for _,line in self.d_type_lines[Call] ] + return [extract_name(line) for _, line in self.d_type_lines[Call]] # ~ # ~ # ~ # Array Dimension @@ -314,7 +268,6 @@ class Entity(object): else: return map(str.strip, x[1:-1].split(',')) - @irpy.lazy_property def allocate(self): # () -> List[Str] @@ -323,11 +276,14 @@ class Entity(object): return [] else: # We never go here - return [var for var in self.others_entity_name + [self.name] if self.d_entity[var].dim] + return [var for var in l_name if self.d_entity[var].dim] # ~ # ~ # ~ # D e c l a r a t i o n # ~ # ~ # ~ + @irpy.lazy_property + def is_protected(self): + return self.text[0].lower.startswith('begin_provider_immu') @irpy.lazy_property def type(self): @@ -337,7 +293,7 @@ class Entity(object): type_ = self.prototype.text.split(',')[0].split('[')[1].strip() if not type_: - logger.error( "Error in definition of %s." % (self.name)) + logger.error("Error in definition of %s." % (self.name)) sys.exit(1) if self.dim: @@ -346,29 +302,20 @@ class Entity(object): return type_ @irpy.lazy_property - def header(self): + def d_header(self): # () -> List[str] '''Compute all the code needed to inistanticant the entity''' - - name = self.name - str_ = " {type_} :: {name} {dim}".format(type_=self.type, name=name, dim=build_dim(self.dim, colons=True)) - - if command_line.coarray: - if not self.dim: - str_ += " [*]" - else: - str_ += " [:]" - - l = [str_] - if self.dim and command_line.align != '1': - l += [" !DIR$ ATTRIBUTES ALIGN: %s :: %s" % (command_line.align, name)] - - if self.is_main: - l += [" logical :: %s_is_built = .False." % (name)] - - return l - + import util + d_template = { + 'name': self.name, + 'type': self.type, + 'main': self.is_main, + 'dim': build_dim( + self.dim, colons=True), + 'protected': '\n'.join(self.allocater + self.builder) if self.is_protected else False + } + return d_template ############################################################ @irpy.lazy_property @@ -391,94 +338,38 @@ class Entity(object): # ~ # ~ # ~ @irpy.lazy_property - def toucher(self): + def d_touche_template(self): # () -> List[str] '''Fabric the f90 routine who handle the cache invalidation''' # Only one by EntityColleciton if not self.is_main: - return [] + return {} - parents = self.parents - name = self.name + from util import mangled - result = ["subroutine touch_%s" % (name)] - - result += build_use(parents+[name],self.d_entity) - result.append(" implicit none") - - if command_line.do_debug: - length = str(len("touch_%s" % (name))) - result += [" character*(%s) :: irp_here = 'touch_%s'" % (length, name)] - result += [" call irp_enter(irp_here)"] - - result += map(lambda x: " %s_is_built = .False." % (x), parents) - result.append(" %s_is_built = .True." % (name)) - - if command_line.do_debug: - result.append(" call irp_leave(irp_here)") - - result.append("end subroutine touch_%s" % (name)) - result.append("") - - return result + return { + 'name': self.name, + 'l_module': + [n for n in build_use( + self.parents + [self.name], self.d_entity, use=False)], + 'l_ancestor': [n for n in mangled(self.parents, self.d_entity)] + } ########################################################## - @irpy.lazy_property - def locker(self): - if not command_line.do_openmp: - return [] - name = self.name - result = ["subroutine irp_lock_%s(set)" % (name)] - result += [ - " use omp_lib", - " implicit none", - " logical, intent(in) :: set", - " integer(kind=omp_nest_lock_kind),save :: %s_lock" % (name), - " integer,save :: ifirst", - ] - if command_line.do_debug: - length = str(len("irp_lock_%s" % (name))) - result += [ - " character*(%s) :: irp_here = 'irp_lock_%s'" % (length, name), - " call irp_enter(irp_here)" - ] - - result += [ - " if (ifirst == 0) then", - " ifirst = 1", - " call omp_init_nest_lock(%s_lock)" % (name), - " endif", - " if (set) then", - " call omp_set_nest_lock(%s_lock)" % (name), - " else", - " call omp_unset_nest_lock(%s_lock)" % (name), - " endif", - ] - if command_line.do_debug: - result.append(" call irp_leave(irp_here)") - result.append("end subroutine irp_lock_%s" % (name)) - result.append("") - return result - - ########################################################## @irpy.lazy_property def free(self): # () -> List[ str ] '''Compute an part of a subroutine used to free a variable''' name = self.name - result = ["!", - "! >>> FREE %s" % (name), - " %s_is_built = .False." % (self.same_as)] + result = ["!", "! >>> FREE %s" % (name), " %s_is_built = .False." % (self.same_as)] if self.dim: - result += [ - " if (allocated(%s)) then"%(name), - " deallocate (%s)"%(name)] + result += [" if (allocated(%s)) then" % (name), " deallocate (%s)" % (name)] if command_line.do_memory: - result += " print *, 'Deallocating %s'"%(name) + result += " print *, 'Deallocating %s'" % (name) result += [" endif"] @@ -494,130 +385,56 @@ class Entity(object): if not self.is_main: return [] + from util import mangled + + import util name = self.name - same_as = self.same_as + l_module = [x for x in build_use([self.name] + self.to_provide, self.d_entity, use=False)] + l_children = [x for x in mangled(self.to_provide, self.d_entity)] - def dimsize(x): - # (str) -> str - '''Compute the number of element in the array''' - try: - b0, b1 = x.split(':') - except ValueError: - return x + l_entity = [self.d_entity[n] for n in self.l_name] - b0_is_digit = b0.replace('-', '').isdigit() - b1_is_digit = b1.replace('-', '').isdigit() - + l = ashes_env.render('provider.f90', { + 'name': name, + 'l_module': l_module, + 'l_children_static': l_children, + 'do_debug': command_line.do_debug, + 'do_openmp': command_line.do_openmp, + 'do_task': command_line.do_Task, + 'do_corray': command_line.do_coarray, + 'dim': ','.join(self.dim), + 'l_entity': [{ + 'name': i.name, + 'dim': ','.join(i.dim) + } for i in l_entity] + }) + return [i for i in l.split('\n') if i.strip()] - if b0_is_digit and b1_is_digit: - size = str(int(b1) - int(b0) + 1) - elif b0_is_digit: - size = "(%s) - (%d)" % (b1, int(b0) - 1) - elif b1_is_digit: - size = "(%d) - (%s)" % (int(b1) + 1, b0) - else: - size = "(%s) - (%s) + 1" % (b1, b0) + @irpy.lazy_property + def allocater(self): + from util import mangled - return size + import util + name = self.name + l_module = [x for x in build_use([self.name] + self.to_provide, self.d_entity, use=False)] + if self.is_protected: + l_module.remove(self.fmodule) - def build_alloc(name): + l_dim = [{'name': name, 'rank': i + 1, 'value': dimsize(k)} for i, k in enumerate(self.dim)] - var = self.d_entity[name] - if var.dim == []: - return [] + l = ashes_env.render('allocater.f90', { + 'name': name, + 'l_module': l_module, + 'do_debug': command_line.do_debug, + 'do_corray': command_line.do_coarray, + 'do_memory': command_line.do_memory, + 'dim': ','.join(self.dim), + 'l_dim': l_dim + }) + return [i for i in l.split('\n') if i.strip()] - from util import build_dim +########################################################## - def print_size(): - return " " * 5 + "print *, ' size: {0}'".format(build_dim(var.dim)) - - def check_dimensions(): - l = ["(%s>0)" % dimsize(x) for x in var.dim] - str_ = ".and.".join(l) - return " if (%s) then" % (str_) - - def dimensions_OK(): - result = [" irp_dimensions_OK = .True."] - for i, k in enumerate(var.dim): - result.append(" irp_dimensions_OK = irp_dimensions_OK.AND.(SIZE(%s,%d)==(%s))" - % (name, i + 1, dimsize(k))) - return result - - def do_allocate(): - if command_line.coarray: - result = " allocate(%s(%s)[*],stat=irp_err)" - else: - result = " allocate(%s(%s),stat=irp_err)" - result = result % (name, ','.join(var.dim)) - if command_line.do_memory: - tmp = "\n print *, %s, 'Allocating %s(%s)'" - d = ','.join(self.dim) - result += tmp % ('size(' + name + ')', name, d) - return result - - result = [" if (allocated (%s) ) then" % (name)] - result += dimensions_OK() - result += [ - " if (.not.irp_dimensions_OK) then", " deallocate(%s,stat=irp_err)" % (name), - " if (irp_err /= 0) then", " print *, irp_here//': Deallocation failed: %s'" % - (name), print_size(), " endif" - ] - - if command_line.do_memory: - result += [" print *, 'Deallocating %s'" % (name)] - result.append(check_dimensions()) - result.append(do_allocate()) - result += [\ - " if (irp_err /= 0) then", - " print *, irp_here//': Allocation failed: %s'"%(name), - print_size(), - " endif", - " endif", - " endif", - " else" ] - result.append(check_dimensions()) - result.append(do_allocate()) - result += [ - " if (irp_err /= 0) then", " print *, irp_here//': Allocation failed: %s'" % - (name), print_size(), " endif", " endif", " endif" - ] - return result - - result = [] - if command_line.directives and command_line.inline in ["all", "providers"]: - result += ["!DEC$ ATTRIBUTES FORCEINLINE :: provide_%s" % (name)] - result += ["subroutine provide_%s" % (name)] - result += build_use([same_as] + self.to_provide, self.d_entity) - if command_line.do_openmp: - result += [" use omp_lib"] - result.append(" implicit none") - length = len("provide_%s" % (name)) - result += [ - " character*(%d) :: irp_here = 'provide_%s'" % (length, name), - " integer :: irp_err ", - " logical :: irp_dimensions_OK", - "!$ integer :: nthreads" - ] - if command_line.do_openmp: - result.append(" call irp_lock_%s(.True.)" % (same_as)) - if command_line.do_assert or command_line.do_debug: - result.append(" call irp_enter(irp_here)") - result += build_call_provide(self.to_provide, self.d_entity) - result += flatten(map(build_alloc, [self.same_as] + self.others_entity_name)) - result += [ - " if (.not.%s_is_built) then" % (same_as), " call bld_%s" % (same_as), - " %s_is_built = .True." % (same_as), "" - ] - result += [" endif"] - if command_line.do_assert or command_line.do_debug: - result.append(" call irp_leave(irp_here)") - if command_line.do_openmp: - result.append(" call irp_lock_%s(.False.)" % (same_as)) - result.append("end subroutine provide_%s" % (name)) - result.append("") - return result - - ########################################################## @irpy.lazy_property def builder(self): if not self.is_main: @@ -651,6 +468,7 @@ class Entity(object): line_prototype.filename))) for vars, line in ps_text[begin + 1:end]: + text.append((vars, line)) text += map(lambda x: ([], Simple_line(line.i, x, line.filename)), build_call_provide(vars, self.d_entity)) @@ -665,13 +483,19 @@ class Entity(object): # Add the use statement result += ["subroutine bld_%s" % (self.name)] - result += build_use([self.name] + self.needs, self.d_entity) + + l_use = build_use([self.name] + self.needs, self.d_entity, use=False) + if self.is_protected: + l_use.remove(self.fmodule) + + result += ['USE %s' % n for n in l_use] import parsed_text # Move the variable to top, and add the text parsed_text.move_to_top_list(text, [Declaration, Implicit, Use]) - result.extend( line.text for _,line in text if not isinstance(line, (Begin_doc, End_doc, Doc, Cont_provider))) + result.extend(line.text for _, line in text + if not isinstance(line, (Begin_doc, End_doc, Doc, Cont_provider))) if command_line.do_profile: result += [ @@ -689,6 +513,11 @@ class Entity(object): #Set by parsed_text.build_needs(...) raise AttributeError + @irpy.lazy_property_mutable + def needed_by(self): + #Set by parsed_text.build_needs(...) + return [] + @irpy.lazy_property def children(self): @@ -708,16 +537,10 @@ class Entity(object): ########################################################## @irpy.lazy_property def parents(self): - if not self.is_main: - return [] - result = [] for x in self.needed_by: result.append(x) - try: - result += self.d_entity[x].parents - except RuntimeError: - pass # Exception will be checked after + result += self.d_entity[x].parents result = OrderedUniqueList(result) if self.name in result: diff --git a/src/irp_stack.py b/src/irp_stack.py deleted file mode 100644 index f168176..0000000 --- a/src/irp_stack.py +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env python -# IRPF90 is a Fortran90 preprocessor written in Python for programming using -# the Implicit Reference to Parameters (IRP) method. -# Copyright (C) 2009 Anthony SCEMAMA -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Anthony Scemama -# LCPQ - IRSAMC - CNRS -# Universite Paul Sabatier -# 118, route de Narbonne -# 31062 Toulouse Cedex 4 -# scemama@irsamc.ups-tlse.fr - - -import util -from command_line import command_line - -do_assert = command_line.do_assert -do_debug = command_line.do_debug -do_openmp = command_line.do_openmp -do_memory = command_line.do_memory - -import irpf90_t - -FILENAME = irpf90_t.irpdir+"irp_stack.irp.F90" - -def create(): - - txt = """ -module irp_stack_mod - integer, parameter :: STACKMAX=1000 - character*(128),allocatable :: irp_stack(:,:) - double precision,allocatable :: irp_cpu(:,:) - integer,allocatable :: stack_index(:) - logical :: alloc = .False. - integer :: nthread - character*(128) :: white = '' -end module - -subroutine irp_enter(irp_where) - use irp_stack_mod - integer :: ithread - character*(*) :: irp_where -""" - if not do_openmp: - txt += """ - ithread = 0 -""" - else: - txt += """ - integer, external :: omp_get_thread_num - integer, external :: omp_get_num_threads - ithread = omp_get_thread_num() -""" - - txt += "$1" - - if do_memory: - txt+=""" - if (.not.alloc) then -""" - if do_openmp: - txt += """ - !$OMP PARALLEL - !$OMP SINGLE - nthread = omp_get_num_threads() - !$OMP END SINGLE - !$OMP END PARALLEL -""" - else: - txt += """ - nthread = 1 - """ - txt += """ - print *, 'Allocating irp_stack(' , STACKMAX , ',0:', nthread, ')' - print *, 'Allocating irp_cpu(' , STACKMAX , ',0:', nthread , ')' - print *, 'Allocating stack_index(0:', nthread, ')' - endif""" - txt +=""" -$2 -end subroutine - -subroutine irp_enter_f(irp_where) - use irp_stack_mod - integer :: ithread - character*(*) :: irp_where - """ - if do_openmp: - txt += """ - integer, external :: omp_get_thread_num - integer, external :: omp_get_num_threads - ithread = omp_get_thread_num() -""" - else: - txt += """ - ithread = 0 -""" - txt += """ -$1 -""" - if do_memory: - txt+=""" - if (.not.alloc) then -""" - if do_openmp: - txt += """ - !$OMP PARALLEL - !$OMP SINGLE - nthread = omp_get_num_threads() - !$OMP END SINGLE - !$OMP END PARALLEL -""" - else: - txt += """ - nthread = 1 -""" - txt +=""" - print *, 'Allocating irp_stack(',STACKMAX,',0:',nthread,')' - print *, 'Allocating irp_cpu(',STACKMAX,',0:',nthread,')' - print *, 'Allocating stack_index(0:',nthread,')' - endif -""" - txt += """ -$2 -end subroutine - -subroutine irp_leave (irp_where) - use irp_stack_mod - character*(*) :: irp_where - integer :: ithread - double precision :: cpu -""" - if do_openmp: - txt += """ - integer, external :: omp_get_thread_num - ithread = omp_get_thread_num() - """ - else: - txt += """ - ithread = 0 - """ - txt += """ -$3 -$4 -end subroutine -""" - - # $1 - if do_assert or do_debug: - s = """ - if (.not.alloc) then - """ - if do_openmp: - s += """ - !$OMP PARALLEL - !$OMP SINGLE - nthread = omp_get_num_threads() - !$OMP END SINGLE - !$OMP END PARALLEL - !$OMP CRITICAL - if (.not.alloc) then - allocate(irp_stack(0:STACKMAX,0:nthread)) - allocate(irp_cpu(0:STACKMAX,0:nthread)) - allocate(stack_index(0:nthread)) - stack_index = 0 - alloc = .True. - endif - !$OMP END CRITICAL - endif - stack_index(ithread) = min(stack_index(ithread)+1,STACKMAX) - irp_stack(stack_index(ithread),ithread) = irp_where""" - else: - s += """ - nthread = 1 - if (.not.alloc) then - allocate(irp_stack(0:STACKMAX,1)) - allocate(irp_cpu(0:STACKMAX,1)) - allocate(stack_index(2)) - stack_index = 0 - alloc = .True. - endif - endif - stack_index(1) = min(stack_index(1)+1,STACKMAX) - irp_stack(stack_index(1),1) = irp_where""" - if do_memory: - txt+=""" - print *, 'Allocating irp_stack(',STACKMAX,'0:',nthread,')' - print *, 'Allocating irp_cpu(',STACKMAX,'0:',nthread,')' - print *, 'Allocating stack_index(0:',nthread,')'""" - else: - s = "" - txt = txt.replace("$1",s) - - # $2 - if do_debug: - txt = txt.replace("$2",""" - print *, ithread, ':', white(1:stack_index(ithread))//'-> ', trim(irp_where) - call cpu_time(irp_cpu(stack_index(ithread),ithread))""") - else: - txt = txt.replace("$2","") - - # $3 - if do_debug: - txt = txt.replace("$3",""" - call cpu_time(cpu) - print *, ithread, ':', white(1:stack_index(ithread))//'<- ', & - trim(irp_stack(stack_index(ithread),ithread)), & - cpu-irp_cpu(stack_index(ithread),ithread)""") - else: - txt = txt.replace("$3","") - - # $4 - if do_debug or do_assert: - txt = txt.replace("$4",""" - stack_index(ithread) = max(0,stack_index(ithread)-1)""") - else: - txt = txt.replace("$4","") - - txt += """ -subroutine irp_trace - use irp_stack_mod - integer :: ithread - integer :: i -""" - if do_openmp: - txt += """ -!$ integer, external :: omp_get_thread_num -!$ ithread = omp_get_thread_num() -""" - else: - txt += """ - ithread = 0 -""" - txt += """ - if (.not.alloc) return - print *, 'Stack trace: ', ithread - print *, '-------------------------' - do i=1,stack_index(ithread) - print *, trim(irp_stack(i,ithread)) - enddo - print *, '-------------------------' -end subroutine - -""" - - util.lazy_write_file(FILENAME,txt) - diff --git a/src/irpf90.py b/src/irpf90.py index 213e63d..d943f34 100644 --- a/src/irpf90.py +++ b/src/irpf90.py @@ -35,17 +35,18 @@ except: from command_line import command_line from irpy_files import Irpy_comm_world + def main(): vim.install() if command_line.do_help: command_line.usage() - return + return if command_line.do_version: from version import version print version - return + return if command_line.do_init: from build_file import create_generalmakefile @@ -55,21 +56,98 @@ def main(): comm_world = Irpy_comm_world() if command_line.do_graph: - comm_world.t_filename_parsed_text # Initialize entity need. Dirty I know. + # Create a dot reprenstion of the dependency graph. + # Merge inside a subgraph the Entity provided together - print 'graph { ' - for name,entity in comm_world.d_entity.items(): - if entity.needs: - print ' {0} -> {1}'.format(name, ' '.join(entity.needs)) - print '}' + def print_full_diagram(l_entity): + + l_entity_not_leaf = [e for e in l_entity if e.needs] + print 'digraph Full { ' + for e in l_entity_not_leaf: + print ' %s -> { %s } ' % (e.name, ' '.join(e.needs)) + print '}' + + def print_subgraph(l_tuple, name, color): + for i, s in enumerate(l_tuple): + print ' subgraph cluster_%s_%s {' % (name, i) + print ' %s ' % ' '.join(s) + print ' color = %s ' % color + print ' }' + + comm_world.t_filename_parsed_text # Initialize entity need. Dirty I know. + + print_full_diagram(comm_world.d_entity.values()) + + print 'digraph Compact { ' + print ' graph [ordering="out" splines=true overlap=false];' + + l_main_usr = set([entity for entity in comm_world.d_entity.values() if entity.is_main]) + l_main_head_usr = set([entity for entity in l_main_usr if entity.l_others_name]) + l_set_main_head_name = [set(e.l_name) for e in l_main_head_usr] + + print_subgraph(l_set_main_head_name, 'usr', color='blue') + + from util import l_dummy_entity + + l_set_dummy_name = l_dummy_entity(comm_world.d_entity) + print_subgraph(l_set_dummy_name, 'dummy', color='red') + + #~=~=~=~= + # Create List Node Uniq + #~=~=~=~= + + from util import split_l_set, flatten + l_main_dummy_name, s_exculde_dummy_name = split_l_set(l_set_dummy_name) + l_name_dummy_name_flatten = flatten(l_set_dummy_name) + + l_main_head_dummy = set([comm_world.d_entity[name] for name in l_name_dummy_name_flatten]) + s_exculde_dummy = set([comm_world.d_entity[name] for name in s_exculde_dummy_name]) + + l_node_uniq = (l_main_usr | l_main_head_dummy) - s_exculde_dummy + + #~=~=~=~= + # Create All edge + #~=~=~=~= + # We need to remove the spurious edge caused by the the dummy multiples providers + d_need = dict() + for e in l_node_uniq: + d_need[e.name] = set(e.needs) + + #~=~=~=~= + # Create All edge + #~=~=~=~= + # Draw the eddge + # If a arrow if arriving into Multipliple provider and if it is bold this mean it use all the entity inside it. + + from util import uniquify + l_set_multiple = uniquify(l_set_dummy_name + l_set_main_head_name) + + l_name_usr = [e.name for e in l_main_head_usr] + for source, l_target in d_need.items(): + + if source in l_name_usr: + color = 'blue' + elif source in l_name_dummy_name_flatten: + color = 'red' + else: + color = 'black' + + for s in l_set_multiple: + if s.issubset(l_target): + print ' %s -> %s [color="%s", penwidth=2]' % (source, sorted(s).pop(), color) + l_target = l_target - s + + if l_target: + print ' %s -> { %s } [color="%s"]' % (source, ' '.join(l_target), color) + + print ' }' return - if command_line.do_preprocess: for filename, text in comm_world.preprocessed_text: - if filename in command_line.preprocessed: - for line in text: - print line.text + if filename in command_line.preprocessed: + for line in text: + print line.text return if command_line.do_touch: @@ -96,10 +174,13 @@ def main(): comm_world.create_buildfile(command_line.do_ninja) comm_world.write_modules() - + comm_world.create_touches() comm_world.create_man() + if command_line.do_debug or command_line.do_assert: + comm_world.create_stack() + if command_line.do_profile: import profile profile.run(comm_world.d_entity) @@ -107,5 +188,6 @@ def main(): if command_line.do_openmp: comm_world.create_lock() + if __name__ == '__main__': main() diff --git a/src/irpf90_indent.py b/src/irpf90_indent.py index 0b62aad..2a02bb1 100755 --- a/src/irpf90_indent.py +++ b/src/irpf90_indent.py @@ -24,249 +24,269 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - import sys import re LENMAX = 70 tabn = 2 -tab = " "*tabn +tab = " " * tabn + class Grep(object): - re_begin_program = re.compile(r"^\s*program\s",flags=re.I) - def begin_program(self,string): - return re.match(self.re_begin_program,string) is not None + re_begin_program = re.compile(r"^\s*program\s", flags=re.I) - re_end_program = re.compile(r"\s*(end\s*!?$|end\s*program)",flags=re.I) - def end_program(self,string): - return re.match(self.re_end_program,string) is not None + def begin_program(self, string): + return re.match(self.re_begin_program, string) is not None - re_begin_subroutine = re.compile(r"^\s*(recursive)?\s*subroutine\s",flags=re.I) - def begin_subroutine(self,string): - return re.match(self.re_begin_subroutine,string) is not None + re_end_program = re.compile(r"\s*(end\s*!?$|end\s*program)", flags=re.I) - re_end_subroutine = re.compile(r"\s*(end\s*!?$|end\s*subroutine)",flags=re.I) - def end_subroutine(self,string): - return re.match(self.re_end_subroutine,string) is not None + def end_program(self, string): + return re.match(self.re_end_program, string) is not None - re_begin_function = re.compile(r"^.*function\s+.*\(",flags=re.I) - def begin_function(self,string): - return re.match(self.re_begin_function,string) is not None + re_begin_subroutine = re.compile(r"^\s*(recursive)?\s*subroutine\s", flags=re.I) - re_end_function = re.compile(r"\s*(end\s*!?$|end\s*function)",flags=re.I) - def end_function(self,string): - return re.match(self.re_end_function,string) is not None + def begin_subroutine(self, string): + return re.match(self.re_begin_subroutine, string) is not None - re_begin_provider = re.compile(r"^\s*&?begin_provider\s",flags=re.I) - def begin_provider(self,string): - return re.match(self.re_begin_provider,string) is not None + re_end_subroutine = re.compile(r"\s*(end\s*!?$|end\s*subroutine)", flags=re.I) - re_end_provider = re.compile(r"^\s*end_provider\s*(!.*)?$", flags=re.I) - def end_provider(self,string): - return re.match(self.re_end_provider,string) is not None + def end_subroutine(self, string): + return re.match(self.re_end_subroutine, string) is not None - re_begin_do = re.compile(r"^\s*do\s+",flags=re.I) - def begin_do(self,string): - return re.match(self.re_begin_do,string) is not None + re_begin_function = re.compile(r"^.*function\s+.*\(", flags=re.I) - re_end_do = re.compile(r"^\s*end\s*do\s*(!.*)?$",flags=re.I) - def end_do(self,string): - return re.match(self.re_end_do,string) is not None + def begin_function(self, string): + return re.match(self.re_begin_function, string) is not None - re_begin_if = re.compile(r"^\s*if(\(|\s+).*(&|then)\s*(!.*)?$",flags=re.I) - def begin_if(self,string): - return re.match(self.re_begin_if,string) is not None + re_end_function = re.compile(r"\s*(end\s*!?$|end\s*function)", flags=re.I) - re_else = re.compile(r"^\s*else",flags=re.I) - def xelse(self,string): - return re.match(self.re_else,string) is not None + def end_function(self, string): + return re.match(self.re_end_function, string) is not None - re_end_if = re.compile(r"^\s*end\s*if\s*(!.*)?$",flags=re.I) - def end_if(self,string): - return re.match(self.re_end_if,string) is not None + re_begin_provider = re.compile(r"^\s*&?begin_provider\s", flags=re.I) - re_begin_select = re.compile(r"^\s*select\s*case",flags=re.I) - def begin_select(self,string): - return re.match(self.re_begin_select,string) is not None + def begin_provider(self, string): + return re.match(self.re_begin_provider, string) is not None - re_case = re.compile(r"^\s*case\s*\(",flags=re.I) - def case(self,string): - return re.match(self.re_case,string) is not None + re_end_provider = re.compile(r"^\s*end_provider\s*(!.*)?$", flags=re.I) - re_end_select = re.compile(r"^\s*end\s*select\s*(!.*)?$",flags=re.I) - def end_select(self,string): - return re.match(self.re_end_select,string) is not None + def end_provider(self, string): + return re.match(self.re_end_provider, string) is not None - re_continuation = re.compile(r"^\s*\S+.*&") - def continuation(self,string): - return re.match(self.re_continuation,string) is not None + re_begin_do = re.compile(r"^\s*do\s+", flags=re.I) + + def begin_do(self, string): + return re.match(self.re_begin_do, string) is not None + + re_end_do = re.compile(r"^\s*end\s*do\s*(!.*)?$", flags=re.I) + + def end_do(self, string): + return re.match(self.re_end_do, string) is not None + + re_begin_if = re.compile(r"^\s*if(\(|\s+).*(&|then)\s*(!.*)?$", flags=re.I) + + def begin_if(self, string): + return re.match(self.re_begin_if, string) is not None + + re_else = re.compile(r"^\s*else", flags=re.I) + + def xelse(self, string): + return re.match(self.re_else, string) is not None + + re_end_if = re.compile(r"^\s*end\s*if\s*(!.*)?$", flags=re.I) + + def end_if(self, string): + return re.match(self.re_end_if, string) is not None + + re_begin_select = re.compile(r"^\s*select\s*case", flags=re.I) + + def begin_select(self, string): + return re.match(self.re_begin_select, string) is not None + + re_case = re.compile(r"^\s*case\s*\(", flags=re.I) + + def case(self, string): + return re.match(self.re_case, string) is not None + + re_end_select = re.compile(r"^\s*end\s*select\s*(!.*)?$", flags=re.I) + + def end_select(self, string): + return re.match(self.re_end_select, string) is not None + + re_continuation = re.compile(r"^\s*\S+.*&") + + def continuation(self, string): + return re.match(self.re_continuation, string) is not None + + re_declaration = re.compile(r"^.*::.*$") + + def declaration(self, string): + return re.match(self.re_declaration, string) is not None - re_declaration = re.compile(r"^.*::.*$") - def declaration(self,string): - return re.match(self.re_declaration,string) is not None grep = Grep() + class indent(object): + def __init__(self): + """Run the program""" + self.run() - def __init__(self): - """Run the program""" - self.run() + def format_declaration(self, string, n): + l, r = string.split('::') + return l.strip().ljust(n) + ' :: ' + r.strip() - def format_declaration(self,string,n): - l,r = string.split('::') - return l.strip().ljust(n) + ' :: '+ r.strip() - - def format_continuation(self,string,n): - buffer = string.split('&') - if len(buffer) == 1: - l = buffer[0] - return l - else: - l, r = buffer - return l.strip().ljust(69-len(n)) + '&'+ r.strip() - - def get_filename(self): - """The file name is the first argument""" - if '_filename' not in self.__dict__: - try: - self._filename = sys.argv[1] - except: - self._filename = None - return self._filename - filename=property(fget=get_filename) - - def get_text(self): - """The text of the file is a list of lines""" - if '_text' not in self.__dict__: - if self.filename is not None: - f = open(self.filename,'r') - self._text = f.read().splitlines() - f.close() - else: - self._text = sys.stdin.read().splitlines() - return self._text - text=property(fget=get_text) - - def indentlevel(self,line): - line = line.rstrip() - k=0 - if len(line) > 0: - while line[k] == ' ': - k+=1 - return k - - def run(self): - lines = self.text - indent0 = " "*self.indentlevel(self.text[0]) - k = indent0 - line = "" - for i in range(len(self.text)): - prevline = line - line = self.text[i].strip() - if grep.continuation(line): - line = self.format_continuation(line,k) - - if grep.continuation(prevline): - print k+2*tab+self.format_continuation(line,k+2*tab) - continue - - if grep.begin_subroutine(line): - print line - k = indent0+tab - continue - - if grep.begin_function(line): - print line - k = indent0+tab - continue - - if grep.begin_program(line): - print line - k = indent0+tab - continue - - if grep.begin_provider(line): - if line[0] != '&': - k = indent0+tab - if grep.begin_provider(self.text[i+1].strip()): - print " "+line - else: - print line + def format_continuation(self, string, n): + buffer = string.split('&') + if len(buffer) == 1: + l = buffer[0] + return l else: - print line - continue + l, r = buffer + return l.strip().ljust(69 - len(n)) + '&' + r.strip() - if grep.declaration(line): - print k+self.format_declaration(line,30) - continue + def get_filename(self): + """The file name is the first argument""" + if '_filename' not in self.__dict__: + try: + self._filename = sys.argv[1] + except: + self._filename = None + return self._filename - if grep.begin_do(line): - print k+line - k += tab - continue + filename = property(fget=get_filename) - if grep.begin_if(line): - print k+line - k += tab - continue + def get_text(self): + """The text of the file is a list of lines""" + if '_text' not in self.__dict__: + if self.filename is not None: + f = open(self.filename, 'r') + self._text = f.read().splitlines() + f.close() + else: + self._text = sys.stdin.read().splitlines() + return self._text - if grep.xelse(line): - print k[:-tabn]+line - continue + text = property(fget=get_text) - if grep.begin_select(line): - print k+line - k += 2*tab - continue + def indentlevel(self, line): + line = line.rstrip() + k = 0 + if len(line) > 0: + while line[k] == ' ': + k += 1 + return k - if grep.case(line): - print k[:-tabn]+line - continue - - if grep.end_do(line): - k = k[:-tabn] - print k+line - continue - - if grep.end_if(line): - k = k[:-tabn] - print k+line - continue - - if grep.end_select(line): - k = k[:-2*tabn] - print k+line - continue - - if grep.end_subroutine(line): - print line + def run(self): + lines = self.text + indent0 = " " * self.indentlevel(self.text[0]) k = indent0 - continue + line = "" + for i in range(len(self.text)): + prevline = line + line = self.text[i].strip() + if grep.continuation(line): + line = self.format_continuation(line, k) - if grep.end_function(line): - print line - k = indent0 - continue + if grep.continuation(prevline): + print k + 2 * tab + self.format_continuation(line, k + 2 * tab) + continue - if grep.end_provider(line): - print line - k = indent0 - continue + if grep.begin_subroutine(line): + print line + k = indent0 + tab + continue - if grep.end_program(line): - print line - k = indent0 - continue + if grep.begin_function(line): + print line + k = indent0 + tab + continue - print k+line + if grep.begin_program(line): + print line + k = indent0 + tab + continue + + if grep.begin_provider(line): + if line[0] != '&': + k = indent0 + tab + if grep.begin_provider(self.text[i + 1].strip()): + print " " + line + else: + print line + else: + print line + continue + + if grep.declaration(line): + print k + self.format_declaration(line, 30) + continue + + if grep.begin_do(line): + print k + line + k += tab + continue + + if grep.begin_if(line): + print k + line + k += tab + continue + + if grep.xelse(line): + print k[:-tabn] + line + continue + + if grep.begin_select(line): + print k + line + k += 2 * tab + continue + + if grep.case(line): + print k[:-tabn] + line + continue + + if grep.end_do(line): + k = k[:-tabn] + print k + line + continue + + if grep.end_if(line): + k = k[:-tabn] + print k + line + continue + + if grep.end_select(line): + k = k[:-2 * tabn] + print k + line + continue + + if grep.end_subroutine(line): + print line + k = indent0 + continue + + if grep.end_function(line): + print line + k = indent0 + continue + + if grep.end_provider(line): + print line + k = indent0 + continue + + if grep.end_program(line): + print line + k = indent0 + continue + + print k + line def main(): - indent() + indent() if __name__ == '__main__': - main() - + main() diff --git a/src/irpf90_t.py b/src/irpf90_t.py index 4e6f793..4f9b65e 100644 --- a/src/irpf90_t.py +++ b/src/irpf90_t.py @@ -24,7 +24,6 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - irpdir = "IRPF90_temp/" mandir = "IRPF90_man/" @@ -32,10 +31,8 @@ from zlib import crc32 import os irp_id = abs(crc32(os.getcwd())) -try: - import irpy -except: - import lib_irpy as irpy +from lib.manager import irpy +from util import logger class Line(object): @@ -51,36 +48,36 @@ class Line(object): def __repr__(self): return "%20s:%5d : %s (%s)" % (type(self).__name__, self.i, self.text, self.filename) + class LineWithName(Line): + @irpy.lazy_property + def subname(self): + buf = self.lower + if not buf.endswith(')'): + buf += "()" - @irpy.lazy_property - def subname(self): - buf = self.lower - if not buf.endswith(')'): - buf += "()" + l_buf = buf.split('(') + l_name = l_buf[0].split() - l_buf = buf.split('(') - l_name = l_buf[0].split() + if len(l_name) < 2: + import loger + logger.error("Syntax Error: %s" % line) + sys.exit(1) + return l_name.pop() - if len(l_name) < 2: - import loger - logger.error("Syntax Error: %s" % line) - sys.exit(1) - return l_name.pop() l_type = [ - 'Empty_line', 'Simple_line', "Declaration", "Continue", "Begin_provider", - "Cont_provider", "End_provider", "Begin_doc", "Doc", "End_doc", - "Begin_shell", "End_shell", "Begin_template", "End_template", "Subst", - "Assert", "Touch", "SoftTouch", "Irp_read", "Irp_write", "Irp_If", - "Irp_Else", "Irp_Endif", "Openmp", "Directive", "Use", "Do", "Enddo", "If", - "Elseif", "Else", "Endif", "Select", "Case", "End_select", "Provide", "NoDep", "Return", "Include", - "Implicit", "Free", "End", "Provide_all","Contains",'Type','End_module','Interface','End_interface', - 'Where','Elsewhere','Endwhere'] + 'Empty_line', 'Simple_line', "Declaration", "Continue", "Begin_provider", "Cont_provider", + "End_provider", "Begin_doc", "Doc", "End_doc", "Begin_shell", "End_shell", "Begin_template", + "End_template", "Subst", "Assert", "Touch", "SoftTouch", "Irp_read", "Irp_write", "Irp_If", + "Irp_Else", "Irp_Endif", "Openmp", "Directive", "Use", "Do", "Enddo", "If", "Elseif", "Else", + "Endif", "Select", "Case", "End_select", "Provide", "NoDep", "Return", "Include", "Implicit", + "Free", "End", "Provide_all", "Contains", 'Type', 'End_module', 'Interface', 'End_interface', + 'Where', 'Elsewhere', 'Endwhere' +] for t in l_type: globals()[t] = type(t, (Line, ), {}) -for t in ['Subroutine', 'Function', 'Program', 'Call','Module']: - globals()[t] = type(t, (LineWithName, ), {}) - +for t in ['Subroutine', 'Function', 'Program', 'Call', 'Module']: + globals()[t] = type(t, (LineWithName, ), {}) diff --git a/src/irpman.py b/src/irpman.py index de50f44..144a61b 100755 --- a/src/irpman.py +++ b/src/irpman.py @@ -24,17 +24,16 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - import os import sys if __name__ == "__main__": - from irpf90_t import mandir - entity = sys.argv[1].lower() + from irpf90_t import mandir + entity = sys.argv[1].lower() - filename = '%s.l'% entity - if filename not in os.listdir(mandir): - print "Error: `%s` does not exist"% entity - sys.exit(-1) + filename = '%s.l' % entity + if filename not in os.listdir(mandir): + print "Error: `%s` does not exist" % entity + sys.exit(-1) - os.system("man %s" % os.path.join(mandir,filename)) + os.system("man %s" % os.path.join(mandir, filename)) diff --git a/src/irpy_files.py b/src/irpy_files.py index b6b8b8d..b438d31 100644 --- a/src/irpy_files.py +++ b/src/irpy_files.py @@ -1,10 +1,8 @@ -from util import parmap,lazy_write_file +from util import parmap, lazy_write_file from util import flatten, listdir +from util import logger -try: - import irpy -except: - import lib_irpy as irpy +from lib.manager import irpy import os import irpf90_t @@ -12,59 +10,56 @@ import sys from command_line import command_line -from util import logger class Irpy_comm_world(object): '''Maestro.''' - def __init__(self,l_dir=None, l_file=None): + def __init__(self, l_dir=None, l_file=None): # (Iter, Iter) -> None # Create directories from itertools import ifilterfalse i_folder = ifilterfalse(os.path.exists, (irpf90_t.irpdir, irpf90_t.mandir)) - map(os.mkdir,i_folder) + map(os.mkdir, i_folder) # List file - - l_dir =l_dir if l_dir else (command_line.include_dir+['.']) + + l_dir = l_dir if l_dir else (command_line.include_dir + ['.']) l_not_dir = [d for d in l_dir if not (os.path.exists(d) and os.path.isdir(d))] if l_not_dir: - logger.error('Try to include no existing directory: [%s]' % ','.join(l_not_dir)) - sys.exit(1) - + logger.error('Try to include no existing directory: [%s]' % ','.join(l_not_dir)) + sys.exit(1) + # Create folder in IRPDIR - i_folder = ifilterfalse(os.path.exists, (os.path.join(irpf90_t.irpdir,d) for d in l_dir)) + i_folder = ifilterfalse(os.path.exists, (os.path.join(irpf90_t.irpdir, d) for d in l_dir)) map(os.mkdir, i_folder) s_folder_abs = set(os.path.abspath(path) for path in l_dir) - - s_file_folder_all = set(flatten(listdir(path,abspath=True) for path in s_folder_abs)) + + s_file_folder_all = set(flatten(listdir(path, abspath=True) for path in s_folder_abs)) # Take everything! - s_file_folder = filter(lambda f: os.path.isfile(f) and not f.startswith("."), s_file_folder_all) + s_file_folder = filter(lambda f: os.path.isfile(f) and not f.startswith("."), + s_file_folder_all) - - - - s_file_tot = set(l_file) if l_file else set() + s_file_tot = set(l_file) if l_file else set() s_file_tot.update(s_file_folder) - s_file_rel = set(os.path.relpath(f,self.cwd) for f in s_file_tot) + s_file_rel = set(os.path.relpath(f, self.cwd) for f in s_file_tot) # Lazy Copy file for f in s_file_rel: - src = os.path.join(self.cwd,f) - text_ref = open(src, 'rb').read() + src = os.path.join(self.cwd, f) + text_ref = open(src, 'rb').read() - dest = os.path.join(self.cwd,irpf90_t.irpdir, f) - lazy_write_file(dest, text_ref) + dest = os.path.join(self.cwd, irpf90_t.irpdir, f) + lazy_write_file(dest, text_ref) if command_line.do_codelet: - s_file_tot.update(command_line.codelet[3]) + s_file_tot.update(command_line.codelet[3]) # No filter the irpf90 file - self.irpf90_files_ordered=sorted(filter(lambda f: f.endswith(".irp.f") ,s_file_rel)) + self.irpf90_files_ordered = sorted(filter(lambda f: f.endswith(".irp.f"), s_file_rel)) @irpy.lazy_property def cwd(self): @@ -75,6 +70,7 @@ class Irpy_comm_world(object): '''Tuple (filename, preprocessed_text)''' from preprocessed_text import Preprocess_text + def worker_preprocess(filename): return (filename, Preprocess_text(filename).preprocessed_text) @@ -104,55 +100,62 @@ class Irpy_comm_world(object): from entity import Entity l_begin = [i for i, line in self.d_type_lines[Begin_provider]] - l_end = [i for i, line in self.d_type_lines[End_provider]] - l_provider = [ self.l_preprocessed_text[begin:end] for begin, end in zip(l_begin, l_end)] - + l_end = [i for i, line in self.d_type_lines[End_provider]] + l_provider = [self.l_preprocessed_text[begin:end] for begin, end in zip(l_begin, l_end)] l_ent = [] for icount, buf in enumerate(l_provider): - from functools import partial - Ent_part = partial(Entity,buf,icount,comm_world=self) + from functools import partial + Ent_part = partial(Entity, buf, icount, comm_world=self) - ent = Ent_part() - l_ent += [ent] + [Ent_part(other) for other in ent.others_entity_name] + ent = Ent_part() + l_ent += [ent] + [Ent_part(name) for name in ent.l_others_name] # O(2) but who care l_duplicate = [x for x in l_ent if l_ent.count(x) > 1] if l_duplicate: - from util import logger - logger.error('You have duplicate PROVIDER: %s' % ' '.join([e.name for e in l_duplicate])) - import sys - sys.exit(1) + from util import logger + logger.error('You have duplicate PROVIDER: %s' % + ' '.join([e.name for e in l_duplicate])) + import sys + sys.exit(1) # Python 2.6 Don't allow list comprehesion d_ent = dict() for e in l_ent: - d_ent[e.name] = e - + d_ent[e.name] = e + # # Second pass - # # Modify parameter of variables - - # Touch Softouch + # Touch Softouch def find_variable(line): + from util import logger + import sys + l_var = line.lower.split()[1:] if len(l_var) < 1: - error.fail(line, "Syntax error") + logger.error("Syntax error: %s", line) + import sys + sys.exit(1) - if any(v for v in l_var if v not in d_ent): - error.fail(line, "Variable %s unknown" % (v, )) + try: + e = next(v for v in l_var if v not in d_ent) + except StopIteration: + pass + else: + logger.error("Entity %s unknown: %s" % (e, line)) + import sys + sys.exit(1) return l_var - d_modif = dict() from irpf90_t import Touch, SoftTouch, Free from util import flatten - for cmd, l_type in [('is_self_touched', [Touch, SoftTouch]), - ('is_free', [Free])]: + for cmd, l_type in [('is_self_touched', [Touch, SoftTouch]), ('is_free', [Free])]: - l_line = flatten( [self.d_type_lines[type_] for type_ in l_type]) - l_name = flatten( [find_variable(line) for _, line in l_line]) + l_line = flatten([self.d_type_lines[type_] for type_ in l_type]) + l_name = flatten([find_variable(line) for _, line in l_line]) d_modif[cmd] = l_name # Read and Write @@ -175,7 +178,6 @@ class Irpy_comm_world(object): Routine is a collection of line between Subroutine / Function ''' - # ~#~#~#~#~# # Create the dict # ~#~#~#~#~# @@ -187,7 +189,9 @@ class Irpy_comm_world(object): from routine import Routine text = self.l_preprocessed_text - l_rou = [ Routine(text[b:e]) for b, e in zip(l_begin, l_end) if not isinstance(text[b], Program)] + l_rou = [ + Routine(text[b:e]) for b, e in zip(l_begin, l_end) if not isinstance(text[b], Program) + ] # Now we can create a dict and at it d_rou = dict() @@ -210,11 +214,11 @@ class Irpy_comm_world(object): from util import uniquify for routine in d_rou.values(): for x in routine.calls: - d_called_by[x].add(routine.name) + d_called_by[x].add(routine.name) for routine in d_rou.values(): routine.called_by = sorted(d_called_by[routine.name]) - + l_set = [d_rou[name].touches_my_self for name in routine.calls if name in d_rou] routine.touches_ancestor = set().union(*l_set) @@ -228,18 +232,19 @@ class Irpy_comm_world(object): import parsed_text vtuple = [(v, s.same_as, s.regexp) for v, s in d_entity.iteritems()] + def worker_parsed(filename_text): filename, text = filename_text return parsed_text.get_parsed_text(filename, text, d_entity, d_routine, vtuple) parsed_text_0 = parmap(worker_parsed, self.t_filename_preprocessed_text) - - from irpf90_t import NoDep,Declaration,Implicit,Use,Cont_provider + from irpf90_t import NoDep, Declaration, Implicit, Use, Cont_provider + def moved_to_top_l(ptext): - l = [NoDep, Declaration, Implicit, Use, Cont_provider] - for _, text in ptext: - parsed_text.move_to_top_list(text, l) + l = [NoDep, Declaration, Implicit, Use, Cont_provider] + for _, text in ptext: + parsed_text.move_to_top_list(text, l) #Touch routine parsed_text.build_sub_needs(parsed_text_0, d_routine) @@ -247,15 +252,15 @@ class Irpy_comm_world(object): parsed_text_1 = parsed_text.add_subroutine_needs(parsed_text_0, d_routine) parsed_text_1 = parsed_text.move_variables(parsed_text_1) - + moved_to_top_l(parsed_text_1) parsed_text.check_opt(parsed_text_1) parsed_text_1 = parsed_text.perform_loop_substitutions(parsed_text_1) #touch entity - stuple = [(s, v.regexp) for s, v in d_routine.iteritems() if v.is_function] - parsed_text.build_needs(parsed_text_1, d_routine, stuple,d_entity) + stuple = [(s, v.regexp) for s, v in d_routine.iteritems() if v.is_function] + parsed_text.build_needs(parsed_text_1, d_routine, stuple, d_entity) return parsed_text_1 @@ -264,7 +269,7 @@ class Irpy_comm_world(object): from module import Fmodule result = dict() for filename, text in self.t_filename_parsed_text: - result[filename] = Fmodule(text, filename,self.d_entity) + result[filename] = Fmodule(text, filename, self.d_entity) return result @@ -276,9 +281,9 @@ class Irpy_comm_world(object): for m in self.d_module.values(): # Module data if m.has_irp_module: - filename = os.path.join(irpdir, '%s.irp.module.F90' % m.filename) - text = '\n'.join(m.header + m.head) - lazy_write_file(filename, '%s\n' % text) + filename = os.path.join(irpdir, '%s.irp.module.F90' % m.filename) + text = '\n'.join(m.head) + lazy_write_file(filename, '%s\n' % text) # Subroutines filename = os.path.join(irpdir, '%s.irp.F90' % m.filename) @@ -286,12 +291,20 @@ class Irpy_comm_world(object): lazy_write_file(filename, '%s\n' % text) def create_stack(self): - import irp_stack - irp_stack.create() + from util import lazy_write_file + from util import ashes_env - def create_buildfile(self,ninja): + str_ = ashes_env.render('irp_stack.f90', { + 'do_debug': command_line.do_debug, + 'do_openmp': command_line.do_openmp, + 'do_memory': command_line.do_memory + }) + filename = os.path.join(irpf90_t.irpdir, 'irp_stack.irp.F90') + lazy_write_file(filename, str_) + + def create_buildfile(self, ninja): import build_file - build_file.run(self.d_module,ninja) + build_file.run(self.d_module, ninja) def create_touches(self): import touches @@ -301,22 +314,10 @@ class Irpy_comm_world(object): import create_man as c_man c_man.run(self.d_entity, self.d_routine) - def create_lock(self): - from util import lazy_write_file - l = sorted(self.d_entity.keys()) - - out = [] - for v in l: - out += self.d_entity[v].locker - - out += [ "subroutine irp_init_locks_%s()"%(irpf90_t.irp_id), - " implicit none" ] - for v in l: - out += [ " call irp_lock_%s(.True.)"%v ] - out += [ " call irp_lock_%s(.False.)"%v ] - out += [ "end subroutine", "" ] - - filename = os.path.join(irpf90_t.irpdir,'irp_locks.irp.F90') - lazy_write_file(filename, '\n'.join(out)) + from util import lazy_write_file + from util import ashes_env + str_ = ashes_env.render('irp_lock.f90', {'entity': sorted(self.d_entity)}) + filename = os.path.join(irpf90_t.irpdir, 'irp_locks.irp.F90') + lazy_write_file(filename, str_) diff --git a/src/lib/__init__.py b/src/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/lib/__init__.pyc b/src/lib/__init__.pyc new file mode 100644 index 0000000..b58c681 Binary files /dev/null and b/src/lib/__init__.pyc differ diff --git a/src/lib/manager.py b/src/lib/manager.py new file mode 100644 index 0000000..ed54d27 --- /dev/null +++ b/src/lib/manager.py @@ -0,0 +1,10 @@ +try: + import irpy +except: + import static_irpy as irpy + +try: + import ashes +except: + import static_ashes as ashes + diff --git a/src/lib/manager.pyc b/src/lib/manager.pyc new file mode 100644 index 0000000..1d835e9 Binary files /dev/null and b/src/lib/manager.pyc differ diff --git a/src/lib/static_ashes.py b/src/lib/static_ashes.py new file mode 100644 index 0000000..010471d --- /dev/null +++ b/src/lib/static_ashes.py @@ -0,0 +1,2602 @@ +# -*- coding: utf-8 -*- + +''' +Copyright (c) 2013, Mahmoud Hashemi + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +''' + + +from __future__ import unicode_literals + +import os +import re +import cgi +import sys +import json +import codecs +import pprint +import string +import fnmatch +import time + + +PY3 = (sys.version_info[0] == 3) +if PY3: + unicode, string_types = str, (str, bytes) +else: + string_types = (str, unicode) + +__version__ = '15.1.1dev' +__author__ = 'Mahmoud Hashemi' +__contact__ = 'mahmoudrhashemi@gmail.com' +__url__ = 'https://github.com/mahmoud/ashes' +__license__ = 'BSD' + + +DEFAULT_EXTENSIONS = ('.dust', '.html', '.xml') +DEFAULT_IGNORED_PATTERNS = ('.#*',) + + +# need to add group for literals +# switch to using word boundary for params section +node_re = re.compile(r'({' + r'(?P\/)?' + r'(?:(?P[\~\#\?\@\:\<\>\+\^\%])\s*)?' + r'(?P[a-zA-Z0-9_\$\.]+|"[^"]+")' + r'(?:\:(?P[a-zA-Z0-9\$\.]+))?' + r'(?P[\|a-z]+)*?' + r'(?P(?:\s+\w+\=(("[^"]*?")|([$\w\.]+)))*)?' + r'\s*' + r'(?P\/)?' + r'\})', + flags=re.MULTILINE) + +key_re_str = '[a-zA-Z_$][0-9a-zA-Z_$]*' +key_re = re.compile(key_re_str) +path_re = re.compile('(' + key_re_str + ')?(\.' + key_re_str + ')+') +comment_re = re.compile(r'(\{!.+?!\})|(\{`.+?`\})', flags=re.DOTALL) + + +def get_path_or_key(pork): + if pork == '.': + pk = ['path', True, []] + elif path_re.match(pork): + f_local = pork.startswith('.') + if f_local: + pork = pork[1:] + pk = ['path', f_local, pork.split('.')] + elif key_re.match(pork): + pk = ['key', pork] + else: + raise ValueError('expected a path or key, not %r' % pork) + return pk + + +def split_leading(text): + leading_stripped = text.lstrip() + leading_ws = text[:len(text) - len(leading_stripped)] + return leading_ws, leading_stripped + + +class Token(object): + def __init__(self, text): + self.text = text + + def get_line_count(self): + # returns 0 if there's only one line, because the + # token hasn't increased the number of lines. + count = len(self.text.splitlines()) - 1 + if self.text[-1] in ('\n', '\r'): + count += 1 + return count + + def __repr__(self): + cn = self.__class__.__name__ + disp = self.text + if len(disp) > 20: + disp = disp[:17] + '...' + return '%s(%r)' % (cn, disp) + + +class CommentToken(Token): + def to_dust_ast(self): + return [['comment', self.text]] + + +class RawToken(Token): + def to_dust_ast(self): + return [['raw', self.text]] + + +class BufferToken(Token): + def to_dust_ast(self): + # It is hard to simulate the PEG parsing in this case, + # especially while supporting universal newlines. + if not self.text: + return [] + rev = [] + remaining_lines = self.text.splitlines() + if self.text[-1] in ('\n', '\r'): + # kind of a bug in splitlines if you ask me. + remaining_lines.append('') + while remaining_lines: + line = remaining_lines.pop() + leading_ws, lstripped = split_leading(line) + if remaining_lines: + if lstripped: + rev.append(['buffer', lstripped]) + rev.append(['format', '\n', leading_ws]) + else: + if line: + rev.append(['buffer', line]) + ret = list(reversed(rev)) + return ret + + +ALL_ATTRS = ('closing', 'symbol', 'refpath', 'contpath', + 'filters', 'params', 'selfclosing') + + +class Tag(Token): + req_attrs = () + ill_attrs = () + + def __init__(self, text, **kw): + super(Tag, self).__init__(text) + self._attr_dict = kw + self.set_attrs(kw) + + @property + def param_list(self): + try: + return params_to_kv(self.params) + except AttributeError: + return [] + + @property + def name(self): + try: + return self.refpath.strip().lstrip('.') + except (AttributeError, TypeError): + return None + + def set_attrs(self, attr_dict, raise_exc=True): + cn = self.__class__.__name__ + all_attrs = getattr(self, 'all_attrs', ()) + if all_attrs: + req_attrs = [a for a in ALL_ATTRS if a in all_attrs] + ill_attrs = [a for a in ALL_ATTRS if a not in all_attrs] + else: + req_attrs = getattr(self, 'req_attrs', ()) + ill_attrs = getattr(self, 'ill_attrs', ()) + + opt_attrs = getattr(self, 'opt_attrs', ()) + if opt_attrs: + ill_attrs = [a for a in ill_attrs if a not in opt_attrs] + for attr in req_attrs: + if attr_dict.get(attr, None) is None: + raise ValueError('%s expected %s' % (cn, attr)) + for attr in ill_attrs: + if attr_dict.get(attr, None) is not None: + raise ValueError('%s does not take %s' % (cn, attr)) + + avail_attrs = [a for a in ALL_ATTRS if a not in ill_attrs] + for attr in avail_attrs: + setattr(self, attr, attr_dict.get(attr, '')) + return True + + @classmethod + def from_match(cls, match): + kw = dict([(str(k), v.strip()) + for k, v in match.groupdict().items() + if v is not None and v.strip()]) + obj = cls(text=match.group(0), **kw) + obj.orig_match = match + return obj + + +class ReferenceTag(Tag): + all_attrs = ('refpath',) + opt_attrs = ('filters',) + + def to_dust_ast(self): + pork = get_path_or_key(self.refpath) + filters = ['filters'] + if self.filters: + f_list = self.filters.split('|')[1:] + for f in f_list: + filters.append(f) + return [['reference', pork, filters]] + + +class SectionTag(Tag): + ill_attrs = ('closing') + + +class ClosingTag(Tag): + all_attrs = ('closing', 'refpath') + + +class SpecialTag(Tag): + all_attrs = ('symbol', 'refpath') + + def to_dust_ast(self): + return [['special', self.refpath]] + + +class BlockTag(Tag): + all_attrs = ('symbol', 'refpath') + + +class PartialTag(Tag): + req_attrs = ('symbol', 'refpath', 'selfclosing') + + def __init__(self, **kw): + super(PartialTag, self).__init__(**kw) + self.subtokens = parse_inline(self.refpath) + + def to_dust_ast(self): + """ + 2014.05.09 + This brings compatibility to the more popular fork of Dust.js + from LinkedIn (v1.0) + + Adding in `params` so `partials` function like sections. + """ + context = ['context'] + contpath = self.contpath + if contpath: + context.append(get_path_or_key(contpath)) + + params = ['params'] + param_list = self.param_list + if param_list: + try: + params.extend(params_to_dust_ast(param_list)) + except ParseError as pe: + pe.token = self + raise + + # tying to make this more standardized + inline_body = inline_to_dust_ast(self.subtokens) + return [['partial', + inline_body, + context, + params, + ]] + + +def parse_inline(source): + if not source: + raise ParseError('empty inline token') + if source.startswith('"') and source.endswith('"'): + source = source[1:-1] + if not source: + return [BufferToken("")] + tokens = tokenize(source, inline=True) + return tokens + + +def inline_to_dust_ast(tokens): + if tokens and all(isinstance(t, BufferToken) for t in tokens): + body = ['literal', ''.join(t.text for t in tokens)] + else: + body = ['body'] + for b in tokens: + body.extend(b.to_dust_ast()) + return body + + +def params_to_kv(params_str): + ret = [] + new_k, v = None, None + p_str = params_str.strip() + k, _, tail = p_str.partition('=') + while tail: + tmp, _, tail = tail.partition('=') + tail = tail.strip() + if not tail: + v = tmp + else: + v, new_k = tmp.split() + ret.append((k.strip(), v.strip())) + k = new_k + return ret + + +def params_to_dust_ast(param_kv): + ret = [] + for k, v in param_kv: + try: + v_body = get_path_or_key(v) + except ValueError: + v_body = inline_to_dust_ast(parse_inline(v)) + ret.append(['param', ['literal', k], v_body]) + return ret + + +def get_tag(match, inline=False): + groups = match.groupdict() + symbol = groups['symbol'] + closing = groups['closing'] + refpath = groups['refpath'] + if closing: + tag_type = ClosingTag + elif symbol is None and refpath is not None: + tag_type = ReferenceTag + elif symbol in '#?^<+@%': + tag_type = SectionTag + elif symbol == '~': + tag_type = SpecialTag + elif symbol == ':': + tag_type = BlockTag + elif symbol == '>': + tag_type = PartialTag + else: + raise ParseError('invalid tag symbol: %r' % symbol) + if inline and tag_type not in (ReferenceTag, SpecialTag): + raise ParseError('invalid inline tag') + return tag_type.from_match(match) + + +def tokenize(source, inline=False): + tokens = [] + com_nocom = comment_re.split(source) + line_counts = [1] + + def _add_token(t): + # i wish i had nonlocal so bad + t.start_line = sum(line_counts) + line_counts.append(t.get_line_count()) + t.end_line = sum(line_counts) + tokens.append(t) + for cnc in com_nocom: + if not cnc: + continue + elif cnc.startswith('{!') and cnc.endswith('!}'): + _add_token(CommentToken(cnc[2:-2])) + continue + elif cnc.startswith('{`') and cnc.endswith('`}'): + _add_token(RawToken(cnc[2:-2])) + continue + prev_end = 0 + start = None + end = None + for match in node_re.finditer(cnc): + start, end = match.start(1), match.end(1) + if prev_end < start: + _add_token(BufferToken(cnc[prev_end:start])) + prev_end = end + try: + _add_token(get_tag(match, inline)) + except ParseError as pe: + pe.line_no = sum(line_counts) + raise + tail = cnc[prev_end:] + if tail: + _add_token(BufferToken(tail)) + return tokens + +######### +# PARSING +######### + + +class Section(object): + def __init__(self, start_tag=None, blocks=None): + if start_tag is None: + refpath = None + name = '' + else: + refpath = start_tag.refpath + name = start_tag.name + + self.refpath = refpath + self.name = name + self.start_tag = start_tag + self.blocks = blocks or [] + + def add(self, obj): + if type(obj) == Block: + self.blocks.append(obj) + else: + if not self.blocks: + self.blocks = [Block()] + self.blocks[-1].add(obj) + + def to_dict(self): + ret = {self.name: dict([(b.name, b.to_list()) for b in self.blocks])} + return ret + + def to_dust_ast(self): + symbol = self.start_tag.symbol + + pork = get_path_or_key(self.refpath) + + context = ['context'] + contpath = self.start_tag.contpath + if contpath: + context.append(get_path_or_key(contpath)) + + params = ['params'] + param_list = self.start_tag.param_list + if param_list: + try: + params.extend(params_to_dust_ast(param_list)) + except ParseError as pe: + pe.token = self + raise + + bodies = ['bodies'] + if self.blocks: + for b in reversed(self.blocks): + bodies.extend(b.to_dust_ast()) + + return [[symbol, + pork, + context, + params, + bodies]] + + +class Block(object): + def __init__(self, name='block'): + if not name: + raise ValueError('blocks need a name, not: %r' % name) + self.name = name + self.items = [] + + def add(self, item): + self.items.append(item) + + def to_list(self): + ret = [] + for i in self.items: + try: + ret.append(i.to_dict()) + except AttributeError: + ret.append(i) + return ret + + def _get_dust_body(self): + # for usage by root block in ParseTree + ret = [] + for i in self.items: + ret.extend(i.to_dust_ast()) + return ret + + def to_dust_ast(self): + name = self.name + body = ['body'] + dust_body = self._get_dust_body() + if dust_body: + body.extend(dust_body) + return [['param', + ['literal', name], + body]] + + +class ParseTree(object): + def __init__(self, root_block): + self.root_block = root_block + + def to_dust_ast(self): + ret = ['body'] + ret.extend(self.root_block._get_dust_body()) + return ret + + @classmethod + def from_tokens(cls, tokens): + root_sect = Section() + ss = [root_sect] # section stack + for token in tokens: + if type(token) == SectionTag: + new_s = Section(token) + ss[-1].add(new_s) + if not token.selfclosing: + ss.append(new_s) + elif type(token) == ClosingTag: + if len(ss) <= 1: + msg = 'closing tag before opening tag: %r' % token.text + raise ParseError(msg, token=token) + if token.name != ss[-1].name: + msg = ('improperly nested tags: %r does not close %r' % + (token.text, ss[-1].start_tag.text)) + raise ParseError(msg, token=token) + ss.pop() + elif type(token) == BlockTag: + if len(ss) <= 1: + msg = 'start block outside of a section: %r' % token.text + raise ParseError(msg, token=token) + new_b = Block(name=token.refpath) + ss[-1].add(new_b) + else: + ss[-1].add(token) + if len(ss) > 1: + raise ParseError('unclosed tag: %r' % ss[-1].start_tag.text, + token=ss[-1].start_tag) + return cls(root_sect.blocks[0]) + + @classmethod + def from_source(cls, src): + tokens = tokenize(src) + return cls.from_tokens(tokens) + + +############## +# Optimize AST +############## +DEFAULT_SPECIAL_CHARS = {'s': ' ', + 'n': '\n', + 'r': '\r', + 'lb': '{', + 'rb': '}'} + +DEFAULT_OPTIMIZERS = { + 'body': 'compact_buffers', + 'special': 'convert_special', + 'format': 'nullify', + 'comment': 'nullify'} + +for nsym in ('buffer', 'filters', 'key', 'path', 'literal', 'raw'): + DEFAULT_OPTIMIZERS[nsym] = 'noop' + +for nsym in ('#', '?', '^', '<', '+', '@', '%', 'reference', + 'partial', 'context', 'params', 'bodies', 'param'): + DEFAULT_OPTIMIZERS[nsym] = 'visit' + +UNOPT_OPTIMIZERS = dict(DEFAULT_OPTIMIZERS) +UNOPT_OPTIMIZERS.update({'format': 'noop', 'body': 'visit'}) + + +def escape(text, esc_func=json.dumps): + return esc_func(text) + + +class Optimizer(object): + def __init__(self, optimizers=None, special_chars=None): + if special_chars is None: + special_chars = DEFAULT_SPECIAL_CHARS + self.special_chars = special_chars + + if optimizers is None: + optimizers = DEFAULT_OPTIMIZERS + self.optimizers = dict(optimizers) + + def optimize(self, node): + # aka filter_node() + nsym = node[0] + optimizer_name = self.optimizers[nsym] + return getattr(self, optimizer_name)(node) + + def noop(self, node): + return node + + def nullify(self, node): + return None + + def convert_special(self, node): + return ['buffer', self.special_chars[node[1]]] + + def visit(self, node): + ret = [node[0]] + for n in node[1:]: + filtered = self.optimize(n) + if filtered: + ret.append(filtered) + return ret + + def compact_buffers(self, node): + ret = [node[0]] + memo = None + for n in node[1:]: + filtered = self.optimize(n) + if not filtered: + continue + if filtered[0] == 'buffer': + if memo is not None: + memo[1] += filtered[1] + else: + memo = filtered + ret.append(filtered) + else: + memo = None + ret.append(filtered) + return ret + + def __call__(self, node): + return self.optimize(node) + + +######### +# Compile +######### + + +ROOT_RENDER_TMPL = \ +'''def render(chk, ctx): + {body} + return {root_func_name}(chk, ctx) +''' + + +def _python_compile(source): + """ + Generates a Python `code` object (via `compile`). + + args: + source: (required) string of python code to be compiled + + this actually compiles the template to code + """ + try: + code = compile(source, '', 'single') + return code + except: + raise + + +def _python_exec(code, name, global_env=None): + """ + this loads a code object (generated via `_python_compile` + + args: + code: (required) code object (generate via `_python_compile`) + name: (required) the name of the function + + kwargs: + global_env: (default None): the environment + """ + if global_env is None: + global_env = {} + else: + global_env = dict(global_env) + if PY3: + exec(code, global_env) + else: + exec("exec code in global_env") + return global_env[name] + + +def python_string_to_code(python_string): + """ + utility function + used to compile python string functions to code object + + args: + ``python_string`` + """ + code = _python_compile(python_string) + return code + + +def python_string_to_function(python_string): + """ + utility function + used to compile python string functions for template loading/caching + + args: + ``python_string`` + """ + code = _python_compile(python_string) + function = _python_exec(code, name='render', global_env=None) + return function + + +class Compiler(object): + """ + Note: Compiler objects aren't really meant to be reused, + the class is just for namespacing and convenience. + """ + sections = {'#': 'section', + '?': 'exists', + '^': 'notexists'} + nodes = {'<': 'inline_partial', + '+': 'region', + '@': 'helper', + '%': 'pragma'} + + def __init__(self, env=None): + if env is None: + env = default_env + self.env = env + + self.bodies = {} + self.blocks = {} + self.block_str = '' + self.index = 0 + self.auto = self.env.autoescape_filter + + def compile(self, ast, name='render'): + python_source = self._gen_python(ast) + python_code = _python_compile(python_source) + python_func = _python_exec(python_code, name=name) + return (python_code, python_func) + + def _gen_python(self, ast): # ast to init? + lines = [] + c_node = self._node(ast) + + block_str = self._root_blocks() + + bodies = self._root_bodies() + lines.extend(bodies.splitlines()) + if block_str: + lines.extend(['', block_str, '']) + body = '\n '.join(lines) + + ret = ROOT_RENDER_TMPL.format(body=body, + root_func_name=c_node) + self.python_source = ret + return ret + + def _root_blocks(self): + if not self.blocks: + self.block_str = '' + return '' + self.block_str = 'ctx = ctx.shift_blocks(blocks)\n ' + pairs = ['"' + name + '": ' + fn for name, fn in self.blocks.items()] + return 'blocks = {' + ', '.join(pairs) + '}' + + def _root_bodies(self): + max_body = max(self.bodies.keys()) + ret = [''] * (max_body + 1) + for i, body in self.bodies.items(): + ret[i] = ('\ndef body_%s(chk, ctx):\n %sreturn chk%s\n' + % (i, self.block_str, body)) + return ''.join(ret) + + def _convert_special(self, node): + return ['buffer', self.special_chars[node[1]]] + + def _node(self, node): + ntype = node[0] + if ntype in self.sections: + stype = self.sections[ntype] + return self._section(node, stype) + elif ntype in self.nodes: + ntype = self.nodes[ntype] + cfunc = getattr(self, '_' + ntype, None) + if not callable(cfunc): + raise TypeError('unsupported node type: "%r"', node[0]) + return cfunc(node) + + def _body(self, node): + index = self.index + self.index += 1 # make into property, equal to len of bodies? + name = 'body_%s' % index + self.bodies[index] = self._parts(node) + return name + + def _parts(self, body): + parts = [] + for part in body[1:]: + parts.append(self._node(part)) + return ''.join(parts) + + def _raw(self, node): + return '.write(%r)' % node[1] + + def _buffer(self, node): + return '.write(%s)' % escape(node[1]) + + def _format(self, node): + return '.write(%s)' % escape(node[1] + node[2]) + + def _reference(self, node): + return '.reference(%s,ctx,%s)' % (self._node(node[1]), + self._node(node[2])) + + def _section(self, node, cmd): + return '.%s(%s,%s,%s,%s)' % (cmd, + self._node(node[1]), + self._node(node[2]), + self._node(node[4]), + self._node(node[3])) + + def _inline_partial(self, node): + bodies = node[4] + for param in bodies[1:]: + btype = param[1][1] + if btype == 'block': + self.blocks[node[1][1]] = self._node(param[2]) + return '' + return '' + + def _region(self, node): + """aka the plus sign ('+') block""" + tmpl = '.block(ctx.get_block(%s),%s,%s,%s)' + return tmpl % (escape(node[1][1]), + self._node(node[2]), + self._node(node[4]), + self._node(node[3])) + + def _helper(self, node): + return '.helper(%s,%s,%s,%s)' % (escape(node[1][1]), + self._node(node[2]), + self._node(node[4]), + self._node(node[3])) + + def _pragma(self, node): + pr_name = node[1][1] + pragma = self.env.pragmas.get(pr_name) + if not pragma or not callable(pragma): + return '' # TODO: raise? + raw_bodies = node[4] + bodies = {} + for rb in raw_bodies[1:]: + bodies[rb[1][1]] = rb[2] + + raw_params = node[3] + params = {} + for rp in raw_params[1:]: + params[rp[1][1]] = rp[2][1] + + try: + ctx = node[2][1][1] + except (IndexError, AttributeError): + ctx = None + + return pragma(self, ctx, bodies, params) + + def _partial(self, node): + """ + 2014.05.09 + This brings compatibility to the more popular fork of Dust.js + from LinkedIn (v1.0) + + Adding in `params` so `partials` function like sections. + updating call to .partial() to include the kwargs + + dust.js reference : + compile.nodes = { + partial: function(context, node) { + return '.partial(' + + compiler.compileNode(context, node[1]) + + ',' + compiler.compileNode(context, node[2]) + + ',' + compiler.compileNode(context, node[3]) + ')'; + }, + """ + if node[0] == 'body': + body_name = self._node(node[1]) + return '.partial(' + body_name + ', %s)' % self._node(node[2]) + return '.partial(%s, %s, %s)' % (self._node(node[1]), + self._node(node[2]), + self._node(node[3])) + + def _context(self, node): + contpath = node[1:] + if contpath: + return 'ctx.rebase(%s)' % (self._node(contpath[0])) + return 'ctx' + + def _params(self, node): + parts = [self._node(p) for p in node[1:]] + if parts: + return '{' + ','.join(parts) + '}' + return 'None' + + def _bodies(self, node): + parts = [self._node(p) for p in node[1:]] + return '{' + ','.join(parts) + '}' + + def _param(self, node): + return ':'.join([self._node(node[1]), self._node(node[2])]) + + def _filters(self, node): + ret = '"%s"' % self.auto + f_list = ['"%s"' % f for f in node[1:]] # repr? + if f_list: + ret += ',[%s]' % ','.join(f_list) + return ret + + def _key(self, node): + return 'ctx.get(%r)' % node[1] + + def _path(self, node): + cur = node[1] + keys = node[2] or [] + return 'ctx.get_path(%s, %s)' % (cur, keys) + + def _literal(self, node): + return escape(node[1]) + + +######### +# Runtime +######### + + +class UndefinedValueType(object): + def __repr__(self): + return self.__class__.__name__ + '()' + + def __str__(self): + return '' + + +UndefinedValue = UndefinedValueType() + +# Prerequisites for escape_url_path + + +def _make_quote_map(allowed_chars): + ret = {} + for i in range(256): + c = chr(i) + esc_c = c if c in allowed_chars else '%{0:02X}'.format(i) + ret[i] = ret[c] = esc_c + return ret + +# The unreserved URI characters (per RFC 3986) +_UNRESERVED_CHARS = (frozenset(string.ascii_letters) + | frozenset(string.digits) + | frozenset('-._~')) +_RESERVED_CHARS = frozenset(":/?#[]@!$&'()*+,;=") # not used +_PATH_RESERVED_CHARS = frozenset("?#") # not used + +_PATH_QUOTE_MAP = _make_quote_map(_UNRESERVED_CHARS | set('/?=&:#')) + +# Escapes/filters + + +def escape_uri_path(text, to_bytes=True): + # actually meant to run on path + query args + fragment + text = to_unicode(text) + if not to_bytes: + return unicode().join([_PATH_QUOTE_MAP.get(c, c) for c in text]) + try: + bytestr = text.encode('utf-8') + except UnicodeDecodeError: + bytestr = text + except: + raise ValueError('expected text or UTF-8 encoded bytes, not %r' % text) + return ''.join([_PATH_QUOTE_MAP[b] for b in bytestr]) + + +def escape_uri_component(text): + return (escape_uri_path(text) # calls to_unicode for us + .replace('/', '%2F') + .replace('?', '%3F') + .replace('=', '%3D') + .replace('&', '%26')) + + +def escape_html(text): + text = to_unicode(text) + # TODO: dust.js doesn't use this, but maybe we should: + # .replace("'", '&squot;') + return cgi.escape(text, True) + + +def escape_js(text): + text = to_unicode(text) + return (text + .replace('\\', '\\\\') + .replace('"', '\\"') + .replace("'", "\\'") + .replace('\r', '\\r') + .replace('\u2028', '\\u2028') + .replace('\u2029', '\\u2029') + .replace('\n', '\\n') + .replace('\f', '\\f') + .replace('\t', '\\t')) + + +def comma_num(val): + try: + return '{0:,}'.format(val) + except ValueError: + return to_unicode(val) + + +def pp_filter(val): + try: + return pprint.pformat(val) + except: + try: + return repr(val) + except: + return 'unreprable object %s' % object.__repr__(val) + + +JSON_PP_INDENT = 2 + + +def ppjson_filter(val): + "A best-effort pretty-printing filter, based on the JSON module" + try: + return json.dumps(val, indent=JSON_PP_INDENT, sort_keys=True) + except TypeError: + return to_unicode(val) + + +# Helpers + +def first_helper(chunk, context, bodies, params=None): + if context.stack.index > 0: + return chunk + if 'block' in bodies: + return bodies['block'](chunk, context) + return chunk + + +def last_helper(chunk, context, bodies, params=None): + if context.stack.index < context.stack.of - 1: + return chunk + if 'block' in bodies: + return bodies['block'](chunk, context) + return chunk + + +def sep_helper(chunk, context, bodies, params=None): + if context.stack.index == context.stack.of - 1: + return chunk + if 'block' in bodies: + return bodies['block'](chunk, context) + return chunk + + +def idx_helper(chunk, context, bodies, params=None): + if 'block' in bodies: + return bodies['block'](chunk, context.push(context.stack.index)) + return chunk + + +def idx_1_helper(chunk, context, bodies, params=None): + if 'block' in bodies: + return bodies['block'](chunk, context.push(context.stack.index + 1)) + return chunk + + +def size_helper(chunk, context, bodies, params): + try: + key = params['key'] + return chunk.write(unicode(len(key))) + except (KeyError, TypeError): + return chunk + + +def _sort_iterate_items(items, sort_key, direction): + if not items: + return items + reverse = False + if direction == 'desc': + reverse = True + if not sort_key: + sort_key = 0 + elif sort_key[0] == '$': + sort_key = sort_key[1:] + if sort_key == 'key': + sort_key = 0 + elif sort_key == 'value': + sort_key = 1 + else: + try: + sort_key = int(sort_key) + except: + sort_key = 0 + return sorted(items, key=lambda x: x[sort_key], reverse=reverse) + + +def iterate_helper(chunk, context, bodies, params): + params = params or {} + body = bodies.get('block') + sort = params.get('sort') + sort_key = params.get('sort_key') + target = params.get('key') + if not body or not target: + context.env.log('warn', 'helper.iterate', 'empty block or target') + return chunk + try: + iter(target) + except: + context.env.log('warn', 'helper.iterate', 'non-iterable target') + return chunk + try: + items = target.items() + is_dict = True + except: + items = target + is_dict = False + if sort: + try: + items = _sort_iterate_items(items, sort_key, direction=sort) + except: + context.env.log('warn', 'helper.iterate', 'failed to sort target') + return chunk + if is_dict: + for key, value in items: + body(chunk, context.push({'$key': key, + '$value': value, + '$type': type(value).__name__, + '$0': key, + '$1': value})) + else: + # all this is for iterating over tuples and the like + for values in items: + try: + key = values[0] + except: + key, value = None, None + else: + try: + value = values[1] + except: + value = None + new_scope = {'$key': key, + '$value': value, + '$type': type(value).__name__} + try: + for i, value in enumerate(values): + new_scope['$%s' % i] = value + except TypeError: + context.env.log('warn', 'helper.iterate', + 'unable to enumerate values') + return chunk + else: + body(chunk, context.push(new_scope)) + return chunk + + +def _do_compare(chunk, context, bodies, params, cmp_op): + "utility function used by @eq, @gt, etc." + params = params or {} + try: + body = bodies['block'] + key = params['key'] + value = params['value'] + typestr = params.get('type') + except KeyError: + context.env.log('warn', 'helper.compare', + 'comparison missing key/value') + return chunk + rkey = _resolve_value(key, chunk, context) + if not typestr: + typestr = _COERCE_REV_MAP.get(type(rkey), 'string') + rvalue = _resolve_value(value, chunk, context) + crkey, crvalue = _coerce(rkey, typestr), _coerce(rvalue, typestr) + if isinstance(crvalue, type(crkey)) and cmp_op(crkey, crvalue): + return chunk.render(body, context) + elif 'else' in bodies: + return chunk.render(bodies['else'], context) + return chunk + + +def _resolve_value(item, chunk, context): + if not callable(item): + return item + try: + return chunk.tap_render(item, context) + except TypeError: + if getattr(context, 'is_strict', None): + raise + return item + + +_COERCE_MAP = { + 'number': float, + 'string': unicode, + 'boolean': bool, +} # Not implemented: date, context +_COERCE_REV_MAP = dict([(v, k) for k, v in _COERCE_MAP.items()]) +_COERCE_REV_MAP[int] = 'number' +try: + _COERCE_REV_MAP[long] = 'number' +except NameError: + pass + + +def _coerce(value, typestr): + coerce_type = _COERCE_MAP.get(typestr.lower()) + if not coerce_type or isinstance(value, coerce_type): + return value + if isinstance(value, string_types): + try: + value = json.loads(value) + except (TypeError, ValueError): + pass + try: + return coerce_type(value) + except (TypeError, ValueError): + return value + + +def _make_compare_helpers(): + from functools import partial + from operator import eq, ne, lt, le, gt, ge + CMP_MAP = {'eq': eq, 'ne': ne, 'gt': gt, 'lt': lt, 'gte': ge, 'lte': le} + ret = {} + for name, op in CMP_MAP.items(): + ret[name] = partial(_do_compare, cmp_op=op) + return ret + + +DEFAULT_HELPERS = {'first': first_helper, + 'last': last_helper, + 'sep': sep_helper, + 'idx': idx_helper, + 'idx_1': idx_1_helper, + 'size': size_helper, + 'iterate': iterate_helper} +DEFAULT_HELPERS.update(_make_compare_helpers()) + + +def make_base(env, stack, global_vars=None): + """`make_base( env, stack, global_vars=None )` + `env` and `stack` are required by the Python implementation. + `global_vars` is optional. set to global_vars. + + 2014.05.09 + This brings compatibility to the more popular fork of Dust.js + from LinkedIn (v1.0) + + adding this to try and create compatibility with Dust + + this is used for the non-activated alternative approach of rendering a + partial with a custom context object + + dust.makeBase = function(global) { + return new Context(new Stack(), global); + }; + """ + return Context(env, stack, global_vars) + + +# Actual runtime objects + +class Context(object): + """\ + The context is a special object that handles variable lookups and + controls template behavior. It is the interface between your + application logic and your templates. The context can be + visualized as a stack of objects that grows as we descend into + nested sections. + + When looking up a key, Dust searches the context stack from the + bottom up. There is no need to merge helper functions into the + template data; instead, create a base context onto which you can + push your local template data. + """ + def __init__(self, env, stack, global_vars=None, blocks=None): + self.env = env + self.stack = stack + if global_vars is None: + global_vars = {} + self.globals = global_vars + self.blocks = blocks + + @classmethod + def wrap(cls, env, context): + if isinstance(context, cls): + return context + return cls(env, Stack(context)) + + def get(self, path, cur=False): + "Retrieves the value `path` as a key from the context stack." + if isinstance(path, (str, unicode)): + if path[0] == '.': + cur = True + path = path[1:] + path = path.split('.') + return self._get(cur, path) + + def get_path(self, cur, down): + return self._get(cur, down) + + def _get(self, cur, down): + # many thanks to jvanasco for his contribution -mh 2014 + """ + * Get a value from the context + * @method `_get` + * @param {boolean} `cur` Get only from the current context + * @param {array} `down` An array of each step in the path + * @private + * @return {string | object} + """ + ctx = self.stack + length = 0 if not down else len(down) # TODO: try/except? + + if not length: + # wants nothing? ok, send back the entire payload + return ctx.head + + first_path_element = down[0] + + value = UndefinedValue + + if cur and not length: + ctx = ctx.head + else: + if not cur: + # Search up the stack for the first_path_element value + while ctx: + if isinstance(ctx.head, dict): + if first_path_element in ctx.head: + value = ctx.head[first_path_element] + break + ctx = ctx.tail + if value is UndefinedValue: + if first_path_element in self.globals: + ctx = self.globals[first_path_element] + else: + ctx = UndefinedValue + else: + ctx = value + else: + # if scope is limited by a leading dot, don't search up tree + if first_path_element in ctx.head: + ctx = ctx.head[first_path_element] + else: + ctx = UndefinedValue + + i = 1 + while ctx and ctx is not UndefinedValue and i < length: + if down[i] in ctx: + ctx = ctx[down[i]] + else: + ctx = UndefinedValue + i += 1 + + if ctx is UndefinedValue: + return None + else: + return ctx + + def push(self, head, index=None, length=None): + """\ + Pushes an arbitrary value `head` onto the context stack and returns + a new `Context` instance. Specify `index` and/or `length` to enable + enumeration helpers.""" + return Context(self.env, + Stack(head, self.stack, index, length), + self.globals, + self.blocks) + + def rebase(self, head): + """\ + Returns a new context instance consisting only of the value at + `head`, plus any previously defined global object.""" + return Context(self.env, + Stack(head), + self.globals, + self.blocks) + + def current(self): + """Returns the head of the context stack.""" + return self.stack.head + + def get_block(self, key): + blocks = self.blocks + if not blocks: + return None + fn = None + for block in blocks[::-1]: + try: + fn = block[key] + if fn: + break + except KeyError: + continue + return fn + + def shift_blocks(self, local_vars): + blocks = self.blocks + if local_vars: + if blocks: + new_blocks = blocks + [local_vars] + else: + new_blocks = [local_vars] + return Context(self.env, self.stack, self.globals, new_blocks) + return self + + +class Stack(object): + def __init__(self, head, tail=None, index=None, length=None): + self.head = head + self.tail = tail + self.index = index or 0 + self.of = length or 1 + # self.is_object = is_scalar(head) + + def __repr__(self): + return 'Stack(%r, %r, %r, %r)' % (self.head, + self.tail, + self.index, + self.of) + + +class Stub(object): + def __init__(self, callback): + self.head = Chunk(self) + self.callback = callback + self._out = [] + + @property + def out(self): + return ''.join(self._out) + + def flush(self): + chunk = self.head + while chunk: + if chunk.flushable: + self._out.append(chunk.data) + elif chunk.error: + self.callback(chunk.error, '') + self.flush = lambda self: None + return + else: + return + self.head = chunk = chunk.next + self.callback(None, self.out) + + +class Stream(object): + def __init__(self): + self.head = Chunk(self) + self.events = {} + + def flush(self): + chunk = self.head + while chunk: + if chunk.flushable: + self.emit('data', chunk.data) + elif chunk.error: + self.emit('error', chunk.error) + self.flush = lambda self: None + return + else: + return + self.head = chunk = chunk.next + self.emit('end') + + def emit(self, etype, data=None): + try: + self.events[etype](data) + except KeyError: + pass + + def on(self, etype, callback): + self.events[etype] = callback + return self + + +def is_scalar(obj): + return not hasattr(obj, '__iter__') or isinstance(obj, string_types) + + +def is_empty(obj): + try: + return obj is None or obj is False or len(obj) == 0 + except TypeError: + return False + + +class Chunk(object): + """\ + A Chunk is a Dust primitive for controlling the flow of the + template. Depending upon the behaviors defined in the context, + templates may output one or more chunks during rendering. A + handler that writes to a chunk directly must return the modified + chunk. + """ + def __init__(self, root, next_chunk=None, taps=None): + self.root = root + self.next = next_chunk + self.taps = taps + self._data, self.data = [], '' + self.flushable = False + self.error = None + + def write(self, data): + "Writes data to this chunk's buffer" + if self.taps: + data = self.taps.go(data) + self._data.append(data) + return self + + def end(self, data=None): + """\ + Writes data to this chunk's buffer and marks it as flushable. This + method must be called on any chunks created via chunk.map. Do + not call this method on a handler's main chunk -- dust.render + and dust.stream take care of this for you. + """ + if data: + self.write(data) + self.data = ''.join(self._data) + self.flushable = True + self.root.flush() + return self + + def map(self, callback): + """\ + Creates a new chunk and passes it to `callback`. Use map to wrap + asynchronous functions and to partition the template for + streaming. chunk.map tells Dust to manufacture a new chunk, + reserving a slot in the output stream before continuing on to + render the rest of the template. You must (eventually) call + chunk.end() on a mapped chunk to weave its content back into + the stream. + """ + cursor = Chunk(self.root, self.next, self.taps) + branch = Chunk(self.root, cursor, self.taps) + self.next = branch + self.data = ''.join(self._data) + self.flushable = True + callback(branch) + return cursor + + def tap(self, tap): + "Convenience methods for applying filters to a stream." + if self.taps: + self.taps = self.taps.push(tap) + else: + self.taps = Tap(tap) + return self + + def untap(self): + "Convenience methods for applying filters to a stream." + self.taps = self.taps.tail + return self + + def render(self, body, context): + """\ + Renders a template block, such as a default block or an else + block. Basically equivalent to body(chunk, context). + """ + return body(self, context) + + def tap_render(self, body, context): + output = [] + + def tmp_tap(data): + if data: + output.append(data) + return '' + self.tap(tmp_tap) + try: + self.render(body, context) + finally: + self.untap() + return ''.join(output) + + def reference(self, elem, context, auto, filters=None): + """\ + These methods implement Dust's default behavior for keys, + sections, blocks, partials and context helpers. While it is + unlikely you'll need to modify these methods or invoke them + from within handlers, the source code may be a useful point of + reference for developers. + """ + if callable(elem): + # this whole callable thing is a quirky thing about dust + try: + elem = elem(self, context) + except TypeError: + if getattr(context, 'is_strict', None): + raise + elem = repr(elem) + else: + if isinstance(elem, Chunk): + return elem + if is_empty(elem): + return self + else: + filtered = context.env.apply_filters(elem, auto, filters) + return self.write(filtered) + + def section(self, elem, context, bodies, params=None): + """\ + These methods implement Dust's default behavior for keys, sections, + blocks, partials and context helpers. While it is unlikely you'll need + to modify these methods or invoke them from within handlers, the + source code may be a useful point of reference for developers.""" + if callable(elem): + try: + elem = elem(self, context, bodies, params) + except TypeError: + if getattr(context, 'is_strict', None): + raise + elem = repr(elem) + else: + if isinstance(elem, Chunk): + return elem + body = bodies.get('block') + else_body = bodies.get('else') + if params: + context = context.push(params) + if not elem and else_body and elem is not 0: + # breaks with dust.js; dust.js doesn't render else blocks + # on sections referencing empty lists. + return else_body(self, context) + + if not body or elem is None: + return self + if elem is True: + return body(self, context) + elif isinstance(elem, dict) or is_scalar(elem): + return body(self, context.push(elem)) + else: + chunk = self + length = len(elem) + head = context.stack.head + for i, el in enumerate(elem): + new_ctx = context.push(el, i, length) + new_ctx.globals.update({'$len': length, + '$idx': i, + '$idx_1': i + 1}) + chunk = body(chunk, new_ctx) + return chunk + + def exists(self, elem, context, bodies, params=None): + """\ + These methods implement Dust's default behavior for keys, sections, + blocks, partials and context helpers. While it is unlikely you'll need + to modify these methods or invoke them from within handlers, the + source code may be a useful point of reference for developers.""" + if not is_empty(elem): + if bodies.get('block'): + return bodies['block'](self, context) + elif bodies.get('else'): + return bodies['else'](self, context) + return self + + def notexists(self, elem, context, bodies, params=None): + """\ + These methods implement Dust's default behavior for keys, + sections, blocks, partials and context helpers. While it is + unlikely you'll need to modify these methods or invoke them + from within handlers, the source code may be a useful point of + reference for developers. + """ + if is_empty(elem): + if bodies.get('block'): + return bodies['block'](self, context) + elif bodies.get('else'): + return bodies['else'](self, context) + return self + + def block(self, elem, context, bodies, params=None): + """\ + These methods implement Dust's default behavior for keys, + sections, blocks, partials and context helpers. While it is + unlikely you'll need to modify these methods or invoke them + from within handlers, the source code may be a useful point of + reference for developers. + """ + body = bodies.get('block') + if elem: + body = elem + if body: + body(self, context) + return self + + def partial(self, elem, context, params=None): + """These methods implement Dust's default behavior for keys, sections, + blocks, partials and context helpers. While it is unlikely you'll need + to modify these methods or invoke them from within handlers, the + source code may be a useful point of reference for developers. + """ + if params: + context = context.push(params) + if callable(elem): + _env = context.env + cback = lambda name, chk: _env.load_chunk(name, chk, context).end() + return self.capture(elem, context, cback) + return context.env.load_chunk(elem, self, context) + + def helper(self, name, context, bodies, params=None): + """\ + These methods implement Dust's default behavior for keys, + sections, blocks, partials and context helpers. While it is + unlikely you'll need to modify these methods or invoke them + from within handlers, the source code may be a useful point of + reference for developers. + """ + return context.env.helpers[name](self, context, bodies, params) + + def capture(self, body, context, callback): + def map_func(chunk): + def stub_cb(err, out): + if err: + chunk.set_error(err) + else: + callback(out, chunk) + stub = Stub(stub_cb) + body(stub.head, context).end() + return self.map(map_func) + + def set_error(self, error): + "Sets an error on this chunk and immediately flushes the output." + self.error = error + self.root.flush() + return self + + +class Tap(object): + def __init__(self, head=None, tail=None): + self.head = head + self.tail = tail + + def push(self, tap): + return Tap(tap, self) + + def go(self, value): + tap = self + while tap: + value = tap.head(value) # TODO: type errors? + tap = tap.tail + return value + + def __repr__(self): + cn = self.__class__.__name__ + return '%s(%r, %r)' % (cn, self.head, self.tail) + + +def to_unicode(obj): + try: + return unicode(obj) + except UnicodeDecodeError: + return unicode(obj, encoding='utf8') + + +DEFAULT_FILTERS = { + 'h': escape_html, + 's': to_unicode, + 'j': escape_js, + 'u': escape_uri_path, + 'uc': escape_uri_component, + 'cn': comma_num, + 'pp': pp_filter, + 'ppjson': ppjson_filter} + + +######### +# Pragmas +######### + + +def esc_pragma(compiler, context, bodies, params): + old_auto = compiler.auto + if not context: + context = 'h' + if context == 's': + compiler.auto = '' + else: + compiler.auto = context + out = compiler._parts(bodies['block']) + compiler.auto = old_auto + return out + + +DEFAULT_PRAGMAS = { + 'esc': esc_pragma +} + + +########### +# Interface +########### + +def load_template_path(path, encoding='utf-8'): + """ + split off `from_path` so __init__ can use + returns a tuple of the source and adjusted absolute path + """ + abs_path = os.path.abspath(path) + if not os.path.isfile(abs_path): + raise TemplateNotFound(abs_path) + with codecs.open(abs_path, 'r', encoding) as f: + source = f.read() + return (source, abs_path) + + +class Template(object): + # no need to set defaults on __init__ + last_mtime = None + is_convertable = True + + def __init__(self, + name, + source, + source_file=None, + optimize=True, + keep_source=True, + env=None, + lazy=False, + ): + if not source and source_file: + (source, source_abs_path) = load_template_path(source_file) + self.name = name + self.source = source + self.source_file = source_file + self.time_generated = time.time() + if source_file: + self.last_mtime = os.path.getmtime(source_file) + self.optimized = optimize + if env is None: + env = default_env + self.env = env + + if lazy: # lazy is only for testing + self.render_func = None + return + (render_code, + self.render_func + ) = self._get_render_func(optimize) + if not keep_source: + self.source = None + + @classmethod + def from_path(cls, path, name=None, encoding='utf-8', **kw): + """classmethod. + Builds a template from a filepath. + args: + ``path`` + kwargs: + ``name`` default ``None``. + ``encoding`` default ``utf-8``. + """ + (source, abs_path) = load_template_path(path) + if not name: + name = path + return cls(name=name, source=source, source_file=abs_path, **kw) + + @classmethod + def from_ast(cls, ast, name=None, **kw): + """classmethod + Builds a template from an AST representation. + This is only provided as an invert to `to_ast` + args: + ``ast`` + kwargs: + ``name`` default ``None``. + """ + template = cls(name=name, source='', lazy=True, **kw) + (render_code, + render_func + ) = template._ast_to_render_func(ast) + template.render_func = render_func + template.is_convertable = False + return template + + @classmethod + def from_python_string(cls, python_string, name=None, **kw): + """classmethod + Builds a template from an python string representation. + This is only provided as an invert to `to_python_string` + args: + ``python_string`` + kwargs: + ``name`` default ``None``. + """ + template = cls(name=name, source='', lazy=True, **kw) + render_code = _python_compile(python_string) + template.render_func = _python_exec(render_code, name='render') + template.is_convertable = False + return template + + @classmethod + def from_python_code(cls, python_code, name=None, **kw): + """classmethod + Builds a template from python code object. + This is only provided as an invert to `to_python_code` + args: + ``python_code`` + kwargs: + ``name`` default ``None``. + """ + template = cls(name=name, source='', lazy=True, **kw) + template.render_func = _python_exec(python_code, name='render') + template.is_convertable = False + return template + + @classmethod + def from_python_func(cls, python_func, name=None, **kw): + """classmethod + Builds a template from an compiled python function. + This is only provided as an invert to `to_python_func` + args: + ``python_func`` + kwargs: + ``name`` default ``None``. + """ + template = cls(name=name, source='', lazy=True, **kw) + template.render_func = python_func + template.is_convertable = False + return template + + def to_ast(self, optimize=True, raw=False): + """Generates the AST for a given template. + This can be inverted with the classmethod `from_ast`. + + kwargs: + ``optimize`` default ``True``. + ``raw`` default ``False``. + + Note: this is just a public function for `_get_ast` + """ + if not self.is_convertable: + raise TemplateConversionException() + return self._get_ast(optimize=optimize, raw=raw) + + def to_python_string(self, optimize=True): + """Generates the Python string representation for a template. + This can be inverted with the classmethod `from_python_string`. + + kwargs: + ``optimize`` default ``True``. + + Note: this is just a public method for `_get_render_string` + """ + if not self.is_convertable: + raise TemplateConversionException() + python_string = self._get_render_string(optimize=optimize) + return python_string + + def to_python_code(self, optimize=True): + """Generates the Python code representation for a template. + This can be inverted with the classmethod `from_python_code`. + + kwargs: + ``optimize`` default ``True``. + + Note: this is just a public method for `_get_render_func` + """ + if not self.is_convertable: + raise TemplateConversionException() + (python_code, + python_string + ) = self._get_render_func(optimize=optimize) + return python_code + + def to_python_func(self, optimize=True): + """Makes the python render func available. + This can be inverted with the classmethod `from_python_func`. + + Note: this is just a public method for `_get_render_func` + """ + if self.render_func: + return self.render_func + if not self.is_convertable: + raise TemplateConversionException() + (render_code, render_func) = self._get_render_func(optimize=optimize) + return render_func + + def render(self, model, env=None): + env = env or self.env + rendered = [] + + def tmp_cb(err, result): + # TODO: get rid of + if err: + print('Error on template %r: %r' % (self.name, err)) + raise RenderException(err) + else: + rendered.append(result) + return result + + chunk = Stub(tmp_cb).head + self.render_chunk(chunk, Context.wrap(env, model)).end() + return rendered[0] + + def render_chunk(self, chunk, context): + if not self.render_func: + # to support laziness for testing + (render_code, + self.render_func + ) = self._get_render_func() + return self.render_func(chunk, context) + + def _get_tokens(self): + if not self.source: + return None + return tokenize(self.source) + + def _get_ast(self, optimize=False, raw=False): + if not self.source: + return None + try: + dast = ParseTree.from_source(self.source).to_dust_ast() + except ParseError as pe: + pe.source_file = self.source_file + raise + if raw: + return dast + return self.env.filter_ast(dast, optimize) + + def _get_render_string(self, optimize=True): + """ + Uses `optimize=True` by default because it makes the output easier to + read and more like dust's docs + + This was previously `_get_render_func(..., ret_str=True)` + """ + ast = self._get_ast(optimize) + if not ast: + return None + # for testing/dev purposes + return Compiler(self.env)._gen_python(ast) + + def _get_render_func(self, optimize=True, ret_str=False): + """ + Uses `optimize=True` by default because it makes the output easier to + read and more like dust's docs + + split `ret_str=True` into `_get_render_string()` + + Note that this doesn't save the render_code/render_func. + It is compiled as needed. + """ + ast = self._get_ast(optimize) + if not ast: + return (None, None) + # consolidated the original code into _ast_to_render_func as-is below + (render_code, + render_func + ) = self._ast_to_render_func(ast) + return (render_code, render_func) + + def _ast_to_render_func(self, ast): + """this was part of ``_get_render_func`` but is better implemented + as an separate function so that AST can be directly loaded. + """ + compiler = Compiler(self.env) + (python_code, + python_func + ) = compiler.compile(ast) + return (python_code, python_func) + + def __repr__(self): + cn = self.__class__.__name__ + name, source_file = self.name, self.source_file + if not source_file: + return '<%s name=%r>' % (cn, name) + return '<%s name=%r source_file=%r>' % (cn, name, source_file) + + +class AshesException(Exception): + pass + + +class TemplateNotFound(AshesException): + def __init__(self, name): + self.name = name + super(TemplateNotFound, self).__init__('could not find template: %r' + % name) + + +class RenderException(AshesException): + pass + + +class ParseError(AshesException): + token = None + source_file = None + + def __init__(self, message, line_no=None, token=None): + self.message = message + self.token = token + self._line_no = line_no + + super(ParseError, self).__init__(self.__str__()) + + @property + def line_no(self): + if self._line_no: + return self._line_no + if getattr(self.token, 'start_line', None) is not None: + return self.token.start_line + return None + + @line_no.setter + def set_line_no(self, val): + self._line_no = val + + def __str__(self): + msg = self.message + infos = [] + if self.source_file: + infos.append('in %s' % self.source_file) + if self.line_no is not None: + infos.append('line %s' % self.line_no) + if infos: + msg += ' (%s)' % ' - '.join(infos) + return msg + + +class TemplateConversionException(AshesException): + def __init__(self): + super(TemplateConversionException, self).__init__('only templates from source ' + 'are convertable') + + +class BaseAshesEnv(object): + template_type = Template + autoescape_filter = 'h' + + def __init__(self, + loaders=None, + helpers=None, + filters=None, + special_chars=None, + optimizers=None, + pragmas=None, + auto_reload=True): + self.templates = {} + self.loaders = list(loaders or []) + self.filters = dict(DEFAULT_FILTERS) + if filters: + self.filters.update(filters) + self.helpers = dict(DEFAULT_HELPERS) + if helpers: + self.helpers.update(helpers) + self.special_chars = dict(DEFAULT_SPECIAL_CHARS) + if special_chars: + self.special_chars.update(special_chars) + self.optimizers = dict(DEFAULT_OPTIMIZERS) + if optimizers: + self.optimizers.update(optimizers) + self.pragmas = dict(DEFAULT_PRAGMAS) + if pragmas: + self.pragmas.update(pragmas) + self.auto_reload = auto_reload + + def log(self, level, name, message): + return # print(level, '-', name, '-', message) + + def render(self, name, model): + tmpl = self.load(name) + return tmpl.render(model, self) + + def load(self, name): + """Loads a template. + + args: + ``name`` template name + """ + try: + template = self.templates[name] + except KeyError: + template = self._load_template(name) + self.register(template) + if self.auto_reload: + if not getattr(template, 'source_file', None): + return template + mtime = os.path.getmtime(template.source_file) + if mtime > template.last_mtime: + template = self._load_template(name) + self.register(template) + return self.templates[name] + + def _load_template(self, name): + for loader in self.loaders: + try: + source = loader.load(name, env=self) + except TemplateNotFound: + continue + else: + return source + raise TemplateNotFound(name) + + def load_all(self, do_register=True, **kw): + """Loads all templates. + + args: + ``do_register`` default ``True` + """ + all_tmpls = [] + for loader in reversed(self.loaders): + # reversed so the first loader to have a template + # will take precendence on registration + if callable(getattr(loader, 'load_all', None)): + tmpls = loader.load_all(self, **kw) + all_tmpls.extend(tmpls) + if do_register: + for t in tmpls: + self.register(t) + return all_tmpls + + def register(self, template, name=None): + if name is None: + name = template.name + self.templates[name] = template + return + + def register_path(self, path, name=None, **kw): + """\ + Reads in, compiles, and registers a single template from a specific + path to a file containing the dust source code. + """ + kw['env'] = self + ret = self.template_type.from_path(path=path, name=name, **kw) + self.register(ret) + return ret + + def register_source(self, name, source, **kw): + """\ + Compiles and registers a single template from source code + string. Assumes caller already decoded the source string. + """ + kw['env'] = self + ret = self.template_type(name=name, source=source, **kw) + self.register(ret) + return ret + + def filter_ast(self, ast, optimize=True): + if optimize: + optimizers = self.optimizers + else: + optimizers = UNOPT_OPTIMIZERS + optimizer = Optimizer(optimizers, self.special_chars) + ret = optimizer.optimize(ast) + return ret + + def apply_filters(self, string, auto, filters): + filters = filters or [] + if not filters: + if auto: + filters = ['s', auto] + else: + filters = ['s'] + elif filters[-1] != 's': + if auto and auto not in filters: + filters += ['s', auto] + else: + filters += ['s'] + for f in filters: + filt_fn = self.filters.get(f) + if filt_fn: + string = filt_fn(string) + return string + + def load_chunk(self, name, chunk, context): + try: + tmpl = self.load(name) + except TemplateNotFound as tnf: + context.env.log('error', 'load_chunk', + 'TemplateNotFound error: %r' % tnf.name) + return chunk.set_error(tnf) + return tmpl.render_chunk(chunk, context) + + def __iter__(self): + return self.templates.itervalues() + + +class AshesEnv(BaseAshesEnv): + """ + A slightly more accessible Ashes environment, with more + user-friendly options exposed. + """ + def __init__(self, paths=None, keep_whitespace=True, *a, **kw): + if isinstance(paths, string_types): + paths = [paths] + self.paths = list(paths or []) + self.keep_whitespace = keep_whitespace + self.is_strict = kw.pop('is_strict', False) + exts = list(kw.pop('exts', DEFAULT_EXTENSIONS)) + + super(AshesEnv, self).__init__(*a, **kw) + + for path in self.paths: + tpl = TemplatePathLoader(path, exts) + self.loaders.append(tpl) + + def filter_ast(self, ast, optimize=None): + optimize = not self.keep_whitespace # preferences override + return super(AshesEnv, self).filter_ast(ast, optimize) + + +def iter_find_files(directory, patterns, ignored=None): + """\ + Finds files under a `directory`, matching `patterns` using "glob" + syntax (e.g., "*.txt"). It's also possible to ignore patterns with + the `ignored` argument, which uses the same format as `patterns. + + (from osutils.py in the boltons package) + """ + if isinstance(patterns, string_types): + patterns = [patterns] + pats_re = re.compile('|'.join([fnmatch.translate(p) for p in patterns])) + + if not ignored: + ignored = [] + elif isinstance(ignored, string_types): + ignored = [ignored] + ign_re = re.compile('|'.join([fnmatch.translate(p) for p in ignored])) + for root, dirs, files in os.walk(directory): + for basename in files: + if pats_re.match(basename): + if ignored and ign_re.match(basename): + continue + filename = os.path.join(root, basename) + yield filename + return + + +def walk_ext_matches(path, exts=None, ignored=None): + if exts is None: + exts = DEFAULT_EXTENSIONS + if ignored is None: + ignored = DEFAULT_IGNORED_PATTERNS + patterns = list(['*.' + e.lstrip('*.') for e in exts]) + + return sorted(iter_find_files(directory=path, + patterns=patterns, + ignored=ignored)) + + +class TemplatePathLoader(object): + def __init__(self, root_path, exts=None, encoding='utf-8'): + self.root_path = os.path.normpath(root_path) + self.encoding = encoding + self.exts = exts or list(DEFAULT_EXTENSIONS) + + def load(self, path, env=None): + env = env or default_env + norm_path = os.path.normpath(path) + if path.startswith('../'): + raise ValueError('no traversal above loader root path: %r' % path) + if not path.startswith(self.root_path): + norm_path = os.path.join(self.root_path, norm_path) + abs_path = os.path.abspath(norm_path) + template_name = os.path.relpath(abs_path, self.root_path) + template_type = env.template_type + return template_type.from_path(name=template_name, + path=abs_path, + encoding=self.encoding, + env=env) + + def load_all(self, env, exts=None, **kw): + ret = [] + exts = exts or self.exts + tmpl_paths = walk_ext_matches(self.root_path, exts) + for tmpl_path in tmpl_paths: + ret.append(self.load(tmpl_path, env)) + return ret + + +class FlatteningPathLoader(TemplatePathLoader): + """ + I've seen this mode of using dust templates in a couple places, + but really it's lazy and too ambiguous. It increases the chances + of silent conflicts and makes it hard to tell which templates refer + to which just by looking at the template code. + """ + def __init__(self, *a, **kw): + self.keep_ext = kw.pop('keep_ext', True) + super(FlatteningPathLoader, self).__init__(*a, **kw) + + def load(self, *a, **kw): + tmpl = super(FlatteningPathLoader, self).load(*a, **kw) + name = os.path.basename(tmpl.name) + if not self.keep_ext: + name, ext = os.path.splitext(name) + tmpl.name = name + return tmpl + +try: + import bottle +except ImportError: + pass +else: + class AshesBottleTemplate(bottle.BaseTemplate): + extensions = list(bottle.BaseTemplate.extensions) + extensions.extend(['ash', 'ashes', 'dust']) + + def prepare(self, **options): + if not self.source: + self.source = self._load_source(self.name) + if self.source is None: + raise TemplateNotFound(self.name) + + options['name'] = self.name + options['source'] = self.source + options['source_file'] = self.filename + for key in ('optimize', 'keep_source', 'env'): + if key in self.settings: + options.setdefault(key, self.settings[key]) + env = self.settings.get('env', default_env) + # I truly despise 2.6.4's unicode kwarg bug + options = dict([(str(k), v) for k, v in options.iteritems()]) + self.tpl = env.register_source(**options) + + def _load_source(self, name): + fname = self.search(name, self.lookup) + if not fname: + return + with codecs.open(fname, "rb", self.encoding) as f: + return f.read() + + def render(self, *a, **kw): + for dictarg in a: + kw.update(dictarg) + context = self.defaults.copy() + context.update(kw) + return self.tpl.render(context) + + from functools import partial as _fp + ashes_bottle_template = _fp(bottle.template, + template_adapter=AshesBottleTemplate) + ashes_bottle_view = _fp(bottle.view, + template_adapter=AshesBottleTemplate) + del bottle + del _fp + + +ashes = default_env = AshesEnv() + + +def _main(): + # TODO: accidentally unclosed tags may consume + # trailing buffers without warning + try: + tmpl = ('{@eq key=hello value="True" type="boolean"}' + '{hello}, world' + '{:else}' + 'oh well, world' + '{/eq}' + ', {@size key=hello/} characters') + ashes.register_source('hi', tmpl) + print(ashes.render('hi', {'hello': 'ayy'})) + except Exception as e: + import pdb;pdb.post_mortem() + raise + + ae = AshesEnv(filters={'cn': comma_num}) + ae.register_source('cn_tmpl', 'comma_numd: {thing|cn}') + # print(ae.render('cn_tmpl', {'thing': 21000})) + ae.register_source('tmpl', '{`{ok}thing`}') + print(ae.render('tmpl', {'thing': 21000})) + + ae.register_source('tmpl2', '{test|s}') + out = ae.render('tmpl2', {'test': [''] * 10}) + print(out) + + ae.register_source('tmpl3', '{@iterate sort="desc" sort_key=1 key=lol}' + '{$idx} - {$0}: {$1}{~n}{/iterate}') + out = ae.render('tmpl3', {'lol': {'uno': 1, 'dos': 2}}) + print(out) + out = ae.render('tmpl3', {'lol': [(1, 2, 3), (4, 5, 6)]}) + print(out) + + print(escape_uri_path("https://en.wikipedia.org/wiki/Asia's_Next_Top_Model_(cycle_3)")) + print(escape_uri_component("https://en.wikipedia.org/wiki/Asia's_Next_Top_Model_(cycle_3)")) + print('') + ae.register_source('tmpl4', '{#iterable}{$idx_1}/{$len}: {.}{@sep}, {/sep}{/iterable}') + out = ae.render('tmpl4', {'iterable': range(100, 108)}) + print(out) + + tmpl = '''\ + {#.} + row{~n} + {#.} + {.}{~n} + {/.} + {/.}''' + ashes.keep_whitespace = False + ashes.autoescape_filter = '' + ashes.register_source('nested_lists', tmpl) + print(ashes.render('nested_lists', [[1, 2], [3, 4]])) + + +class CLIError(ValueError): + pass + + +def _simple_render(template_path, template_literal, env_path_list, + model_path, model_literal, + trim_whitespace, filter, no_filter, + output_path, output_encoding, verbose): + # TODO: default value (placeholder for missing values) + env = AshesEnv(env_path_list) + env.keep_whitespace = not trim_whitespace + if filter in env.filters: + env.autoescape_filter = filter + else: + raise CLIError('unexpected filter %r, expected one of %r' + % (filter, env.filters)) + if no_filter: + env.autoescape_filter = '' + + if template_literal: + tmpl_obj = env.register_source('_literal_template', template_literal) + else: + if not template_path: + raise CLIError('expected template or template literal') + try: + tmpl_obj = env.load(template_path) + except (KeyError, TemplateNotFound): + tmpl_obj = env.register_path(template_path) + + if model_literal: + model = json.loads(model_literal) + elif not model_path: + raise CLIError('expected model or model literal') + elif model_path == '-': + model = json.load(sys.stdin) + else: + with open(model_path) as f: + model = json.load(f) + + output_text = tmpl_obj.render(model) + output_bytes = output_text.encode(output_encoding) + if output_path == '-': + print(output_bytes) + else: + with open(output_path, 'w') as f: + f.write(output_bytes) + return + + +def main(): + # using optparse for backwards compat with 2.6 (and earlier, maybe) + from optparse import OptionParser + + prs = OptionParser(description="render a template using a JSON input", + version='ashes %s' % (__version__,)) + ao = prs.add_option + ao('--env-path', + help="paths to search for templates, separate paths with :") + ao('--filter', default='h', + help="autoescape values with this filter, defaults to 'h' for HTML") + ao('--no-filter', action="store_true", + help="disables default HTML-escaping filter, overrides --filter") + ao('--trim-whitespace', action="store_true", + help="removes whitespace on template load") + ao('-m', '--model', dest='model_path', + help="path to the JSON model file, default - for stdin") + ao('-M', '--model-literal', + help="the literal string of the JSON model, overrides model") + ao('-o', '--output', dest='output_path', default='-', + help="path to the output file, default - for stdout") + ao('--output-encoding', default='utf-8', + help="encoding for the output, default utf-8") + ao('-t', '--template', dest='template_path', + help="path of template to render, absolute or relative to env-path") + ao('-T', '--template-literal', + help="the literal string of the template, overrides template") + ao('--verbose', help="emit extra output on stderr") + + opts, _ = prs.parse_args() + kwargs = dict(opts.__dict__) + + kwargs['env_path_list'] = (kwargs.pop('env_path') or '').split(':') + try: + _simple_render(**kwargs) + except CLIError as clie: + err_msg = '%s; use --help option for more info.' % (clie.args[0],) + prs.error(err_msg) + return + + +if __name__ == '__main__': + main() diff --git a/src/lib/static_ashes.pyc b/src/lib/static_ashes.pyc new file mode 100644 index 0000000..6eabe59 Binary files /dev/null and b/src/lib/static_ashes.pyc differ diff --git a/src/lib_irpy.py b/src/lib/static_irpy.py old mode 100755 new mode 100644 similarity index 99% rename from src/lib_irpy.py rename to src/lib/static_irpy.py index 574ca3c..a7d2974 --- a/src/lib_irpy.py +++ b/src/lib/static_irpy.py @@ -1,4 +1,5 @@ -#Handle the execution stack +#!/usr/bin/python + from collections import defaultdict d_path = defaultdict(list) d_last_caller = defaultdict(lambda: None) diff --git a/src/module.py b/src/module.py index 90b3826..cf55612 100644 --- a/src/module.py +++ b/src/module.py @@ -1,4 +1,4 @@ -#!/unr/bin/env python +#!/usr/bin/env python # IRPF90 is a Fortran90 preprocessor written in Python for programming using # the Implicit Reference to Parameters (IRP) method. # Copyright (C) 2009 Anthony SCEMAMA @@ -30,6 +30,7 @@ import preprocessed_text from util import * from entity import Entity + def put_info(text, filename): lenmax = 80 - len(filename) @@ -37,20 +38,19 @@ def put_info(text, filename): str_ = '{text:{width}} ! {filename}:{i:4}' for _, line in text: - line.text = str_.format(text=line.text,filename=line.filename,i=line.i,width=lenmax) + line.text = str_.format(text=line.text, filename=line.filename, i=line.i, width=lenmax) return text class Fmodule(object): - header = [ "! -*- F90 -*-", - "!", - "!-----------------------------------------------!", - "! This file was generated with the irpf90 tool. !", - "! !", - "! DO NOT MODIFY IT BY HAND !", - "!-----------------------------------------------!", - ""] + header = [ + "! -*- F90 -*-", "!", "!-----------------------------------------------!", + "! This file was generated with the irpf90 tool. !", + "! !", + "! DO NOT MODIFY IT BY HAND !", + "!-----------------------------------------------!", "" + ] def __init__(self, text, filename, d_variable): self.text = put_info(text, filename) @@ -74,19 +74,20 @@ class Fmodule(object): @irpy.lazy_property def head(self): '''The module who containt the declaration of the entity''' - body = list(self.use) - body += list(self.dec) - body += [header for var in self.l_entity for header in var.header] + if self.use or self.dec or self.l_entity: - if body: - result = ["module %s" % (self.name)] - result += body - result += ["end module %s" % (self.name)] + d_template = { + 'name': self.name, + 'use': list(self.use), + 'usr_declaration': list(self.dec), + 'irp_declaration': [e.d_header for e in self.l_entity], + 'coarray': command_line.coarray, + 'align': False if command_line.align == 1 else command_line.align + } + return [i for i in ashes_env.render('module.f90', d_template).split('\n') if i] else: - result = [] - - return result + return [] @irpy.lazy_property def has_irp_module(self): @@ -106,17 +107,17 @@ class Fmodule(object): result = [] for var in self.l_entity: result += var.provider - result += var.builder + if not var.is_protected: + result += var.builder + result += var.allocater if var.is_read: result += var.reader if var.is_written: result += var.writer - return result @irpy.lazy_property def residual_text_use_dec(self): - def remove_providers(text): result = [] inside = False @@ -137,18 +138,18 @@ class Fmodule(object): skip_interface = False for vars, line in text: if type(line) in [Interface, End_interface]: - skip_interface = not skip_interface + skip_interface = not skip_interface if skip_interface: - result.append((vars, line)) - continue - + result.append((vars, line)) + continue if type(line) in [Subroutine, Function, Program]: - #Deep copy... + #Deep copy... variable_list = list(vars) elif type(line) == End: - result += [([], Use(line.i, x, line.filename)) for x in build_use(variable_list, self.d_all_variable)] + result += [([], Use(line.i, x, line.filename)) + for x in build_use(variable_list, self.d_all_variable)] else: variable_list += vars @@ -160,15 +161,15 @@ class Fmodule(object): '''Extract the global declaration statement and module use form the declaration of function. ''' inside = 0 - result,dec,use,module = [],[],[],[] + result, dec, use, module = [], [], [], [] for vars, line in text: - if isinstance(line, (Subroutine, Function, Program,Interface,Module)): + if isinstance(line, (Subroutine, Function, Program, Interface, Module)): inside += 1 if type(line) == Module: - module.append((vars,line)) + module.append((vars, line)) if inside: result.append((vars, line)) @@ -178,10 +179,9 @@ class Fmodule(object): elif type(line) == Declaration: dec.append((vars, line)) - - if isinstance(line,(End,End_interface,End_module)): + if isinstance(line, (End, End_interface, End_module)): inside += -1 - + if inside: print 'Something wrong append' sys.exit(1) @@ -190,9 +190,10 @@ class Fmodule(object): result = remove_providers(self.text) result = modify_functions(result) - + from collections import namedtuple - Residual_text_use_dec = namedtuple('Residual_text_use_dec', ['use', 'module', 'dec', 'result']) + Residual_text_use_dec = namedtuple('Residual_text_use_dec', + ['use', 'module', 'dec', 'result']) return Residual_text_use_dec(*extract_use_dec_text(result)) @@ -208,26 +209,26 @@ class Fmodule(object): @irpy.lazy_property def dec(self): '''The declaration of this module + + Note: + Because user can define F90 Type, we need to keep the correct order. + + Warning: + If we uniquify that can cause a problem. + ```TYPE toto + INTEGER :: n + END TYPE toto + INTEGER :: n + ``` + Fix: + We need to support TYPE keyword. - Note: - Because user can define F90 Type, we need to keep the correct order. - - Warning: - If we uniquify that can cause a problem with the type in guess. - ```type toto - integer :: n - end type toto - integer :: n - ``` - Fix: - We need to support Type keyword. - - ''' + ''' l = [" %s" % line.text for _, line in self.residual_text_use_dec.dec] from util import uniquify if len(l) != len(uniquify(l)): - raise NotImplementedError + raise NotImplementedError return l @@ -241,23 +242,24 @@ class Fmodule(object): result += map(lambda x: ([], Simple_line(line.i, x, line.filename)), build_call_provide(vars, self.d_all_variable)) - from parsed_text import move_to_top_list, move_interface move_to_top_list(result, [Declaration, Implicit, Use]) move_interface(result) - return [line.text for _, line in result] + return [line.text for _, line in result] @irpy.lazy_property def needed_modules(self): - l = set(x.split(',only').pop(0).split()[1] for x in self.generated_text + self.head + self.residual_text if x.lstrip().startswith("use ")) + l = set( + x.split(',only').pop(0).split()[1] + for x in self.generated_text + self.head + self.residual_text + if x.lstrip().startswith("use ")) if self.name in l: l.remove(self.name) return l - @irpy.lazy_property def needed_modules_irp(self): return [i for i in self.needed_modules if i.endswith("_mod")] @@ -265,5 +267,3 @@ class Fmodule(object): @irpy.lazy_property def needed_modules_usr(self): return [i for i in self.needed_modules if not i.endswith("_mod")] - - diff --git a/src/parsed_text.py b/src/parsed_text.py index b55d3e2..9b44480 100644 --- a/src/parsed_text.py +++ b/src/parsed_text.py @@ -34,7 +34,8 @@ regexps_re_string_sub = regexps.re_string.sub def find_variables_in_line(line, vtuple): line_lower = regexps_re_string_sub('', line.lower) - return [same_as for v, same_as, regexp in vtuple if v in line_lower and regexp(line_lower)] + #return [same_as for v,same_as, regexp in vtuple if v in line_lower and regexp(line_lower)] + return [v for v, same_as, regexp in vtuple if v in line_lower and regexp(line_lower)] def find_funcs_in_line(line, stuple): @@ -56,16 +57,19 @@ def check_touch(variables, line, vars, main_vars): if main_var not in variables: error.fail(line, "Variable %s unknown" % (main_var, )) x = variables[main_var] - return [main_var] + x.others_entity_name + return [main_var] + x.l_others_name all_others = uniquify(flatten(map(fun, main_vars))) all_others.sort() vars.sort() for x, y in zip(vars, all_others): if x != y: - message = "The following entities should be touched:\n" + message = "The following entities should be touched:" message = "\n".join([message] + map(lambda x: "- %s" % (x, ), all_others)) - error.fail(line, message) + from util import logger + logger.error("%s (%s)" % (message, line)) + import sys + sys.exit(1) from collections import namedtuple @@ -96,7 +100,8 @@ def get_parsed_text(filename, text, variables, subroutines, vtuple): varlist.append(v) variable_list = find_variables_in_line(line, vtuple) - variable_list.remove(variables[v].same_as) + variable_list.remove(v) + # variable_list.remove(variables[v].same_as) append(Parsed_text(variable_list, line)) @@ -109,7 +114,9 @@ def get_parsed_text(filename, text, variables, subroutines, vtuple): l = filter(lambda x: x not in varlist, l) for v in l: if v not in variables: - error.fail(line, "Variable %s is unknown" % (v)) + logger.error("Variable %s is unknown (%s)" % (v, line)) + import sys + sys.exit(1) append(Parsed_text(l, Provide(line.i, "", line.filename))) append(Parsed_text(l, Simple_line(line.i, "!%s" % (line.text), line.filename))) @@ -119,6 +126,8 @@ def get_parsed_text(filename, text, variables, subroutines, vtuple): for v in l: if v not in variables: error.fail(line, "Variable %s is unknown" % (v)) + sys.exit(1) + l = map(lambda x: "-%s" % (x), l) append(Parsed_text(l, Simple_line(line.i, "!%s" % (line.text), line.filename))) elif type(line) in [Touch, SoftTouch]: @@ -149,8 +158,9 @@ def get_parsed_text(filename, text, variables, subroutines, vtuple): def fun(x): if x not in variables: error.fail(line, "Variable %s unknown" % (x, )) - return Parsed_text( - [], Simple_line(line.i, " %s_is_built = .True." % (x, ), line.filename)) + return Parsed_text([], + Simple_line(line.i, " %s_is_built = .True." % + (x, ), line.filename)) result += map(fun, main_vars[:-1]) if type(line) == SoftTouch: @@ -198,6 +208,7 @@ def get_parsed_text(filename, text, variables, subroutines, vtuple): return (filename, result) + ###################################################################### def move_to_top_list(text, it): # ( List[ List[Entity], Line], Iterator) @@ -232,13 +243,13 @@ def move_to_top_list(text, it): for i, (l_var, line) in enumerate(text): t = type(line) - if t in [Begin_provider, Module,Program, Subroutine, Function]: + if t in [Begin_provider, Module, Program, Subroutine, Function]: l_begin.append(i) elif t in [End_provider, End]: l_begin.pop() elif l_begin and t in it: - d_permutation[t].append( (l_begin[-1], [l_var, line]) ) + d_permutation[t].append((l_begin[-1], [l_var, line])) # Put the sentinel, will be deleted after the insertion text[i] = None @@ -265,32 +276,34 @@ def move_to_top_list(text, it): # Now do the Delete part of the move. Fortunatly we put a sentinel to know the line to delete for i in reversed(xrange(len(text))): if text[i] is None: - del text[i] + del text[i] -def move_interface(parsed_text,s_type=(Use,Implicit,Declaration,Subroutine,Function,Module)): - # ( List[ List[Entity], Line], Iterator) - '''Move everything containt into 'interface' below the first instance of s_type who preced it +def move_interface(parsed_text, s_type=(Use, Implicit, Declaration, Subroutine, Function, Module)): + # ( List[ List[Entity], Line], Iterator) + '''Move everything containt into 'interface' below the first instance of s_type who preced it Note: = This function is impure ''' - # Get the born of the interface - i_begin = [ i for i, (_, line) in enumerate(parsed_text) if isinstance(line,Interface) ] - i_end = [ i+1 for i, (_, line) in enumerate(parsed_text) if isinstance(line,End_interface) ] + # Get the born of the interface + i_begin = [i for i, (_, line) in enumerate(parsed_text) if isinstance(line, Interface)] + i_end = [i + 1 for i, (_, line) in enumerate(parsed_text) if isinstance(line, End_interface)] - # Get the begin of the insert - i_insert = [] - for begin in i_begin: - i_insert.append(next(i+1 for i in range(begin,-1,-1) if isinstance(parsed_text[i][1], s_type))) + # Get the begin of the insert + i_insert = [] + for begin in i_begin: + i_insert.append( + next(i + 1 for i in range(begin, -1, -1) if isinstance(parsed_text[i][1], s_type))) # Do the insert and the delete in one passe - for insert, begin, end in zip(i_insert,i_begin,i_end): - parsed_text[insert:insert] = parsed_text[begin:end] + for insert, begin, end in zip(i_insert, i_begin, i_end): + parsed_text[insert:insert] = parsed_text[begin:end] + + padding = end - begin + parsed_text[begin + padding:end + padding] = [] - padding = end-begin - parsed_text[begin+padding:end+padding] = [] ###################################################################### def build_sub_needs(parsed_text, d_subroutine): @@ -303,10 +316,14 @@ def build_sub_needs(parsed_text, d_subroutine): l_buffer = [] for _, text in parsed_text: - l_begin = [ i for i, (_, line) in enumerate(text) if isinstance(line, (Subroutine, Function, Program))] + l_begin = [ + i for i, (_, line) in enumerate(text) + if isinstance(line, (Subroutine, Function, Program)) + ] l_end = [i for i, (_, line) in enumerate(text) if isinstance(line, End)] - l_buffer += [(d_subroutine[text[b].line.subname], text[b + 1:e]) for b, e in zip(l_begin, l_end) if not isinstance(text[b].line, Program)] + l_buffer += [(d_subroutine[text[b].line.subname], text[b + 1:e]) + for b, e in zip(l_begin, l_end) if not isinstance(text[b].line, Program)] for sub, text in l_buffer: sub.needs = set(v for vs, _ in text for v in vs) @@ -329,11 +346,59 @@ def add_subroutine_needs(parsed_text, subroutines): ###################################################################### +def raise_entity(text): + #(List[ Tuple[List[Entity], Tuple[int,List[Line]] ]] + '''Working progress''' + l_token = [] + d_level_var = dict() + d_level_var[0] = [] + + skip_interface = False + lvl = 0 + + for i, (e, line) in enumerate(text): + type_ = type(line) + + if type_ in [Interface, End_interface]: + skip_interface = not skip_interface + + if skip_interface: + continue + + if type_ in [Begin_provider, Program, Subroutine, Function, If]: + l_token.append(i) + lvl += 1 + d_level_var[lvl] = e[:] + + elif type_ in [End_provider, End, Endif]: + i = l_token.pop() + text[i] = (d_level_var[lvl], text[i][1]) + + lvl += -1 + + elif type_ in [Else, Elseif]: + i = l_token.pop() + text[i] = (d_level_var[lvl], text[i][1]) + + assert (type(text[i][1]) == If) + + l_token.append(i) + d_level_var[lvl] = e[:] + + else: + d_level_var[lvl] += e[:] + text[i] = ([], line) + + assert (lvl == 0) + + def move_variables(parsed_text): #(List[ Tuple[List[Entity], Tuple[int,List[Line]] ]] - '''Move variables into the top of the declaraiton''' + '''Move variables into the top of the declaraiton. + + This need to be optimised to handle the fact that we can have multi-provider + ''' - def func(filename, text): result = [] append = result.append @@ -346,16 +411,16 @@ def move_variables(parsed_text): old_elsevars = [] revtext = list(text) revtext.reverse() - + skip_interface = False try: for vars, line in revtext: if type(line) in [Interface, End_interface]: - skip_interface = not skip_interface + skip_interface = not skip_interface if skip_interface: - append(([], line)) - continue + append(([], line)) + continue if type(line) in [End_provider, End]: varlist = [] @@ -406,6 +471,10 @@ def move_variables(parsed_text): result.reverse() + #print '@@@@@@@@@@@@@' + #for i in text: + # print i + # 2nd pass text = result result = [] @@ -447,7 +516,14 @@ def move_variables(parsed_text): main_result = [] for filename, text in parsed_text: + #for i in text: + # print i main_result.append((filename, func(filename, text))) + + #print '===========' + #for i in main_result[-1][1]: + # print i + return main_result @@ -459,6 +535,7 @@ def build_needs(parsed_text, subroutines, stuple, variables): # Needs and to_provide # ~#~#~#~#~# + # Loop of the main Entity for filename, text in parsed_text: l_begin = [i for i, (_, line) in enumerate(text) if isinstance(line, Begin_provider)] @@ -485,6 +562,7 @@ def build_needs(parsed_text, subroutines, stuple, variables): entity.needs = uniquify(l_needs) + # Now do the Other entity for v in variables: main = variables[v].same_as if main != v: @@ -492,25 +570,17 @@ def build_needs(parsed_text, subroutines, stuple, variables): variables[v].to_provide = variables[main].to_provide # ~#~#~#~#~# - # Needs and to_provide + # Needs_by # ~#~#~#~#~# + from collections import defaultdict - for v in variables: - variables[v].needed_by = [] - for v in variables: - main = variables[v].same_as - if main != v: - variables[v].needed_by = variables[main].needed_by + d_needed_by = defaultdict(list) + for var in variables.values(): + for x in var.needs: + d_needed_by[x].append(var.name) - for v in variables: - var = variables[v] - if var.is_main: - for x in var.needs: - variables[x].needed_by.append(var.same_as) - - for v in variables: - var = variables[v] - var.needed_by = uniquify(var.needed_by) + for v in d_needed_by: + variables[v].needed_by = uniquify(d_needed_by[v]) ###################################################################### from command_line import command_line @@ -549,4 +619,3 @@ def perform_loop_substitutions(parsed_text): append((vars, line)) main_result.append((filename, result)) return main_result - diff --git a/src/preprocessed_text.py b/src/preprocessed_text.py index 7aa75bf..4f56767 100644 --- a/src/preprocessed_text.py +++ b/src/preprocessed_text.py @@ -37,7 +37,7 @@ re_enddo = re.compile("end +do") re_endwhere = re.compile("end +where") re_endtype = re.compile("end +type.*") -re_endmodule = re.compile("end +module",re.I) +re_endmodule = re.compile("end +module", re.I) re_endselect = re.compile("end +select") re_endinterface = re.compile("end +interface") @@ -57,8 +57,11 @@ simple_dict = { "subst": Subst, "end_doc": End_doc, "begin_provider": Begin_provider, + "begin_provider_immutable": Begin_provider, "&begin_provider": Cont_provider, + "&begin_provider_immutable": Cont_provider, "end_provider": End_provider, + "end_provider_immutable": End_provider, "assert": Assert, "touch": Touch, "soft_touch": SoftTouch, @@ -99,6 +102,7 @@ simple_dict = { "endwhere": Endwhere, } + def get_canonized_text(text_lower): text_canonized = text_lower @@ -109,7 +113,7 @@ def get_canonized_text(text_lower): text_canonized = re_endif.sub("endif", text_canonized) text_canonized = re_endselect.sub("endselect", text_canonized) text_canonized = re_endinterface.sub("endinterface", text_canonized) - text_canonized = re_endwhere.sub('endwhere',text_canonized) + text_canonized = re_endwhere.sub('endwhere', text_canonized) for c in """()'"[]""": text_canonized = text_canonized.replace(c, " %s " % c) @@ -119,7 +123,7 @@ def get_canonized_text(text_lower): def get_type(i, filename, line, line_lower, line_lower_canonized, is_doc): # ( int,str,str,str,str,bool) -> Irpf90_t '''Find the type of a text line''' - + line = line.rstrip() l_word = line_lower_canonized.split() @@ -147,12 +151,10 @@ def get_type(i, filename, line, line_lower, line_lower_canonized, is_doc): type_ = simple_dict[firstword] return [type_(i, line, filename)], is_doc - #label do-loop (outer: do i=1,sze) reg_do_lab = ur":\s+do\s+" - if re.search(reg_do_lab,line_lower): - return [Do(i,line,filename)], is_doc - + if re.search(reg_do_lab, line_lower): + return [Do(i, line, filename)], is_doc lower_line = line_lower.strip()[1:] @@ -173,7 +175,7 @@ def get_type(i, filename, line, line_lower, line_lower_canonized, is_doc): "irpf90 may not work with preprocessor directives. You can use" "Irp_if ... Irp_else ... Irp_endif" "instead of" - "#ifdef ... #else ... #endif"%line) + "#ifdef ... #else ... #endif" % line) return result, is_doc if firstword.startswith("case("): @@ -213,9 +215,14 @@ def save_and_execute(irpdir, scriptname, code, interpreter): # Execute shell import util try: - text = util.check_output('PYTHONPATH=$PYTHONPATH:. %s %s' % (interpreter, irpdir_scriptname), shell=True, bufsize=-1, cwd=os.path.join(irpdir,'..')) + text = util.check_output( + 'PYTHONPATH=$PYTHONPATH:. %s %s' % (interpreter, irpdir_scriptname), + shell=True, + bufsize=-1, + cwd=os.path.join(irpdir, '..')) except: - util.logger.error("Something wrong append with embeded '%s' script: %s"% (interpreter, irpdir_scriptname)) + util.logger.error("Something wrong append with embeded '%s' script: %s" % + (interpreter, irpdir_scriptname)) import sys sys.exit(1) @@ -230,35 +237,35 @@ def execute_shell(text): # (List[Line]) -> List[Line] '''Execute the embedded shell scripts''' - - l_begin = [i for i,line in enumerate(text) if isinstance(line,Begin_shell)] - l_end = [i for i,line in enumerate(text) if isinstance(line,End_shell)] - l_output= [] + l_begin = [i for i, line in enumerate(text) if isinstance(line, Begin_shell)] + l_end = [i for i, line in enumerate(text) if isinstance(line, End_shell)] + l_output = [] # ~=~=~=~ # E x e c u t e S h e l l # ~=~=~=~ from util import logger import sys + def fail(l, a, b): - logger.error("%s In Begin_Shell, %s '%s'" % (l,a, b)) + logger.error("%s In Begin_Shell, %s '%s'" % (l, a, b)) sys.exit(1) - for begin,end in zip(l_begin,l_end): + for begin, end in zip(l_begin, l_end): header = text[begin] header_text = header.text for bracket in ['[', ']']: - n = header_text.count(bracket) - assert n <= 1, fail(header_text, "Too many", bracket) - assert n >= 1, fail(header_text, "Missing", bracket) + n = header_text.count(bracket) + assert n <= 1, fail(header_text, "Too many", bracket) + assert n >= 1, fail(header_text, "Missing", bracket) else: - interpreter = header_text[header_text.find('[')+1: header_text.find(']')].strip() - script = ['%s\n' % l.text for l in text[begin+1:end] ] - scriptname="%s_shell_%d" % (header.filename, header.i) + interpreter = header_text[header_text.find('[') + 1:header_text.find(']')].strip() + script = ['%s\n' % l.text for l in text[begin + 1:end]] + scriptname = "%s_shell_%d" % (header.filename, header.i) - l_output.append(save_and_execute(irpdir, scriptname, script,interpreter)) + l_output.append(save_and_execute(irpdir, scriptname, script, interpreter)) # ~=~=~=~ # R e p l a c e @@ -268,12 +275,12 @@ def execute_shell(text): text_new = text[:] # Because we use slicing and we want to include the end line - l_end_include = [i+1 for i in l_end] + l_end_include = [i + 1 for i in l_end] padding = 0 - for begin,end, out in zip(l_begin,l_end_include,l_output): - text_new[begin+padding:end+padding] = out - padding += len(out) - (end-begin) - + for begin, end, out in zip(l_begin, l_end_include, l_output): + text_new[begin + padding:end + padding] = out + padding += len(out) - (end - begin) + return text_new @@ -340,7 +347,11 @@ def execute_templates(text): for v in variables: script += " t0 = t0.replace('%s',d['%s'])\n" % (v, v) script += " print t0\n" - result += save_and_execute(irpdir, scriptname="%s_template_%d" % (line.filename, line.i), code=script,interpreter="python") + result += save_and_execute( + irpdir, + scriptname="%s_template_%d" % (line.filename, line.i), + code=script, + interpreter="python") else: subst += line.text + '\n' @@ -414,24 +425,23 @@ def remove_comments(text, form): def remove_after_bang(str_): # str -> str i_bang = str_.find('!') - - if i_bang == -1: - return str_ - else: - sentinel, inside = None, False - for i,c in enumerate(str_): - if c == '"' or c == "'": - if not inside: - inside = True - sentinel = c - elif sentinel == c: - inside = False - elif c == '!' and not inside: - return str_[:i].strip() - - return str_ - + if i_bang == -1: + return str_ + else: + sentinel, inside = None, False + for i, c in enumerate(str_): + if c == '"' or c == "'": + if not inside: + inside = True + sentinel = c + elif sentinel == c: + inside = False + + elif c == '!' and not inside: + return str_[:i].strip() + + return str_ if form == Free_form: for line in text: @@ -444,7 +454,7 @@ def remove_comments(text, form): if (newline != "" and newline[0] != "!#"): text = remove_after_bang(line.text) if text: - line.text = text + line.text = text result.append(line) return result @@ -544,7 +554,7 @@ def irp_simple_statements(text): def process_return(line): assert type(line) == Return - if command_line.do_assert or command_line.do_debug: + if command_line.do_debug: newline = Simple_line(line.i, " call irp_leave(irp_here)", line.filename) result = [newline, line] else: @@ -600,7 +610,7 @@ def irp_simple_statements(text): def process_end(line): '''Add irp_leave if necessary''' - if command_line.do_assert or command_line.do_debug: + if command_line.do_debug: i = line.i f = line.filename result = [Simple_line(i, " call irp_leave(irp_here)", f), line] @@ -611,11 +621,15 @@ def irp_simple_statements(text): def process_begin_provider(line): assert type(line) == Begin_provider import string - trans = string.maketrans("[]"," ") + trans = string.maketrans("[]", " ") buffer = line.lower.translate(trans).split(',') if len(buffer) < 2: - error.fail(line, "Error in Begin_provider statement") + import sys + print line + print "Error in Begin_provider statement" + sys.exit(1) + varname = buffer[1].strip() length = len(varname) i = line.i @@ -624,7 +638,7 @@ def irp_simple_statements(text): Begin_provider(i, line.text, (f, varname)), Declaration(i, " character*(%d) :: irp_here = '%s'" % (length, varname), f) ] - if command_line.do_assert or command_line.do_debug: + if command_line.do_debug: result += [Simple_line(i, " call irp_enter(irp_here)", f), ] return result @@ -644,10 +658,12 @@ def irp_simple_statements(text): length = len(subname) i = line.i f = line.filename - result = [ line, Declaration(i, " character*(%d) :: irp_here = '%s'" % (length, subname), f)] + result = [ + line, Declaration(i, " character*(%d) :: irp_here = '%s'" % (length, subname), f) + ] - if command_line.do_assert or command_line.do_debug: - result += [Simple_line(i, " call irp_enter_f(irp_here)", f), ] + if command_line.do_debug: + result += [Simple_line(i, " call irp_enter_routine(irp_here)", f), ] return result def process_function(line): @@ -659,27 +675,42 @@ def irp_simple_statements(text): result = [ line, Declaration(i, " character*(%d) :: irp_here = '%s'" % (length, subname), f) ] - if command_line.do_assert or command_line.do_debug: - result += [Simple_line(i, " call irp_enter_f(irp_here)", f), ] + if command_line.do_debug: + result += [Simple_line(i, " call irp_enter_routine(irp_here)", f), ] return result def process_program(line): assert type(line) == Program program_name = line.lower.split()[1] temp = [Program(0, "program irp_program", program_name)] + + if command_line.do_Task: + for i in [" call omp_set_nested(.TRUE.)", "!$omp parallel", "!$omp single"]: + temp += [Simple_line(0, i, line.filename)] + if command_line.do_profile: temp += [Simple_line(0, "call irp_init_timer()", line.filename)] - if command_line.do_openmp: - temp += [Simple_line(0, " call irp_init_locks_%s()" % (irp_id), line.filename)] +# Need to choose between lazy lock or are big full initialization +# if command_line.do_openmp: +# temp += [Simple_line(0, " call irp_init_locks_%s()" % (irp_id), line.filename)] + if command_line.do_debug or command_line.do_assert: + temp += [Simple_line(0, " CALL irp_stack_init", line.filename)] + temp += [Call(0, " call %s" % (program_name), line.filename)] if command_line.do_profile: temp += [Simple_line(0, "call irp_print_timer()", line.filename)] temp += [Simple_line(0, " call irp_finalize_%s()" % (irp_id), line.filename)] + + if command_line.do_Task: + for i in ["!$omp taskwait", "!$omp end single", "!$omp end parallel"]: + temp += [Simple_line(0, i, line.filename)] + temp += [End(0, "end program", line.filename)] result = temp + process_subroutine( Subroutine(line.i, "subroutine %s" % (program_name, ), line.filename)) + return result d = { @@ -696,7 +727,6 @@ def irp_simple_statements(text): Program: process_program, } - result = [] for line in text: buffer = [line] @@ -740,13 +770,14 @@ def process_old_style_do(text): DO 1 i=1,10''' def change_matching_enddo(begin, number): - for i,line in enumerate(text[begin+1:]): - if isinstance(line,(Continue,Enddo)) and line.text.split()[0] == number: - text[begin+1+i] = Enddo(line.i, " enddo", line.filename) - return + for i, line in enumerate(text[begin + 1:]): + if isinstance(line, (Continue, Enddo)) and line.text.split()[0] == number: + text[begin + 1 + i] = Enddo(line.i, " enddo", line.filename) + return from util import logger - logger.error(text[begin], "(%s) Old-style do loops should end with 'continue' or 'end do'" % text[begin]) + logger.error(text[begin], "(%s) Old-style do loops should end with 'continue' or 'end do'" % + text[begin]) from util import sys sys.exit(1) @@ -786,8 +817,8 @@ def change_single_line_ifs(text): buffer = line.text begin = buffer.find('(') if begin == -1: - logger.error("No '(' in if statemnt: %s" % line) - sys.exit(1) + logger.error("No '(' in if statemnt: %s" % line) + sys.exit(1) level = 0 instring = False @@ -812,7 +843,7 @@ def change_single_line_ifs(text): i = line.i f = line.filename result.append(If(i, "%s then" % (test, ), f)) - result += get_type(i, f, code, code.lower(),code.lower(), False)[0] + result += get_type(i, f, code, code.lower(), code.lower(), False)[0] result.append(Endif(i, " endif", f)) else: result.append(line) @@ -831,35 +862,40 @@ def check_begin_end(raw_text): Maybe more of one 'x' statement in defined cause in 'ifdef/else/endif' statement. ''' - d_block = {Enddo: [Do], - Endif: [If], - End_provider: [Begin_provider], - End_doc: [Begin_doc], - End: [Program, Subroutine, Function], - End_module: [Module], - End_interface: [Interface]} + d_block = { + Enddo: [Do], + Endif: [If], + End_provider: [Begin_provider], + End_doc: [Begin_doc], + End: [Program, Subroutine, Function], + End_module: [Module], + End_interface: [Interface] + } from collections import defaultdict d_type = defaultdict(list) for line in raw_text: d_type[type(line)].append(line) - + for t_end, l_begin in d_block.iteritems(): n_end = len(d_type[t_end]) n_begin = sum(len(d_type[t_begin]) for t_begin in l_begin) - + if n_end != n_begin: - - if n_end > n_begin: - logger.error("You have more close statement than open statement (%s) (%s)",line.filename,t_end) - else: - logger.error('You have more end statement than open statenemt for (%s) (%s)' % (line.filename, t_end)) - - for i in zip([l for i in l_begin for l in d_type[i]], d_type[t_end]): + + if n_end > n_begin: + logger.error("You have more close statement than open statement (%s) (%s)", + line.filename, t_end) + else: + logger.error('You have more end statement than open statenemt for (%s) (%s)' % + (line.filename, t_end)) + + for i in zip([l for i in l_begin for l in d_type[i]], d_type[t_end]): logger.debug(i) - sys.exit(1) + sys.exit(1) + ###################################################################### def remove_ifdefs(text): @@ -911,13 +947,13 @@ class Preprocess_text(object): def text(self): with open(self.filename, 'r') as f: str_ = f.read() - - #Dirty thing. We will replace 'end program' by 'end subroutine' - #because afterward the program will be replaced by a subroutine... + +#Dirty thing. We will replace 'end program' by 'end subroutine' +#because afterward the program will be replaced by a subroutine... import re transform = re.compile(re.escape('end program'), re.IGNORECASE) - + return transform.sub('end subroutine', str_) @irpy.lazy_property_mutable @@ -951,7 +987,8 @@ class Preprocess_text(object): result = [] is_doc = False - for i, (l, ll, llc) in enumerate(zip(self.lines, self.lines_lower, self.lines_lower_canonized)): + for i, (l, ll, + llc) in enumerate(zip(self.lines, self.lines_lower, self.lines_lower_canonized)): line, is_doc = get_type(i + 1, self.filename, l, ll, llc, is_doc) result += line return result @@ -978,6 +1015,5 @@ class Preprocess_text(object): return result - if __name__ == '__main__': debug() diff --git a/src/profile.py b/src/profile.py index 02094e2..77344c2 100644 --- a/src/profile.py +++ b/src/profile.py @@ -24,7 +24,6 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - rdtsc = """ #ifdef __i386 double irp_rdtsc_(void) { @@ -47,14 +46,16 @@ import os import threading from irpf90_t import irpdir + def build_rdtsc(): - filename = irpdir+"irp_rdtsc.c" - file = open(filename,'w') - file.write(rdtsc) - file.close() + filename = irpdir + "irp_rdtsc.c" + file = open(filename, 'w') + file.write(rdtsc) + file.close() + def build_module(variables): - data = """ + data = """ module irp_timer double precision :: irp_profile(3,%(n)d) integer :: irp_order(%(n)d) @@ -153,23 +154,23 @@ subroutine irp_print_timer() print *, 'rdtsc latency :', irp_rdtsc_shift, ' cycles' end """ - label = {} - for i in variables: - vi = variables[i] - label[vi.label] = vi.same_as - text = [] - lmax = 0 - for l in label: - text.append(" irp_profile_label(%d) = '%s'"%(l,label[l])) - lmax = max(lmax,l) - text.sort() - text = '\n'.join(text) - data = data%{'text': text, 'n':lmax} - file = open("IRPF90_temp/irp_profile.irp.F90",'w') - file.write(data) - file.close() + label = {} + for i in variables: + vi = variables[i] + label[vi.label] = vi.same_as + text = [] + lmax = 0 + for l in label: + text.append(" irp_profile_label(%d) = '%s'" % (l, label[l])) + lmax = max(lmax, l) + text.sort() + text = '\n'.join(text) + data = data % {'text': text, 'n': lmax} + file = open("IRPF90_temp/irp_profile.irp.F90", 'w') + file.write(data) + file.close() + def run(d_entity): - build_module(d_entity) - build_rdtsc() - + build_module(d_entity) + build_rdtsc() diff --git a/src/regexps.py b/src/regexps.py index 61d77e1..c78e83e 100644 --- a/src/regexps.py +++ b/src/regexps.py @@ -24,34 +24,18 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - import re re_comment = re.compile(r"^([^'!]*)('[^']*'[^']*)*!") -re_decl = re.compile( "".join( [ r"^\ *", - r"(integer[(::)?\* ,]+", - r"|double *precision[(::)?\* ,]+", - r"|logical[(::)?\* ,]+", - r"|character[(::)?\* ,]+", - r"|real[(::)?\* ,]+", - r"|dimension[(::)?\* ,]+", - r"|parameter[(::)?\* ,]+", - r"|data */", - r"|allocatable *(::)?", - r"|common */", - r"|namelist */", - r"|save */", - r"|complex[(::)?\* ,]+", - r"|intrinsic *(::)?", - r"|external *(::)?", - r"|equivalence *(::)?", - r"|type", - r"|endtype", - r")[^=(]" -] ) ) +re_decl = re.compile("".join([ + r"^\ *", r"(integer[(::)?\* ,]+", r"|double *precision[(::)?\* ,]+", r"|logical[(::)?\* ,]+", + r"|character[(::)?\* ,]+", r"|real[(::)?\* ,]+", r"|dimension[(::)?\* ,]+", + r"|parameter[(::)?\* ,]+", r"|data */", r"|allocatable *(::)?", r"|common */", r"|namelist */", + r"|save */", r"|complex[(::)?\* ,]+", r"|intrinsic *(::)?", r"|external *(::)?", + r"|equivalence *(::)?", r"|type", r"|endtype", r")[^=(]" +])) -re_test = re.compile(r"\( *(.*)(\.[a-zA-Z]*\.|[<>]=?|[=/]=)([^=]*)\)") +re_test = re.compile(r"\( *(.*)(\.[a-zA-Z]*\.|[<>]=?|[=/]=)([^=]*)\)") re_string = re.compile(r"'.*?'") - diff --git a/src/routine.py b/src/routine.py index 8f31b49..6b69a77 100644 --- a/src/routine.py +++ b/src/routine.py @@ -24,81 +24,80 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - from irpf90_t import * from util import logger +from lib.manager import irpy -try: - import irpy -except: - import lib_irpy as irpy -class Routine(object): - ''' +class Routine(object): + ''' A collection of list corresponding of a Routine (Subroutine, or function) ''' + ############################################################ + def __init__(self, text): + assert type(text) == list + assert len(text) > 0 - ############################################################ - def __init__(self,text): - assert type(text) == list - assert len(text) > 0 + self.text = text + self.prototype = self.text[0] + assert isinstance(self.prototype, (Subroutine, Function)) - self.text = text - self.prototype = self.text[0] - assert isinstance(self.prototype, (Subroutine, Function)) - - ############################################################ - @irpy.lazy_property_mutable - def called_by(self): - raise AttributeError - - ############################################################ - @irpy.lazy_property - def name(self): - '''Name is lowercase''' - return self.prototype.subname - - ############################################################ - @irpy.lazy_property - def is_function(self): - return "function" in self.prototype.lower - - ############################################################ - @irpy.lazy_property - def is_subroutine(self): - return "subroutine" in self.prototype.lower - - ############################################################ - @irpy.lazy_property - def doc(self): - - l_doc = [ l for l in self.text if isinstance(l,Doc) ] - if not l_doc: - logger.info("Subroutine '%s' is not documented"%(self.name)) - return [l.text.lstrip()[1:] for l in l_doc] - - ############################################################ - @irpy.lazy_property - def touches_my_self(self): - return set(x for line in self.text for x in line.text.split()[1:] if isinstance(line,(Touch, SoftTouch))) - - @irpy.lazy_property_mutable - def touches_ancestor(self): + ############################################################ + @irpy.lazy_property_mutable + def called_by(self): raise AttributeError - @irpy.lazy_property - def touches(self): +############################################################ + + @irpy.lazy_property + def name(self): + '''Name is lowercase''' + return self.prototype.subname + + ############################################################ + @irpy.lazy_property + def is_function(self): + return "function" in self.prototype.lower + + ############################################################ + @irpy.lazy_property + def is_subroutine(self): + return "subroutine" in self.prototype.lower + + ############################################################ + @irpy.lazy_property + def doc(self): + + l_doc = [l for l in self.text if isinstance(l, Doc)] + if not l_doc: + logger.info("Subroutine '%s' is not documented" % (self.name)) + return [l.text.lstrip()[1:] for l in l_doc] + + ############################################################ + @irpy.lazy_property + def touches_my_self(self): + return set(x for line in self.text for x in line.text.split()[1:] + if isinstance(line, (Touch, SoftTouch))) + + @irpy.lazy_property_mutable + def touches_ancestor(self): + raise AttributeError + + @irpy.lazy_property + def touches(self): return list(self.touches_my_self.union(self.touches_ancestor)) - ############################################################ - @irpy.lazy_property - def regexp(self): - import re - return re.compile(r"([^a-z0-9'\"_]|^)%s([^a-z0-9_]|$)"%(self.name),re.I) + ############################################################ + @irpy.lazy_property + def regexp(self): + import re + return re.compile(r"([^a-z0-9'\"_]|^)%s([^a-z0-9_]|$)" % (self.name), re.I) - ############################################################ - @irpy.lazy_property - def calls(self): - return set(line.text.split('(',1)[0].split()[1].lower() for line in self.text if isinstance(line,Call)) + ############################################################ + @irpy.lazy_property + def calls(self): + return set( + line.text.split('(', 1)[0].split()[1].lower() for line in self.text + if isinstance(line, Call)) diff --git a/src/templates/allocater.f90 b/src/templates/allocater.f90 new file mode 100644 index 0000000..fa50bbe --- /dev/null +++ b/src/templates/allocater.f90 @@ -0,0 +1,50 @@ +{?dim} +SUBROUTINE allocate_{name} + + {#l_module} + USE {.} + {/l_module} + + IMPLICIT NONE + + CHARACTER*(9+{@size key=name/}),PARAMETER :: irp_here = 'allocate_{name}' + INTEGER :: irp_err + LOGICAL :: alloc + + alloc = ALLOCATED({name}) + + IF ( alloc .AND.( & + {#l_dim} + ( SIZE({name},{rank}) == {value} ) {@sep}.OR.{/sep} & + {/l_dim})) THEN + + RETURN + ELSE IF (.NOT.alloc) THEN + GO TO 666 + ELSE + {?do_memory} PRINT*, irp_here//': Deallocated {name}' {/do_memory} + DEALLOCATE({name},STAT=irp_err) + + IF (irp_err /= 0) THEN + PRINT*, irp_here//': Deallocation failed: {name}' + PRINT*,' size: {dim}' + ENDIF + + GO TO 666 + ENDIF + + 666 CONTINUE + {?do_memory} PRINT*, irp_here//': Allocate {name} ({dim})'{/do_memory} + + {^do_corray} + ALLOCATE({name} ({dim}), STAT=irp_err) + {:else} + ALLOCATE({name} ({dim}[*]), STAT=irp_err) + {/do_corray} + + IF (irp_err /= 0) then + PRINT*, irp_here//': Allocation failed: {name}' + PRINT*,' size: {dim}' + ENDIF +END SUBROUTINE allocate_{name} +{/dim} diff --git a/src/templates/finalize.f90 b/src/templates/finalize.f90 new file mode 100644 index 0000000..fa4c3f3 --- /dev/null +++ b/src/templates/finalize.f90 @@ -0,0 +1,13 @@ +SUBROUTINE irp_finalize_{id} + {#use} + USE {.} + {/use} + + IMPLICIT NONE + {#entity_array} + IF (ALLOCATED({name})) THEN + {name_root}_is_built = .FALSE. + DEALLOCATE({name}) + ENDIF + {/entity_array} +END SUBROUTINE irp_finalize_{id} diff --git a/src/templates/general.make b/src/templates/general.make new file mode 100644 index 0000000..e3668fe --- /dev/null +++ b/src/templates/general.make @@ -0,0 +1,94 @@ +# make | ninja +############## + +BUILD_SYSTEM= {BUILD_SYSTEM} + +####### + +IRPF90= irpf90 +IRPF90FLAGS= {irpf90_flags} + +.EXPORT_ALL_VARIABLES: + +LIB = {LIB} +SRC = {SRC} +OBJ = {OBJ} + +# Compiler ! Will be overwriten by the ENV one if avalaible. +FC ?= {FC} +FCFLAGS ?= {FCFLAGS} + +CC ?= {CC} +CFLAGS ?= {CFLAGS} +CXX ?= {CXX} +CXXFLAGS ?= {CXXFLAGS} + +# Dark magic below modify with caution! +# "You are Not Expected to Understand This" +# . +# /^\ . +# /\ "V", +# /__\ I O o +# //..\\ I . +# \].`[/ I +# /l\/j\ (] . O +# /. ~~ ,\/I . +# \\L__j^\/I o +# \/--v} I o . +# | | I _________ +# | | I c(` ')o +# | l I \. ,/ +# _/j L l\_! _// \\_ + +#Misc +AR ?= {ar} +RANLIB ?= {ranlib} + +# Variable need by IRPF90 +BUILD_SYSTEM_stripped=$(strip $(BUILD_SYSTEM)) +ifeq ($(BUILD_SYSTEM_stripped),ninja) + BUILD_FILE=IRPF90_temp/build.ninja + IRPF90FLAGS += -j +else ifeq ($(BUILD_SYSTEM_stripped),make) + BUILD_FILE=IRPF90_temp/build.make + BUILD_SYSTEM += -j +else +DUMMY: + $(error 'Wrong BUILD_SYSTEM: $(BUILD_SYSTEM)') +endif + +# Actual Rule +EXE := $(shell egrep -ri '^\s*program' *.irp.f | cut -d'.' -f1) +ARCH = $(addprefix $(CURDIR)/,IRPF90_temp/irpf90.a) + +.PHONY: clean all + +all: $(EXE) + +define run + $(BUILD_SYSTEM_stripped) -C $(dir $(BUILD_FILE) ) -f $(notdir $(BUILD_FILE) ) $(1) +endef + +#We allow for the user to ask for 'relative' path +#But the IRPF90 build system use absolute path. +$(EXE): $(ARCH) + @printf "%b" "\033[0;32m Build $@...\033[m\n" + @$(call run, $(addprefix $(CURDIR)/, $@)) && touch $@ + + +.NOTPARALLEL: $(BUILD_FILE) $(ARCH) + +$(ARCH): $(BUILD_FILE) + @printf "%b" "\033[0;32m Creating the archive...\033[m\n" + @$(call run, $@) && touch $@ + +$(BUILD_FILE): $(shell find . -maxdepth 2 -path ./IRPF90_temp -prune -o -name '*.irp.f' -print) + @printf "%b" "\033[0;32m Running the IRPF90-compiler...\033[m\n" + @$(IRPF90) $(IRPF90FLAGS) + +clean: + rm -f -- $(BUILD_FILE) $(EXE) + +veryclean: clean + rm -rf IRPF90_temp/ IRPF90_man/ irpf90_entities dist tags + diff --git a/src/templates/ioer.f90 b/src/templates/ioer.f90 new file mode 100644 index 0000000..2fde02c --- /dev/null +++ b/src/templates/ioer.f90 @@ -0,0 +1,71 @@ +SUBROUTINE writer_{name}(irp_num) + + USE {fmodule} + IMPLICIT NONE + + CHARACTER*(*), INTENT(IN) :: irp_num + LOGICAL :: irp_is_open = .TRUE. + INTEGER :: irp_iunit = 9 + + {?do_debug} + CHARACTER*(7+{@size key=name/}),PARAMETER :: irp_here = 'writer_{name}' + {/do_debug} + + {?do_debug} CALL irp_enter(irp_here) {/do_debug} + + IF (.NOT.{same_as}_is_built) THEN + CALL provide_{same_as} + ENDIF + + {#children} + CALL writer_{.}(irp_num) + {/children} + + DO WHILE (irp_is_open) + irp_iunit = irp_iunit + 1 + INQUIRE(UNIT=irp_iunit, OPENED=irp_is_open) + END DO + + {#group_entity} + OPEN(UNIT=irp_iunit,file='irpf90_{name}_'//trim(irp_num),FORM='FORMATTED',STATUS='UNKNOWN',ACTION='WRITE') + WRITE(irp_iunit,*) {name}{dim} + CLOSE(irp_iunit) + {/group_entity} + + {?do_debug} CALL irp_leave(irp_here) {/do_debug} + +END SUBROUTINE writer_{name} + +!TOKEN_SPLIT + +SUBROUTINE reader_{name}(irp_num) + + USE {fmodule} + IMPLICIT NONE + + CHARACTER*(*), INTENT(IN) :: irp_num + LOGICAL :: irp_is_open = .TRUE. + INTEGER :: irp_iunit = 9 + + {?do_debug} + CHARACTER*(5+{@size key=name/}),PARAMETER :: irp_here = 'read_{name}' + {/do_debug} + + {?do_debug} CALL irp_enter(irp_here) {/do_debug} + + DO WHILE (irp_is_open) + irp_iunit = irp_iunit + 1 + INQUIRE(UNIT=irp_iunit, OPENED=irp_is_open) + END DO + + {#group_entity} + OPEN(UNIT=irp_iunit,file='irpf90_{name}_'//trim(irp_num),FORM='FORMATTED',STATUS='OLD',ACTION='READ') + READ(irp_iunit,*) {name}{dim} + CLOSE(irp_iunit) + {/group_entity} + + CALL touch_{same_as} + {?do_debug} CALL irp_leave(irp_here) {/do_debug} + +END SUBROUTINE reader_{name} + diff --git a/src/templates/irp_lock.f90 b/src/templates/irp_lock.f90 new file mode 100644 index 0000000..a7c7ca1 --- /dev/null +++ b/src/templates/irp_lock.f90 @@ -0,0 +1,31 @@ +{#entity} +SUBROUTINE irp_lock_{.}(set) + + USE omp_lib + IMPLICIT NONE + LOGICAL, INTENT(in) :: set + INTEGER(KIND=omp_lock_kind),SAVE :: {.}_lock + INTEGER, SAVE :: ifirst = 0 + + {?do_debug} + CHARACTER*(9+{@size key={.}/}),PARAMETER :: irp_here = 'irp_lock_{name}' + {/do_debug} + + {?do_debug} CALL irp_enter(irp_here) {/do_debug} + + IF (ifirst == 0) then + ifirst = 1 + CALL omp_init_lock({.}_lock) + ENDIF + + IF (set) THEN + CALL omp_set_lock({.}_lock) + ELSE + CALL omp_unset_lock({.}_lock) + ENDIF + + {?do_debug} CALL irp_leave(irp_here) {/do_debug} + +END SUBROUTINE irp_lock_{.} +{/entity} + diff --git a/src/templates/irp_stack.f90 b/src/templates/irp_stack.f90 new file mode 100644 index 0000000..6c72a29 --- /dev/null +++ b/src/templates/irp_stack.f90 @@ -0,0 +1,151 @@ +MODULE irp_stack_mod + INTEGER, PARAMETER :: STACKMAX=1000 + CHARACTER*(128) ,allocatable :: irp_stack(:,:) + DOUBLE PRECISION ,allocatable :: irp_cpu(:,:) + INTEGER ,allocatable :: stack_index(:) + INTEGER :: nthread + CHARACTER*(128) :: white = '' +END MODULE + +SUBROUTINE irp_stack_init + USE irp_stack_mod + + IMPLICIT NONE + + INTEGER :: ithread + {?do_openmp} + INTEGER, EXTERNAL :: omp_get_thread_num + INTEGER, EXTERNAL :: omp_get_max_threads + {/do_openmp} + INTEGER :: ierr + {^do_openmp} + ithread = 0 + {:else} + ithread = omp_get_thread_num() + {/do_openmp} + + {^do_openmp} !$OMP CRITICAL {/do_openmp} + IF (.NOT.ALLOCATED(stack_index) ) THEN + + {^do_openmp} + nthread = 1 + {:else} + nthread = omp_get_max_threads() + {/do_openmp} + + {?do_memory} + print *, 'Allocating irp_stack(0:',STACKMAX,',0:',nthread,')' + print *, 'Allocating irp_cpu(0:',STACKMAX,',0:',nthread,')' + print *, 'Allocating stack_index(0:',nthread,')' + {/do_memory} + + ALLOCATE ( irp_stack(0:STACKMAX, 0:nthread), & + irp_cpu(0:STACKMAX, 0:nthread), & + stack_index(0:nthread) ) + IF (ierr /=0 ) THEN + print*, 'Failed Allocating irp_stack, irp_cpu, stack_index' + ENDIF + stack_index = 0 + END IF + +END SUBROUTINE + +SUBROUTINE irp_enter_routine(irp_where) + USE irp_stack_mod + + IMPLICIT NONE + + CHARACTER*(*), INTENT(in) :: irp_where + INTEGER :: ithread + REAL :: cpu + {?do_openmp} + INTEGER, EXTERNAL :: omp_get_thread_num + {/do_openmp} + + {^do_openmp} + ithread = 0 + {:else} + ithread = omp_get_thread_num() + {/do_openmp} + + stack_index(ithread) = min(stack_index(ithread)+1,STACKMAX) + irp_stack(stack_index(ithread),ithread) = irp_where + +END SUBROUTINE irp_enter_routine + +SUBROUTINE irp_enter(irp_where) + USE irp_stack_mod + + IMPLICIT NONE + + CHARACTER*(*), INTENT(in) :: irp_where + INTEGER :: ithread + {?do_openmp} + INTEGER, EXTERNAL :: omp_get_thread_num + {/do_openmp} + + {^do_openmp} + ithread = 0 + {:else} + ithread = omp_get_thread_num() + {/do_openmp} + + print *, ithread, ':', white(1:stack_index(ithread))//'-> ', trim(irp_where) + CALL cpu_time(irp_cpu(stack_index(ithread),ithread)) +END SUBROUTINE irp_enter + + +SUBROUTINE irp_leave(irp_where) + USE irp_stack_mod + + IMPLICIT NONE + + CHARACTER*(*), INTENT(in) :: irp_where + INTEGER :: ithread + REAL :: cpu + {?do_openmp} + INTEGER, EXTERNAL :: omp_get_thread_num + {/do_openmp} + + {^do_openmp} + ithread = 0 + {:else} + ithread = omp_get_thread_num() + {/do_openmp} + + CALL cpu_time(cpu) + print *, ithread, ':', white(1:stack_index(ithread))//'<- ', & + trim(irp_stack(stack_index(ithread),ithread)), & + cpu - irp_cpu(stack_index(ithread),ithread) + + stack_index(ithread) = max(0,stack_index(ithread)-1) + +END SUBROUTINE irp_leave + + +SUBROUTINE irp_trace + USE irp_stack_mod + + IMPLICIT NONE + + INTEGER :: ithread + {?do_openmp} + INTEGER, EXTERNAL :: omp_get_thread_num + {/do_openmp} + INTEGER :: i + + {^do_openmp} + ithread = 0 + {:else} + ithread = omp_get_thread_num() + {/do_openmp} + + print *, 'Stack trace: ', ithread + print *, '-------------------------' + DO i=1,stack_index(ithread) + print *, trim(irp_stack(i,ithread)) + END DO + print *, '-------------------------' + +END SUBROUTINE irp_trace + diff --git a/src/templates/module.f90 b/src/templates/module.f90 new file mode 100644 index 0000000..0fcf15d --- /dev/null +++ b/src/templates/module.f90 @@ -0,0 +1,39 @@ +! -*- F90 -*- +! +!-----------------------------------------------! +! This file was generated with the irpf90 tool. ! +! ! +! DO NOT MODIFY IT BY HAND ! +!-----------------------------------------------! + +MODULE {name} + +{#use} + USE {.} +{/use} + +{#usr_declaration} + {.} +{/usr_declaration} + +{#irp_declaration} + + {^dim} + {type} {?protected}, PROTECTED {/protected} :: {name} {?coarray} [*] {/coarray} + {:else} + + {?align} !DIR$ ATTRIBUTES ALIGN: {align} :: {name} {/align} + {type} {?protected}, PROTECTED {/protected} :: {name} {dim} {?coarray} [:] {/coarray} + {/dim} + + {?main} + LOGICAL :: {name}_is_built = .FALSE. + {/main} +{/irp_declaration} + + CONTAINS + {#irp_declaration} +{protected|s} + {/irp_declaration} + +END MODULE {name} diff --git a/src/templates/provider.f90 b/src/templates/provider.f90 new file mode 100644 index 0000000..ea593db --- /dev/null +++ b/src/templates/provider.f90 @@ -0,0 +1,45 @@ +{?inline}!DEC$ ATTRIBUTES FORCEINLINE :: provide_{name}{/inline} +SUBROUTINE provide_{name} + + {#l_module} + USE {.} + {/l_module} + + IMPLICIT NONE + + {?do_debug} + CHARACTER*(8+{@size key=name/}),PARAMETER :: irp_here = 'provide_{name}' + {/do_debug} + + {?do_debug} CALL irp_enter(irp_here) {/do_debug} + + {?do_openmp} + CALL irp_lock_{name}(.TRUE.) + IF (.NOT.{name}_is_built) THEN + {/do_openmp} + + {#l_children_static} + {@first} {?do_task}!$OMP TASKGROUP{/do_task} {/first} + {?do_openmp}!$OMP flush({.}_is_built){/do_openmp} + IF (.NOT.{.}_is_built) THEN + {?do_task}!$OMP TASK{/do_task} + CALL provide_{.} + {?do_task}!$OMP END TASK{/do_task} + ENDIF + {@last} {?do_task}!$OMP END TASKGROUP{/do_task} {/last} + {/l_children_static} + + {#l_entity} + {?dim} CALL allocate_{name} {/dim} + {/l_entity} + + CALL bld_{name} + {name}_is_built = .TRUE. + + {?do_openmp} + ENDIF + CALL irp_lock_{name}(.FALSE.) + {/do_openmp} + + {?do_debug} CALL irp_leave(irp_here) {/do_debug} +END SUBROUTINE provide_{name} diff --git a/src/templates/touch.f90 b/src/templates/touch.f90 new file mode 100644 index 0000000..ef49250 --- /dev/null +++ b/src/templates/touch.f90 @@ -0,0 +1,26 @@ +{#entity} + +SUBROUTINE touch_{name} + + {#l_module} + USE {.} + {/l_module} + + IMPLICIT NONE + {?do_debug} + CHARACTER*(6+{@size key=name/}),PARAMETER :: irp_here = 'touch_{name}' + {/do_debug} + + {?do_debug} CALL irp_enter(irp_here) {/do_debug} + + {#l_ancestor} + {.}_is_built = .FALSE. + {/l_ancestor} + + {name}_is_built = .TRUE. + + {?do_debug} CALL irp_leave(irp_here) {/do_debug} + +END SUBROUTINE touch_{name} + +{/entity} diff --git a/src/touches.py b/src/touches.py index b2533bf..fce01f8 100644 --- a/src/touches.py +++ b/src/touches.py @@ -24,42 +24,40 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr -from irpf90_t import irp_id,irpdir +from irpf90_t import irp_id, irpdir import os -from util import lazy_write_file +from command_line import command_line -def create(modules,variables): - # (Dict[str,Module]. Dict[str, Variable]) -> None - '''Create the fortran90 finalize subroutine and the touched one''' - - finalize = "subroutine irp_finalize_%s\n"%(irp_id) - for m in filter(lambda x: not modules[x].is_main and modules[x].has_irp_module, modules): - finalize += " use %s\n"%(modules[m].name) +def create(modules, variables): + # (Dict[str,Module]. Dict[str, Variable]) -> None + '''Create the fortran90 finalize subroutine and the touched one''' - main_modules_name =[ m.name for m in modules.values() if m.is_main] - - out = [] - for v,var in variables.iteritems(): + main_modules_name = [m.name for m in modules.values() if m.is_main] - if var.fmodule not in main_modules_name: - #if var.is_self_touched: - out += var.toucher - if var.dim: - finalize += " if (allocated(%s)) then\n"%v - finalize += " %s_is_built = .False.\n"%var.same_as - finalize += " deallocate(%s)\n"%v - finalize += " endif\n" + d_template_finalize = { + 'id': irp_id, + 'use': [m.name for m in modules.values() if not m.is_main and m.has_irp_module], + 'entity_array': [{ + 'name': e.name, + 'name_root': e.same_as + } for e in variables.values() if e.fmodule not in main_modules_name and e.dim] + } - finalize += "end\n" + d_template_touch = { + 'do_debug': command_line.do_debug, + 'entity': [ + e.d_touche_template for e in variables.values() + if e.fmodule not in main_modules_name and e.d_touche_template + ] + } + import util + str_out = util.ashes_env.render('touch.f90', d_template_touch) + util.ashes_env.render( + 'finalize.f90', d_template_finalize) - if out: - out = map(lambda x: "%s\n"%(x),out) + filename = os.path.join(irpdir, 'irp_touches.irp.F90') + util.lazy_write_file(filename, '%s\n' % util.remove_empy_lines(str_out)) - out += finalize - - filename=os.path.join(irpdir,'irp_touches.irp.F90') - lazy_write_file(filename,''.join(out)) if __name__ == '__main__': - create() + create() diff --git a/src/util.py b/src/util.py index 64609f8..ae9b4dd 100644 --- a/src/util.py +++ b/src/util.py @@ -24,7 +24,6 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - # ~#~#~#~#~# # L o g e r # ~#~#~#~#~# @@ -42,19 +41,33 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger('Irpf90') logger.setLevel(30) +# ~#~#~#~#~# +# A S H E S _ T E M P L A T E S +# ~#~#~#~#~# +from lib.manager import ashes +import os +ashes_env = ashes.AshesEnv([os.path.join(os.path.dirname(__file__), 'templates')]) + + +def remove_empy_lines(text): + return os.linesep.join([s for s in text.splitlines() if s.strip()]) + # ~#~#~#~#~# # / / _ R E L A T E D # ~#~#~#~#~# -def chunkify(l,n_chunk): + +def chunkify(l, n_chunk): # (List[any], int) -> List [ List[any] ] '''Split the list on n_chunk''' len_ = len(l) - n = max(1, len_ / n_chunk ) - return [ l[i:i + n] for i in xrange(0, len_, n) ] + n = max(1, len_ / n_chunk) + return [l[i:i + n] for i in xrange(0, len_, n)] import multiprocessing + + def parmap(f, it, parallel=False): # (Callable, Iterable, bool) -> List '''Parallel version of the std map function @@ -70,7 +83,7 @@ def parmap(f, it, parallel=False): ''' if not parallel: - return map(f, it) + return map(f, it) nproc = multiprocessing.cpu_count() @@ -98,12 +111,12 @@ def parmap(f, it, parallel=False): # In this implementation, we minimizise the communication # (aka 1 job by processor) - it_chunk = chunkify(l=it,n_chunk=nproc) + it_chunk = chunkify(l=it, n_chunk=nproc) + def F(chunk): # (List[any]) -> (List[any]) '''Same as 'f' but for a chunck''' - return map(f,chunk) - + return map(f, chunk) q_in = multiprocessing.JoinableQueue() q_out = multiprocessing.Queue() @@ -112,8 +125,8 @@ def parmap(f, it, parallel=False): stop_condition = None def worker(): - # () -> None - '''Read a task from q_in, excute it, and store it in q_out + # () -> None + '''Read a task from q_in, excute it, and store it in q_out Note: - We use 'F' and not 'f'. @@ -121,19 +134,19 @@ def parmap(f, it, parallel=False): - We get, and put an idx to allow the possibility of ordering afterward - We store any exeception, to raise her afterward ''' - for i, x in iter(q_in.get, stop_condition): + for i, x in iter(q_in.get, stop_condition): try: result = F(x) except BaseException as e: t = e - else: + else: t = (i, result) q_out.put(t) q_in.task_done() - q_in.task_done() + q_in.task_done() # Process' creation l_proc = [multiprocessing.Process(target=worker) for _ in range(nproc)] @@ -152,15 +165,15 @@ def parmap(f, it, parallel=False): q_in.join() # Get all the chunk and join the process - l_res = [q_out.get() for _ in range(len(it_chunk))] + l_res = [q_out.get() for _ in range(len(it_chunk))] for p in l_proc: p.join() - + # Check if error have occured try: from itertools import ifilter - e = next(ifilter(lambda t: isinstance(t,BaseException), l_res)) + e = next(ifilter(lambda t: isinstance(t, BaseException), l_res)) except StopIteration: # Now we need first to order the result, and secondly to flatte it return [item for _, chunk in sorted(l_res) for item in chunk] @@ -172,6 +185,8 @@ def parmap(f, it, parallel=False): # ~#~#~#~#~# import hashlib import os + + def cached_file(filename, text): # (str,str) -> bool '''Check if file locatte at filename containt the same data as text @@ -183,7 +198,7 @@ def cached_file(filename, text): def digest(data): # (str) -> str '''compute an uniq data id''' - return hashlib.md5(data).hexdigest() + return hashlib.md5(data).hexdigest() try: text_ref = open(filename, 'rb').read() @@ -193,7 +208,7 @@ def cached_file(filename, text): return digest(text_ref) == digest(text) -def lazy_write_file(filename, text, conservative=False,touch=False): +def lazy_write_file(filename, text, conservative=False, touch=False): # (str, str, bool) -> None '''Write data lazily in filename location. @@ -205,7 +220,8 @@ def lazy_write_file(filename, text, conservative=False,touch=False): with open(filename, 'w') as f: f.write(text) elif touch: - os.utime(filename,None) + os.utime(filename, None) + def listdir(directory, abspath=False): #(str, bool) -> List[str] @@ -217,6 +233,7 @@ def listdir(directory, abspath=False): else: return [os.path.abspath(os.path.join(directory, f)) for f in l_filename] + def check_output(*popenargs, **kwargs): """Run command with arguments and return its output as a byte string. Backported from Python 2.7 as it's implemented as pure python on stdlib. @@ -241,22 +258,31 @@ def check_output(*popenargs, **kwargs): # ~#~#~#~#~# -def uniquify(l,sort=False): +def uniquify(l, sort=False): # (Iter, bool) -> List[Any] - '''Uniquify a immutable iterable. Don't preserve the order''' - r = list(set(l)) + '''Uniquify a immutable iterable. Don't preserve the order. Or maybe.''' + + #Be carefull that element in Iter can be unshable. + try: + r = list(set(l)) + except TypeError: + used = list() + r = [x for x in l if x not in used and (used.append(x) or True)] + if not sort: return r else: return sorted(r) + def OrderedUniqueList(l): # (Iter, bool) -> List[Any] '''Uniquify a immutable iterable. Don't preserve the order''' return sorted(set(l)) + def flatten(l_2d): - # (List [ List[Any] ]) -> List + # (List [ Iter[Any] ]) -> List '''Construct a copy of the 2d list collapsed into one dimension. Note: @@ -269,6 +295,28 @@ def flatten(l_2d): # ~#~#~#~#~# # I R P _ R E L A T E D # ~#~#~#~#~# +def dimsize(x): + # (str) -> str + '''Compute the number of element in the array''' + try: + b0, b1 = x.split(':') + except ValueError: + return x + + b0_is_digit = b0.replace('-', '').isdigit() + b1_is_digit = b1.replace('-', '').isdigit() + + if b0_is_digit and b1_is_digit: + size = str(int(b1) - int(b0) + 1) + elif b0_is_digit: + size = "(%s) - (%d)" % (b1, int(b0) - 1) + elif b1_is_digit: + size = "(%d) - (%s)" % (int(b1) + 1, b0) + else: + size = "(%s) - (%s) + 1" % (b1, b0) + return size + + def build_dim(l_dim, colons=False): # (List[str],bool) -> str '''Contruct a valid fortran90 array dimension code from a list dimension @@ -286,20 +334,71 @@ def build_dim(l_dim, colons=False): return "(%s)" % (",".join(l_dim_colons)) -def build_use(l_ent, d_ent): +def mangled(l_ent, d_ent): + # (List, Dict[str,Entity]) -> list + '''Create a uniq list of providier (merge the multione) ''' + return OrderedUniqueList(d_ent[name].same_as for name in l_ent) + + +def build_use(l_ent, d_ent, use=True): # (List, Dict[str,Entity]) -> list '''Contruct the fortran90 'use' statement for the list of entity''' - return OrderedUniqueList(" use %s" % d_ent[x].fmodule for x in l_ent) + + l_name = OrderedUniqueList(d_ent[x].fmodule for x in l_ent) + if not use: + return l_name + else: + return [" use %s" % n for n in l_name] + def build_call_provide(l_ent, d_ent): # (List, Dict[str,Entity]) -> list '''Construct the fortran 90 call the provider needed by the list of entity''' - def fun(x): - return [ " if (.not.%s_is_built) then" % x, - " call provide_%s" % x, - " endif"] # Get the corect name (in the case of multiple provider line) - l_same_as = OrderedUniqueList(d_ent[x].same_as for x in l_ent) - return flatten(map(fun, l_same_as)) + l_same_as = mangled(l_ent, d_ent) + def bld_f90(x): + return [" if (.not.%s_is_built) then" % x, " call provide_%s" % x, " endif"] + + return flatten(map(bld_f90, l_same_as)) + + +def che_merge(sets): + #(List[Set] -> List[Set] + """Merge a list of set is they are not disjoint. + Note: + This will destry sets + """ + results = [] + upd, isd, pop = set.update, set.isdisjoint, sets.pop + while sets: + if not [ + upd(sets[0], pop(i)) for i in range(len(sets) - 1, 0, -1) + if not isd(sets[0], sets[i]) + ]: + results.append(pop(0)) + return results + + +def l_dummy_entity(d_entity): + # Dict[str:Entity] -> List[set] + from itertools import combinations + l_candidate_botom = [(i, j) for i, j in combinations(d_entity.keys(), 2) + if d_entity[i].needs == d_entity[j].needs] + l_dummy = [ + set([i, j]) for i, j in l_candidate_botom if d_entity[i].needed_by == d_entity[j].needed_by + ] + + return che_merge(l_dummy) + + +def split_l_set(l_set_org): + #(List[set] -> (List, Set) + '''Split the list of set into a list of Head and and the concetenad of all the tail + + Note: Head and Tail a not defined in set. Head in one element of the set, and tail the rest. + ''' + l_set = [set(s) for s in l_set_org] + l_main = [s.pop() for s in l_set] + return l_main, set(flatten(l_set)) diff --git a/src/vim.py b/src/vim.py index b0d6778..c97be6e 100644 --- a/src/vim.py +++ b/src/vim.py @@ -24,25 +24,26 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - import os + def install(): - VIM = os.environ["HOME"]+"/.vim" - try: - if os.access(VIM+"/syntax/irpf90.vim",os.F_OK): - return - if not os.access(VIM,os.F_OK): - os.mkdir(VIM) - file = open(VIM+"/filetype.vim","a") - file.write("au BufRead,BufNewFile *.irp.f setfiletype irpf90") - file.close() - if not os.access(VIM+"/syntax",os.F_OK): - os.mkdir(VIM+"/syntax") - wd = os.path.abspath(os.path.dirname(__file__)) - os.symlink(wd+"/irpf90.vim",VIM+"/syntax/irpf90.vim") - except: - pass + VIM = os.environ["HOME"] + "/.vim" + try: + if os.access(VIM + "/syntax/irpf90.vim", os.F_OK): + return + if not os.access(VIM, os.F_OK): + os.mkdir(VIM) + file = open(VIM + "/filetype.vim", "a") + file.write("au BufRead,BufNewFile *.irp.f setfiletype irpf90") + file.close() + if not os.access(VIM + "/syntax", os.F_OK): + os.mkdir(VIM + "/syntax") + wd = os.path.abspath(os.path.dirname(__file__)) + os.symlink(wd + "/irpf90.vim", VIM + "/syntax/irpf90.vim") + except: + pass + if __name__ == "__main__": - install() + install()