From b91e0048dff66ba786cc58eecb6f8c465c07f16c Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Wed, 20 May 2015 16:01:41 +0200 Subject: [PATCH] Add cache for the fortran --- scripts/compilation/cache_compile.py | 63 +++++++++++++++++++++++ scripts/compilation/create_ninja_build.py | 48 +++++++++++++---- src/Makefile.common | 54 ------------------- 3 files changed, 100 insertions(+), 65 deletions(-) create mode 100755 scripts/compilation/cache_compile.py diff --git a/scripts/compilation/cache_compile.py b/scripts/compilation/cache_compile.py new file mode 100755 index 00000000..bdd2e36b --- /dev/null +++ b/scripts/compilation/cache_compile.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import os +import sys +import shelve +import hashlib +import re + +r = re.compile(ur'-c\s+(\S+\.[fF]90)\s+-o\s+(\S+\.o)') +p = re.compile(ur'-I IRPF90_temp/\S*\s+') +mod = re.compile(ur'module\s+(?P\S+).+end\s?module\s+(?P=mod)?', re.MULTILINE | re.IGNORECASE) + +TMPDIR="/tmp/qp_compiler/" + +def main(): + # Create temp directory + if "qp_compiler" not in os.listdir("/tmp"): + os.mkdir("/tmp/qp_compiler/") + + line = sys.argv[1:] + command = " ".join(line) + command_clean = p.sub('',command) + + try: + match = r.search(command_clean) + input = match.group(1) + output = match.group(2) + except: + os.system(command) + return + m = hashlib.md5() + + # Fread : read input + with open(input,'r') as file: + fread = file.read() + m.update( " ".join( [ command, fread ] )) + + # Md5 Key containing command + content of Fread + key = TMPDIR+m.hexdigest() + try: + # Try to return the content of the .o file + with open(key,'r') as file: + result = file.read() + except IOError: + # Compile the file -> .o + os.system(command) + # Read the .o + with open(output,'r') as file: + result = file.read() + # Copy the .o in database + if not mod.search(fread.replace('\n',' ')): + with open(key,'w') as file: + file.write(result) + else: + print input+' -> module' + else: + # Write the .o file + with open(output,'w') as file: + file.write(result) + +if __name__ == '__main__': + main() + diff --git a/scripts/compilation/create_ninja_build.py b/scripts/compilation/create_ninja_build.py index 9a313e3c..ce39a322 100755 --- a/scripts/compilation/create_ninja_build.py +++ b/scripts/compilation/create_ninja_build.py @@ -51,7 +51,6 @@ def get_l_ezfio_irp(l_all_needed_molule, path_module): l_module_abs = [join(qpackage_root_src, m) for m in l_all_needed_molule] l_irp = [] - l_ezfio_module_abs = [] for m in l_module_abs + [path_module.abs]: @@ -59,10 +58,9 @@ def get_l_ezfio_irp(l_all_needed_molule, path_module): if file.endswith(".irp.f"): l_irp.append(join(m, file)) if file == "EZFIO.cfg": - l_ezfio_module_abs.append(m) l_irp.append(join(m, "ezfio_interface.irp.f")) - return l_irp, l_ezfio_module_abs + return l_irp def ninja_ezfio_cfg_rule(): @@ -75,10 +73,17 @@ def ninja_ezfio_cfg_rule(): return l_string -def ninja_ezfio_cfg_build(l_ezfio_module_abs): +def ninja_ezfio_cfg_build(): # Build l_string = [] - for m in l_ezfio_module_abs: + + from os import listdir + from os.path import isfile, join + qp_src = qpackage_root_src + + l = [join(qp_src, m) for m in listdir(qp_src) if isfile(join(qp_src, m, "EZFIO.cfg")) ] + + for m in l: ez_interface = join(m, "ezfio_interface.irp.f") ez_cfg = join(m, "EZFIO.cfg") @@ -132,8 +137,13 @@ def ninja_symlink_build(l_source, l_destination): # | def ninja_irpf90_make_rule(): # Rule - l_string = ["rule build_irpf90.make"] + l_string = ["pool irp_pool"] + l_string += [" depth = 1"] + l_string += [""] + + l_string += ["rule build_irpf90.make"] l_string += [" command = cd $module ; irpf90 $include_dir $irpf90_flag ; cd -"] + l_string += [" pool = irp_pool"] l_string += [""] return l_string @@ -189,7 +199,7 @@ def ninja_binary_rule(): # Rule l_string = ["rule build_binary"] - l_string += [" command = cd $module ; make $binary ; touch $binary; cd -"] + l_string += [" command = cd $module ; make -j 1 $binary ; touch $binary; cd -"] l_string += [""] return l_string @@ -215,9 +225,21 @@ def ninja_binary_build(l_bin, path_module): l_string += ["build build_all_binary_{0}: phony {1}".format(path_module.rel, str_l_abs_bin)] + l_string += [""] return l_string + +def ninja_all_binary_build(l_module): + l_build_name = ["build_all_binary_{0}".format(m) for m in l_module] + str_l_build_name = " ".join(l_build_name) + + l_string = ["build build_all_binary: phony {0}".format(str_l_build_name)] + l_string += [""] + + return l_string + + if __name__ == "__main__": with open(join(qpackage_root_src, "NEEDED_MODULES"), "r") as f: @@ -231,7 +253,9 @@ if __name__ == "__main__": from collections import namedtuple - for module_to_consider in ["Hartree_Fock", "AOs"]: #l_module_to_compile: +# l_module_to_compile = ["AOs", "CAS_SD", "Hartree_Fock"] + + for module_to_consider in l_module_to_compile: Path = namedtuple('Path', ['abs', 'rel']) @@ -245,16 +269,15 @@ if __name__ == "__main__": l_string += ninja_makefile_depend_build(l_all_needed_molule, path_module) # EZFIO.cfg rule and build - l_irp, l_ezfio_module_abs = get_l_ezfio_irp(l_all_needed_molule, path_module) - l_string += ninja_ezfio_cfg_build(l_ezfio_module_abs) + l_irp = get_l_ezfio_irp(l_all_needed_molule, path_module) # Symlink rule and build l_source, l_destination = get_source_destination(l_all_needed_molule, path_module) + l_string += ninja_symlink_build(l_source, l_destination) # irpf90.make - l_string += ninja_irpf90_make_build(path_module, l_all_needed_molule + ["include"], l_irp, @@ -263,4 +286,7 @@ if __name__ == "__main__": l_binary = get_program(path_module) l_string += ninja_binary_build(l_binary, path_module) + l_string += ninja_ezfio_cfg_build() + + l_string += ninja_all_binary_build(l_module_to_compile) print "\n".join(l_string) diff --git a/src/Makefile.common b/src/Makefile.common index ea8c3af0..e3e163ba 100644 --- a/src/Makefile.common +++ b/src/Makefile.common @@ -1,66 +1,12 @@ -.PHONY: default -.SECONDARY: symlink ezfio_interface - -default: all .gitignore - # Include the user's config include $(QPACKAGE_ROOT)/src/Makefile.config -# Create the NEEDED_CHILDREN_MODULES variable, needed for IRPF90 -NEEDED_CHILDREN_MODULES=$(shell module_handler.py print_genealogy) - # Define the Makefile common variables EZFIO_DIR=$(QPACKAGE_ROOT)/EZFIO EZFIO=$(EZFIO_DIR)/lib/libezfio_irp.a -INCLUDE_DIRS=$(NEEDED_CHILDREN_MODULES) include - -clean_links: - rm -f $(INCLUDE_DIRS) $$(basename $$PWD) LIB+=$(EZFIO) $(MKL) IRPF90+=$(patsubst %, -I %, $(INCLUDE_DIRS)) $(IRPF90_FLAGS) -# Update Makefile.depend -Makefile.depend: $(wildcard */Makefile) - ${QPACKAGE_ROOT}/scripts/module/module_handler.py save_makefile_depend - - -# Create symlink -symlink: $(wildcard $(QPACKAGE_ROOT)/src/*/NEEDED_CHILDREN_MODULES) - ${QPACKAGE_ROOT}/scripts/module/module_handler.py create_symlink - -# Define the EZFIO rules -$(EZFIO): $(wildcard $(QPACKAGE_ROOT)/src/*/*.ezfio_config) $(wildcard $(QPACKAGE_ROOT)/src/*/EZFIO.cfg) - $(QPACKAGE_ROOT)/scripts/ezfio_interface/prepare_ezfio.sh - cd $(EZFIO_DIR);\ - export FC="$(FC)" ; export FCFLAGS="$(FCFLAGS)" ; export IRPF90="$(IRPF90)" ;\ - $(MAKE) ;\ - $(MAKE) Python - -# Update EZFIO interface (create the irp.f90 and the ocaml) -ezfio_interface: symlink $(wildcard $(QPACKAGE_ROOT)/src/*/EZFIO.cfg) - ${QPACKAGE_ROOT}/scripts/ezfio_interface/ei_handler.py --irpf90 --ocaml --recursif - -irpf90.make: $(filter-out IRPF90_temp/%, $(wildcard */*.irp.f)) $(wildcard *.irp.f) $(wildcard .inc.f) Makefile.depend Makefile $(EZFIO) $(wildcard *.py) symlink ezfio_interface - echo $(IRPF90) - include Makefile.depend include irpf90.make - -# Need NEEDED_CHILDREN_MODULES and IRPMAN -README.rst: NEEDED_CHILDREN_MODULES irpf90.make - ${QPACKAGE_ROOT}/scripts/module/update_README.py - -tree_dependancy.png: NEEDED_CHILDREN_MODULES - ${QPACKAGE_ROOT}/scripts/module/module_handler.py create_png - -#Need all the executable -.gitignore: irpf90.make - ${QPACKAGE_ROOT}/scripts/module/create_gitignore.sh - -# -# -# # Frequent typos -# clena: clean -# veryclena: roger -# vercylean: roger