10
0
mirror of https://github.com/QuantumPackage/qp2.git synced 2024-12-22 12:23:43 +01:00

update scripts

This commit is contained in:
Kevin Gasperich 2022-06-22 17:17:12 -05:00
parent 61f205c192
commit c714fdc5f3
7 changed files with 360 additions and 43 deletions

View File

@ -43,6 +43,7 @@ EZFIO_LIB = join("$QP_ROOT", "lib", "libezfio_irp.a")
ZMQ_LIB = join("$QP_ROOT", "lib", "libf77zmq.a") + " " + join("$QP_ROOT", "lib", "libzmq.a") + " -lstdc++ -lrt -ldl"
ROOT_BUILD_NINJA = join("$QP_ROOT", "config", "build.ninja")
ROOT_BUILD_NINJA_EXP = join(QP_ROOT, "config", "build.ninja")
ROOT_BUILD_NINJA_EXP_tmp = join(QP_ROOT, "config", "build.ninja.tmp")
header = r"""#
# _______ _____
# __ __ \___ _______ _________ /____ ________ ___
@ -57,7 +58,7 @@ header = r"""#
# /_/ \__,_/ \___/ /_/|_| \__,_/ _\__, / \___/
# /____/
#
# https://github.com/LCPQ/quantum_package,
# https://github.com/QuantumPackage/qp2,
#
# Generated automatically by {0}
#
@ -107,6 +108,17 @@ def ninja_create_env_variable(pwd_config_file):
lib_usr = get_compilation_option(pwd_config_file, "LIB")
str_lib = " ".join([lib_lapack, EZFIO_LIB, ZMQ_LIB, LIB, lib_usr])
# Read all LIB files in modules
libfile = "LIB"
try:
content = ""
with open(libfile,'r') as f:
content = f.read()
str_lib += " "+content
except IOError:
pass
l_string.append("LIB = {0} ".format(str_lib))
l_string.append("")
@ -149,7 +161,7 @@ def get_l_module_with_ezfio_cfg():
from os import listdir
from os.path import isfile
return [real_join(QP_SRC, m) for m in listdir(QP_SRC)
return [real_join(QP_SRC, m) for m in sorted(listdir(QP_SRC))
if isfile(real_join(QP_SRC, m, "EZFIO.cfg"))]
@ -328,13 +340,13 @@ def ninja_symlink_build(path_module, l_symlink):
if not l_symlink:
return []
l_folder = [s.destination for s in l_symlink]
l_folder = [s.destination for s in sorted(l_symlink,key=lambda x:x.destination)]
l_string = ["build l_symlink_{0} : phony {1}".format(path_module.rel,
" ".join(map(comp_path,l_folder))),
""]
for symlink in l_symlink:
for symlink in sorted(l_symlink, key=lambda x: x.source+x.destination):
l_string += ["build {0}: build_symlink {1}".format(*list(map(comp_path,(symlink.destination, symlink.source)))), ""]
return l_string
@ -360,7 +372,7 @@ def ninja_gitignore_build(path_module, d_binaries, l_symlink):
path_gitignore = comp_path(join(path_module.abs, ".gitignore"))
l_b = list(map(comp_path,[i.abs for i in d_binaries[path_module]]))
l_b = sorted(list(map(comp_path,[i.abs for i in d_binaries[path_module]])))
root = "build {0}: build_gitignore {1}".format(path_gitignore,
" ".join(l_b))
@ -389,7 +401,7 @@ def get_l_file_for_module(path_module):
l_template = []
for f in os.listdir(path_module.abs):
for f in sorted(os.listdir(path_module.abs)):
if f.lower().endswith(tuple([".template.f", ".include.f"])):
l_template.append(join(path_module.abs, f))
elif f.endswith(".irp.f"):
@ -495,10 +507,10 @@ def ninja_irpf90_make_build(path_module, l_needed_molule, d_irp):
# D e p e n d a n c y #
# ~#~#~#~#~#~#~#~#~#~ #
l_depend = list(map(comp_path,d_irp[path_module]["l_depend"]))
l_src = list(map(comp_path,d_irp[path_module]["l_src"]))
l_obj = list(map(comp_path,d_irp[path_module]["l_obj"]))
l_template = list(map(comp_path,d_irp[path_module]["l_template"]))
l_depend = sorted(list(map(comp_path,d_irp[path_module]["l_depend"])))
l_src = sorted(list(map(comp_path,d_irp[path_module]["l_src"])))
l_obj = sorted(list(map(comp_path,d_irp[path_module]["l_obj"])))
l_template = sorted(list(map(comp_path,d_irp[path_module]["l_template"])))
if l_needed_molule:
l_symlink = ["l_symlink_{0}".format(path_module.rel)]
@ -511,7 +523,7 @@ def ninja_irpf90_make_build(path_module, l_needed_molule, d_irp):
# N i n j a _ b u i l d #
# ~#~#~#~#~#~#~#~#~#~#~ #
l_include_dir = ["-I {0}".format(m.rel) for m in l_needed_molule]
l_include_dir = ["-I {0}".format(m.rel) for m in sorted(l_needed_molule, key=lambda x:x.rel+x.abs) ]
l_string = [
"build {0}: build_irpf90.ninja {1}".format(str_creation, str_depend),
@ -642,7 +654,7 @@ def ninja_binaries_build(path_module, l_children, d_binaries):
# ~#~#~ #
ninja_module_path = join(comp_path(path_module.abs), "IRPF90_temp/build.ninja")
l_abs_bin = list(map(comp_path,[binary.abs for binary in d_binaries[path_module]]))
l_abs_bin = sorted(list(map(comp_path,[binary.abs for binary in d_binaries[path_module]])))
# ~#~#~#~#~#~ #
# s t r i n g #
@ -659,8 +671,7 @@ def ninja_binaries_build(path_module, l_children, d_binaries):
def ninja_module_build(path_module, d_binaries):
l_abs_bin = list(map(comp_path,[binary.abs for binary in d_binaries[path_module]]))
l_abs_bin = sorted(list(map(comp_path,[binary.abs for binary in d_binaries[path_module]])))
path_readme = os.path.join(comp_path(path_module.abs), "README.rst")
l_string = ["build module_{0}: phony {1}".format(path_module.rel,
@ -698,6 +709,11 @@ def save_subninja_file(path_module):
" description = Cleaning module {0}".format(path_module.rel),
""]
l_string += ["rule make_tidy",
" command = module_handler.py tidy {0}".format(path_module.rel),
" description = Cleaning module {0}".format(path_module.rel),
""]
l_string += ["rule executables",
" command = make -C {0} executables .gitignore qp_edit.native qp_run.native".format(join("$QP_ROOT","ocaml")),
" description = Updating OCaml executables",
@ -708,6 +724,7 @@ def save_subninja_file(path_module):
"build local: make_local_binaries dummy_target", "",
"build executables: executables local dummy_target", "",
"default executables", "", "build clean: make_clean dummy_target",
"", "build tidy: make_tidy dummy_target",
""]
path_ninja_cur = join(path_module.abs, "build.ninja")
@ -734,6 +751,10 @@ def create_build_ninja_global():
" command = module_handler.py clean --all",
" description = Cleaning all modules", ""]
l_string += ["rule make_tidy",
" command = module_handler.py tidy --all",
" description = Cleaning all modules", ""]
l_string += ["rule make_ocaml",
" command = make -C {0}/ocaml".format("$QP_ROOT"),
" pool = console",
@ -748,6 +769,8 @@ def create_build_ninja_global():
"default ocaml_target",
"",
"build clean: make_clean dummy_target",
"",
"build tidy: make_tidy dummy_target",
"", ]
path_ninja_cur = join(QP_ROOT, "build.ninja")
@ -830,14 +853,14 @@ if __name__ == "__main__":
dict_root = module_instance.dict_root
dict_root_path = dict_module_genelogy_path(dict_root)
l_all_module = list(d_genealogy_path.keys())
l_all_module = sorted(list(d_genealogy_path.keys()))
# ~#~#~#~#~#~#~#~#~#~#~#~#~ #
# M o d u l e _ t o _ i r p #
# ~#~#~#~#~#~#~#~#~#~#~#~#~ #
d_binaries = get_dict_binaries(l_all_module, mode="development")
l_module = list(d_binaries.keys())
l_module = sorted(list(d_binaries.keys()), key=lambda x: x.rel+x.abs)
# ~#~#~#~#~#~#~#~#~#~#~#~ #
@ -851,7 +874,6 @@ if __name__ == "__main__":
# ~#~#~#~#~#~#~#~#~#~#~#~ #
for module_to_compile in l_module:
if module_to_compile.rel == "dummy":
continue
@ -884,6 +906,21 @@ if __name__ == "__main__":
# S a v e s #
# ~#~#~#~#~ #
with open(ROOT_BUILD_NINJA_EXP, "w+") as f:
with open(ROOT_BUILD_NINJA_EXP_tmp, "w+") as f:
f.write(header)
f.write("\n".join(l_string))
with open(ROOT_BUILD_NINJA_EXP_tmp, "r") as f:
a = f.read()
try:
with open(ROOT_BUILD_NINJA_EXP, "r") as f:
b = f.read()
except:
b = None
if a != b:
os.rename(ROOT_BUILD_NINJA_EXP_tmp, ROOT_BUILD_NINJA_EXP)
else:
os.remove(ROOT_BUILD_NINJA_EXP_tmp)

View File

@ -82,6 +82,8 @@ END_PROVIDER
mpi_correspondance = {"integer": "MPI_INTEGER",
"integer*8": "MPI_INTEGER8",
"character*(32)": "MPI_CHARACTER",
"character*(64)": "MPI_CHARACTER",
"character*(256)": "MPI_CHARACTER",
"logical": "MPI_LOGICAL",
"double precision": "MPI_DOUBLE_PRECISION"}

View File

@ -6,12 +6,18 @@ Module utilitary
Usage:
module_handler.py print_descendant [<module_name>...]
module_handler.py clean [ --all | <module_name>...]
module_handler.py create_git_ignore [<module_name>...]
module_handler.py tidy [ --all | <module_name>...]
module_handler.py create_git_ignore [ --all | <module_name>...]
Options:
print_descendant Print the genealogy of the needed modules
clean Used for ninja clean
tidy A light version of clean, where only the intermediate
files are removed
create_git_ignore deprecated
NEED The path of NEED file.
by default try to open the file in the current path
"""
import os
import sys
@ -25,7 +31,7 @@ try:
from docopt import docopt
from qp_path import QP_SRC, QP_ROOT, QP_PLUGINS, QP_EZFIO
except ImportError:
print("source .quantum_package.rc")
print("source quantum_package.rc")
raise
@ -110,6 +116,7 @@ def get_l_module_descendant(d_child, l_module):
print("Error: ", file=sys.stderr)
print("`{0}` is not a submodule".format(module), file=sys.stderr)
print("Check the typo (spelling, case, '/', etc.) ", file=sys.stderr)
# pass
sys.exit(1)
return list(set(l))
@ -209,7 +216,7 @@ if __name__ == '__main__':
# Remove all produced ezfio_config files
for filename in os.listdir( os.path.join(QP_EZFIO, "config") ):
os.remove( os.path.join(QP_EZFIO, "config", filename) )
elif not arguments['<module_name>']:
dir_ = os.getcwd()
@ -230,11 +237,11 @@ if __name__ == '__main__':
for module in l_module:
print(" ".join(sorted(m.l_descendant_unique([module]))))
if arguments["clean"]:
if arguments["clean"] or arguments["tidy"]:
l_dir = ['IRPF90_temp', 'IRPF90_man']
l_file = ["irpf90_entities", "tags", "irpf90.make", "Makefile",
"Makefile.depend", ".ninja_log", ".ninja_deps",
"Makefile.depend", ".ninja_log", ".ninja_deps",
"ezfio_interface.irp.f"]
for module in l_module:
@ -242,25 +249,25 @@ if __name__ == '__main__':
l_symlink = m.l_descendant_unique([module])
l_exe = get_binaries(module_abs)
for f in l_dir:
try:
shutil.rmtree(os.path.join(module_abs, f))
except:
pass
for symlink in l_symlink:
try:
os.unlink(os.path.join(module_abs, symlink))
except:
pass
for f in l_file:
try:
os.remove(os.path.join(module_abs, f))
except:
pass
if arguments["clean"]:
for f in l_dir:
try:
shutil.rmtree(os.path.join(module_abs, f))
except:
pass
for symlink in l_symlink:
try:
os.unlink(os.path.join(module_abs, symlink))
except:
pass
for f in l_file:
try:
os.remove(os.path.join(module_abs, f))
except:
pass
for f in l_exe:
try:
@ -268,6 +275,4 @@ if __name__ == '__main__':
except:
pass
if arguments["create_git_ignore"]:
pass

View File

@ -0,0 +1,175 @@
program check_omp_v2
use omp_lib
implicit none
integer :: accu, accu2
integer :: s, n_setting
logical :: verbose, test_versions
logical, allocatable :: is_working(:)
verbose = .False.
test_versions = .True.
n_setting = 4
allocate(is_working(n_setting))
is_working = .False.
! set the number of threads
call omp_set_num_threads(2)
do s = 1, n_setting
accu = 0
accu2 = 0
call omp_set_max_active_levels(1)
call omp_set_nested(.False.)
if (s==1) then
!call set_multiple_levels_omp()
cycle
elseif (s==2) then
call omp_set_max_active_levels(5)
elseif (s==3) then
call omp_set_nested(.True.)
else
call omp_set_nested(.True.)
call omp_set_max_active_levels(5)
endif
! Level 1
!$OMP PARALLEL
if (verbose) then
print*,'Num threads level 1:',omp_get_num_threads()
endif
! Level 2
!$OMP PARALLEL
if (verbose) then
print*,'Num threads level 2:',omp_get_num_threads()
endif
! Level 3
!$OMP PARALLEL
if (verbose) then
print*,'Num threads level 3:',omp_get_num_threads()
endif
call check_omp_in_subroutine(accu2)
! Level 4
!$OMP PARALLEL
if (verbose) then
print*,'Num threads level 4:',omp_get_num_threads()
endif
!$OMP ATOMIC
accu = accu + 1
!$OMP END ATOMIC
!$OMP END PARALLEL
!$OMP END PARALLEL
!$OMP END PARALLEL
!$OMP END PARALLEL
if (verbose) then
print*,'Setting:',s,'accu=',accu
print*,'Setting:',s,'accu2=',accu2
endif
if (accu == 16 .and. accu2 == 16) then
is_working(s) = .True.
endif
enddo
if (verbose) then
if (is_working(2)) then
print*,'The parallelization works on 4 levels with:'
print*,'call omp_set_max_active_levels(5)'
print*,''
print*,'Please use the irpf90 flags -DSET_MAX_ACT in qp2/config/${compiler_name}.cfg'
elseif (is_working(3)) then
print*,'The parallelization works on 4 levels with:'
print*,'call omp_set_nested(.True.)'
print*,''
print*,'Please use the irpf90 flag -DSET_NESTED in qp2/config/${compiler_name}.cfg'
elseif (is_working(4)) then
print*,'The parallelization works on 4 levels with:'
print*,'call omp_set_nested(.True.)'
print*,'+'
print*,'call omp_set_max_active_levels(5)'
print*,''
print*,'Please use the irpf90 flags -DSET_NESTED -DSET_MAX_ACT in qp2/config/${compiler_name}.cfg'
else
print*,'The parallelization on multiple levels does not work with:'
print*,'call omp_set_max_active_levels(5)'
print*,'or'
print*,'call omp_set_nested(.True.)'
print*,'or'
print*,'call omp_set_nested(.True.)'
print*,'+'
print*,'call omp_set_max_active_levels(5)'
print*,''
print*,'Try an other compiler and good luck...'
endif
! if (is_working(1)) then
! print*,''
! print*,'=========================================================='
! print*,'Your actual set up works for parallelization with 4 levels'
! print*,'=========================================================='
! print*,''
! else
! print*,''
! print*,'==================================================================='
! print*,'Your actual set up does not work for parallelization with 4 levels'
! print*,'Please look at the previous messages to understand the requirements'
! print*,'==================================================================='
! print*,''
! endif
endif
! List of working flags
if (test_versions) then
print*,'Tests:',is_working(2:4)
endif
! IRPF90_FLAGS
if (is_working(2)) then
print*,'-DSET_MAX_ACT'
elseif (is_working(3)) then
print*,'-DSET_NESTED'
elseif (is_working(4)) then
print*,'-DSET_MAX_ACT -DSET_NESTED'
else
print*,'ERROR'
endif
end
subroutine check_omp_in_subroutine(accu2)
implicit none
integer, intent(inout) :: accu2
!$OMP PARALLEL
!$OMP ATOMIC
accu2 = accu2 + 1
!$OMP END ATOMIC
!$OMP END PARALLEL
end

View File

@ -0,0 +1,19 @@
#!/bin/sh
# take one argument which is the compiler used
# return the required IRPF90_FLAGS for the $1 compiler
if [ -z "$1" ]
then
echo "Give the compiler in argument"
else
$1 --version > /dev/null \
&& $1 -O0 -fopenmp check_omp.f90 \
&& ./a.out | tail -n 1
# if there is an error or if the compiler is not found
$1 --version > /dev/null || echo 'compiler not found'
fi

30
scripts/verif_omp/study_omp.sh Executable file
View File

@ -0,0 +1,30 @@
#!/bin/sh
# list of compilers
list_comp="ifort gfortran-7 gfortran-8 gfortran-9"
# file to store the results
FILE=results.dat
touch $FILE
rm $FILE
# Comments
echo "1: omp_set_max_active_levels(5)" >> $FILE
echo "2: omp_set_nested(.True.)" >> $FILE
echo "3: 1 + 2" >> $FILE
echo "" >> $FILE
echo "1 2 3" >> $FILE
# loop on the comp
for comp in $list_comp
do
$comp --version > /dev/null \
&& $comp -O0 -fopenmp check_omp.f90 \
&& echo $(./a.out | grep "Tests:" | cut -d ":" -f2- ) $(echo " : ") $($comp --version | head -n 1) >> $FILE
done
# Display
cat $FILE

View File

@ -0,0 +1,49 @@
#!/bin/bash
# Compiler
COMP=$1
# Path to file.cfg
config_PATH="../../config/"
END="*.cfg"
CONFIG="/config/"
#LIST=${config_PATH}${COMP}${END} # without ${QP_ROOT}
LIST=${QP_ROOT}${CONFIG}${COMP}${END}
if [ -z "$1" ]
then
echo "Give the compiler in argument"
else
# List of the config files for the compiler
#list_files=$(ls ../../config/$comp*.cfg) #does not give the right list
list_files=${LIST}
echo "Files that will be modified:"
echo $list_files
# Flags that must be added
FLAGS=$(./check_required_setup.sh $COMP)
# Add the flags
for file in $list_files
do
echo $file
BASE="IRPF90_FLAGS : --ninja"
ACTUAL=$(grep "$BASE" $file)
# To have only one time each flag
grep " -DSET_MAX_ACT" $file && ${ACTUAL/" -DSET_MAX"/""}
grep " -DSET_NESTED" $file && ${ACTUAL/" -DSET_NESTED"/""}
SPACE=" "
NEW=${ACTUAL}${SPACE}${FLAGS}
# Debug
#echo ${NEW}
sed "s/${ACTUAL}/${NEW}/" $file
# -i # to change the files
done
fi