update scripts

scripts/compilation/qp_create_ninja

@@ -43,6 +43,7 @@ EZFIO_LIB = join("$QP_ROOT", "lib", "libezfio_irp.a")
 ZMQ_LIB = join("$QP_ROOT", "lib", "libf77zmq.a") + " "  + join("$QP_ROOT", "lib", "libzmq.a") + " -lstdc++ -lrt -ldl"
 ROOT_BUILD_NINJA = join("$QP_ROOT", "config", "build.ninja")
 ROOT_BUILD_NINJA_EXP =  join(QP_ROOT, "config", "build.ninja")
+ROOT_BUILD_NINJA_EXP_tmp =  join(QP_ROOT, "config", "build.ninja.tmp")
 header = r"""#
 #  _______                     _____
 #  __  __ \___  _______ _________  /____  ________ ___
@@ -57,7 +58,7 @@ header = r"""#
 #               /_/     \__,_/ \___/ /_/|_| \__,_/ _\__, / \___/
 #                                                  /____/
 #
-# https://github.com/LCPQ/quantum_package,
+# https://github.com/QuantumPackage/qp2,
 #
 # Generated automatically by {0}
 #
@@ -107,6 +108,17 @@ def ninja_create_env_variable(pwd_config_file):
     lib_usr = get_compilation_option(pwd_config_file, "LIB")
 
     str_lib = " ".join([lib_lapack, EZFIO_LIB, ZMQ_LIB, LIB, lib_usr])
+
+    # Read all LIB files in modules
+    libfile = "LIB"
+    try:
+        content = ""
+        with open(libfile,'r') as f:
+            content = f.read()
+            str_lib += " "+content
+    except IOError:
+        pass
+
     l_string.append("LIB = {0} ".format(str_lib))
 
     l_string.append("")
@@ -149,7 +161,7 @@ def get_l_module_with_ezfio_cfg():
     from os import listdir
     from os.path import isfile
 
-    return [real_join(QP_SRC, m) for m in listdir(QP_SRC)
+    return [real_join(QP_SRC, m) for m in sorted(listdir(QP_SRC))
             if isfile(real_join(QP_SRC, m, "EZFIO.cfg"))]
 
 
@@ -328,13 +340,13 @@ def ninja_symlink_build(path_module, l_symlink):
     if not l_symlink:
         return []
 
-    l_folder = [s.destination for s in l_symlink]
+    l_folder = [s.destination for s in sorted(l_symlink,key=lambda x:x.destination)]
 
     l_string = ["build l_symlink_{0} : phony {1}".format(path_module.rel,
                                                          " ".join(map(comp_path,l_folder))),
                 ""]
 
-    for symlink in l_symlink:
+    for symlink in sorted(l_symlink, key=lambda x: x.source+x.destination):
         l_string += ["build {0}: build_symlink {1}".format(*list(map(comp_path,(symlink.destination, symlink.source)))), ""]
 
     return l_string
@@ -360,7 +372,7 @@ def ninja_gitignore_build(path_module, d_binaries, l_symlink):
 
     path_gitignore = comp_path(join(path_module.abs, ".gitignore"))
 
-    l_b = list(map(comp_path,[i.abs for i in d_binaries[path_module]]))
+    l_b = sorted(list(map(comp_path,[i.abs for i in d_binaries[path_module]])))
 
     root = "build {0}: build_gitignore {1}".format(path_gitignore,
                                                    " ".join(l_b))
@@ -389,7 +401,7 @@ def get_l_file_for_module(path_module):
 
     l_template = []
 
-    for f in os.listdir(path_module.abs):
+    for f in sorted(os.listdir(path_module.abs)):
         if f.lower().endswith(tuple([".template.f", ".include.f"])):
             l_template.append(join(path_module.abs, f))
         elif f.endswith(".irp.f"):
@@ -495,10 +507,10 @@ def ninja_irpf90_make_build(path_module, l_needed_molule, d_irp):
     # D e p e n d a n c y #
     # ~#~#~#~#~#~#~#~#~#~ #
 
-    l_depend = list(map(comp_path,d_irp[path_module]["l_depend"]))
+    l_depend = sorted(list(map(comp_path,d_irp[path_module]["l_depend"])))
-    l_src = list(map(comp_path,d_irp[path_module]["l_src"]))
+    l_src = sorted(list(map(comp_path,d_irp[path_module]["l_src"])))
-    l_obj = list(map(comp_path,d_irp[path_module]["l_obj"]))
+    l_obj = sorted(list(map(comp_path,d_irp[path_module]["l_obj"])))
-    l_template = list(map(comp_path,d_irp[path_module]["l_template"]))
+    l_template = sorted(list(map(comp_path,d_irp[path_module]["l_template"])))
 
     if l_needed_molule:
         l_symlink = ["l_symlink_{0}".format(path_module.rel)]
@@ -511,7 +523,7 @@ def ninja_irpf90_make_build(path_module, l_needed_molule, d_irp):
     # N i n j a _ b u i l d #
     # ~#~#~#~#~#~#~#~#~#~#~ #
 
-    l_include_dir  = ["-I {0}".format(m.rel) for m in l_needed_molule]
+    l_include_dir  = ["-I {0}".format(m.rel) for m in sorted(l_needed_molule, key=lambda x:x.rel+x.abs) ]
 
     l_string = [
         "build {0}: build_irpf90.ninja {1}".format(str_creation, str_depend),
@@ -642,7 +654,7 @@ def ninja_binaries_build(path_module, l_children, d_binaries):
     # ~#~#~ #
 
     ninja_module_path = join(comp_path(path_module.abs), "IRPF90_temp/build.ninja")
-    l_abs_bin = list(map(comp_path,[binary.abs for binary in d_binaries[path_module]]))
+    l_abs_bin = sorted(list(map(comp_path,[binary.abs for binary in d_binaries[path_module]])))
 
     # ~#~#~#~#~#~ #
     # s t r i n g #
@@ -659,8 +671,7 @@ def ninja_binaries_build(path_module, l_children, d_binaries):
 
 def ninja_module_build(path_module, d_binaries):
 
-    l_abs_bin = list(map(comp_path,[binary.abs for binary in d_binaries[path_module]]))
+    l_abs_bin = sorted(list(map(comp_path,[binary.abs for binary in d_binaries[path_module]])))
-
     path_readme = os.path.join(comp_path(path_module.abs), "README.rst")
 
     l_string = ["build module_{0}: phony {1}".format(path_module.rel,
@@ -698,6 +709,11 @@ def save_subninja_file(path_module):
                  "  description = Cleaning module {0}".format(path_module.rel),
                  ""]
 
+    l_string += ["rule make_tidy",
+                 "  command = module_handler.py tidy {0}".format(path_module.rel),
+                 "  description = Cleaning module {0}".format(path_module.rel),
+                 ""]
+
     l_string += ["rule executables",
             "  command = make -C {0} executables .gitignore qp_edit.native qp_run.native".format(join("$QP_ROOT","ocaml")),
                  "  description = Updating OCaml executables",
@@ -708,6 +724,7 @@ def save_subninja_file(path_module):
                  "build local: make_local_binaries dummy_target", "",
                  "build executables: executables local dummy_target", "",
                  "default executables", "", "build clean: make_clean dummy_target",
+                 "", "build tidy: make_tidy dummy_target",
                  ""]
 
     path_ninja_cur = join(path_module.abs, "build.ninja")
@@ -734,6 +751,10 @@ def create_build_ninja_global():
                  "  command = module_handler.py clean --all",
                  "  description = Cleaning all modules", ""]
 
+    l_string += ["rule make_tidy",
+                 "  command = module_handler.py tidy --all",
+                 "  description = Cleaning all modules", ""]
+
     l_string += ["rule make_ocaml",
                  "  command = make -C {0}/ocaml".format("$QP_ROOT"),
                  "  pool = console",
@@ -748,6 +769,8 @@ def create_build_ninja_global():
                  "default ocaml_target",
                  "",
                  "build clean: make_clean dummy_target",
+                 "",
+                 "build tidy: make_tidy dummy_target",
                  "", ]
 
     path_ninja_cur = join(QP_ROOT, "build.ninja")
@@ -830,14 +853,14 @@ if __name__ == "__main__":
     dict_root = module_instance.dict_root
     dict_root_path = dict_module_genelogy_path(dict_root)
 
-    l_all_module = list(d_genealogy_path.keys())
+    l_all_module = sorted(list(d_genealogy_path.keys()))
 
     # ~#~#~#~#~#~#~#~#~#~#~#~#~ #
     # M o d u l e _ t o _ i r p #
     # ~#~#~#~#~#~#~#~#~#~#~#~#~ #
 
     d_binaries = get_dict_binaries(l_all_module, mode="development")
-    l_module = list(d_binaries.keys())
+    l_module = sorted(list(d_binaries.keys()), key=lambda x: x.rel+x.abs)
 
 
     # ~#~#~#~#~#~#~#~#~#~#~#~ #
@@ -851,7 +874,6 @@ if __name__ == "__main__":
     # ~#~#~#~#~#~#~#~#~#~#~#~ #
 
     for module_to_compile in l_module:
-    
         if module_to_compile.rel == "dummy":
                 continue
 
@@ -884,6 +906,21 @@ if __name__ == "__main__":
     # S a v e s #
     # ~#~#~#~#~ #
 
-    with open(ROOT_BUILD_NINJA_EXP, "w+") as f:
+    with open(ROOT_BUILD_NINJA_EXP_tmp, "w+") as f:
         f.write(header)
         f.write("\n".join(l_string))
+
+    with open(ROOT_BUILD_NINJA_EXP_tmp, "r") as f:
+        a = f.read()
+
+    try:
+        with open(ROOT_BUILD_NINJA_EXP, "r") as f:
+           b = f.read()
+    except:
+           b = None
+
+    if a != b:
+       os.rename(ROOT_BUILD_NINJA_EXP_tmp, ROOT_BUILD_NINJA_EXP)
+    else:
+       os.remove(ROOT_BUILD_NINJA_EXP_tmp)
+

scripts/ezfio_interface/ezfio_generate_provider.py

@@ -82,6 +82,8 @@ END_PROVIDER
     mpi_correspondance   = {"integer": "MPI_INTEGER",
                             "integer*8": "MPI_INTEGER8",
                             "character*(32)": "MPI_CHARACTER",
+                            "character*(64)": "MPI_CHARACTER",
+                            "character*(256)": "MPI_CHARACTER",
                             "logical": "MPI_LOGICAL",
                             "double precision": "MPI_DOUBLE_PRECISION"}
 

scripts/module/module_handler.py

@@ -6,12 +6,18 @@ Module utilitary
 Usage:
     module_handler.py print_descendant      [<module_name>...]
     module_handler.py clean                 [ --all | <module_name>...]
-    module_handler.py create_git_ignore     [<module_name>...]
+    module_handler.py tidy                  [ --all | <module_name>...]
+    module_handler.py create_git_ignore     [ --all | <module_name>...]
 
 Options:
     print_descendant        Print the genealogy of the needed modules
+    clean                   Used for ninja clean
+    tidy                    A light version of clean, where only the intermediate
+                            files are removed
+    create_git_ignore       deprecated
     NEED                    The path of NEED file.
                             by default try to open the file in the current path
+
 """
 import os
 import sys
@@ -25,7 +31,7 @@ try:
     from docopt import docopt
     from qp_path import QP_SRC, QP_ROOT, QP_PLUGINS, QP_EZFIO
 except ImportError:
-    print("source .quantum_package.rc")
+    print("source quantum_package.rc")
     raise
 
 
@@ -110,6 +116,7 @@ def get_l_module_descendant(d_child, l_module):
                 print("Error: ", file=sys.stderr)
                 print("`{0}` is not a submodule".format(module), file=sys.stderr)
                 print("Check the typo (spelling, case, '/', etc.) ", file=sys.stderr)
+#                pass
                 sys.exit(1)
 
     return list(set(l))
@@ -209,7 +216,7 @@ if __name__ == '__main__':
         # Remove all produced ezfio_config files
         for filename in os.listdir( os.path.join(QP_EZFIO, "config") ):
             os.remove( os.path.join(QP_EZFIO, "config", filename) )
-            
+
 
     elif not arguments['<module_name>']:
         dir_ = os.getcwd()
@@ -230,11 +237,11 @@ if __name__ == '__main__':
         for module in l_module:
             print(" ".join(sorted(m.l_descendant_unique([module]))))
 
-    if arguments["clean"]:
+    if arguments["clean"] or arguments["tidy"]:
 
         l_dir = ['IRPF90_temp', 'IRPF90_man']
         l_file = ["irpf90_entities", "tags", "irpf90.make", "Makefile",
-                  "Makefile.depend", ".ninja_log", ".ninja_deps", 
+                  "Makefile.depend", ".ninja_log", ".ninja_deps",
                   "ezfio_interface.irp.f"]
 
         for module in l_module:
@@ -242,25 +249,25 @@ if __name__ == '__main__':
             l_symlink = m.l_descendant_unique([module])
             l_exe = get_binaries(module_abs)
 
+            for f in l_dir:
+                try:
+                    shutil.rmtree(os.path.join(module_abs, f))
+                except:
+                    pass
+
+            for symlink in l_symlink:
+                try:
+                    os.unlink(os.path.join(module_abs, symlink))
+                except:
+                    pass
+
+            for f in l_file:
+                try:
+                    os.remove(os.path.join(module_abs, f))
+                except:
+                    pass
+
             if arguments["clean"]:
-                for f in l_dir:
-                    try:
-                        shutil.rmtree(os.path.join(module_abs, f))
-                    except:
-                        pass
-
-                for symlink in l_symlink:
-                    try:
-                        os.unlink(os.path.join(module_abs, symlink))
-                    except:
-                        pass
-
-                for f in l_file:
-                    try:
-                        os.remove(os.path.join(module_abs, f))
-                    except:
-                        pass
-
                 for f in l_exe:
 
                     try:
@@ -268,6 +275,4 @@ if __name__ == '__main__':
                     except:
                         pass
 
-            if arguments["create_git_ignore"]:
-               pass
 

scripts/verif_omp/check_omp.f90

@@ -0,0 +1,175 @@
+program check_omp_v2
+ 
+  use omp_lib
+
+  implicit none
+
+  integer :: accu, accu2
+  integer :: s, n_setting
+  logical :: verbose, test_versions
+  logical, allocatable :: is_working(:)
+
+  verbose = .False.
+  test_versions = .True.
+  n_setting = 4
+
+  allocate(is_working(n_setting))
+
+  is_working = .False.
+
+  ! set the number of threads
+  call omp_set_num_threads(2) 
+
+  do s = 1, n_setting
+
+    accu = 0
+    accu2 = 0
+
+    call omp_set_max_active_levels(1)
+    call omp_set_nested(.False.)
+
+    if (s==1) then
+      !call set_multiple_levels_omp()
+      cycle
+    elseif (s==2) then
+      call omp_set_max_active_levels(5)
+    elseif (s==3) then
+      call omp_set_nested(.True.)
+    else
+      call omp_set_nested(.True.)
+      call omp_set_max_active_levels(5)
+    endif
+
+    ! Level 1
+    !$OMP PARALLEL 
+      if (verbose) then
+        print*,'Num threads level 1:',omp_get_num_threads()
+      endif
+
+      ! Level 2
+      !$OMP PARALLEL 
+        if (verbose) then
+          print*,'Num threads level 2:',omp_get_num_threads()
+        endif
+
+        ! Level 3
+        !$OMP PARALLEL 
+          if (verbose) then
+            print*,'Num threads level 3:',omp_get_num_threads()
+          endif
+
+          call check_omp_in_subroutine(accu2)
+     
+          ! Level 4
+          !$OMP PARALLEL 
+
+            if (verbose) then
+              print*,'Num threads level 4:',omp_get_num_threads()
+            endif
+
+           !$OMP ATOMIC    
+           accu = accu + 1
+           !$OMP END ATOMIC 
+       
+          !$OMP END PARALLEL
+     
+     
+        !$OMP END PARALLEL
+
+
+      !$OMP END PARALLEL
+
+
+    !$OMP END PARALLEL
+
+  if (verbose) then
+    print*,'Setting:',s,'accu=',accu
+    print*,'Setting:',s,'accu2=',accu2
+  endif
+
+  if (accu == 16 .and. accu2 == 16) then
+    is_working(s) = .True.
+  endif
+
+  enddo
+
+  if (verbose) then
+    if (is_working(2)) then
+      print*,'The parallelization works on 4 levels with:'
+      print*,'call omp_set_max_active_levels(5)'
+      print*,''
+      print*,'Please use the irpf90 flags -DSET_MAX_ACT in qp2/config/${compiler_name}.cfg'
+    elseif (is_working(3)) then
+      print*,'The parallelization works on 4 levels with:'
+      print*,'call omp_set_nested(.True.)'
+      print*,''
+      print*,'Please use the irpf90 flag -DSET_NESTED in qp2/config/${compiler_name}.cfg'
+    elseif (is_working(4)) then
+      print*,'The parallelization works on 4 levels with:'
+      print*,'call omp_set_nested(.True.)'
+      print*,'+'
+      print*,'call omp_set_max_active_levels(5)'
+      print*,''
+      print*,'Please use the irpf90 flags -DSET_NESTED -DSET_MAX_ACT in qp2/config/${compiler_name}.cfg'
+    else
+      print*,'The parallelization on multiple levels does not work with:'
+      print*,'call omp_set_max_active_levels(5)'
+      print*,'or'
+      print*,'call omp_set_nested(.True.)'
+      print*,'or'
+      print*,'call omp_set_nested(.True.)'
+      print*,'+'
+      print*,'call omp_set_max_active_levels(5)'
+      print*,''
+      print*,'Try an other compiler and good luck...'
+    endif
+
+   ! if (is_working(1)) then
+   !   print*,''
+   !   print*,'=========================================================='
+   !   print*,'Your actual set up works for parallelization with 4 levels'
+   !   print*,'=========================================================='
+   !   print*,''
+   ! else
+   !   print*,''
+   !   print*,'==================================================================='
+   !   print*,'Your actual set up does not work for parallelization with 4 levels'
+   !   print*,'Please look at the previous messages to understand the requirements'
+   !   print*,'==================================================================='
+   !   print*,''
+   ! endif
+  endif
+
+  ! List of working flags
+  if (test_versions) then
+    print*,'Tests:',is_working(2:4)
+  endif
+
+  ! IRPF90_FLAGS
+  if (is_working(2)) then
+    print*,'-DSET_MAX_ACT'
+  elseif (is_working(3)) then
+    print*,'-DSET_NESTED'
+  elseif (is_working(4)) then
+    print*,'-DSET_MAX_ACT -DSET_NESTED'
+  else
+    print*,'ERROR'
+  endif
+
+end
+
+subroutine check_omp_in_subroutine(accu2)
+
+  implicit none
+ 
+  integer, intent(inout) :: accu2
+
+  !$OMP PARALLEL
+
+  !$OMP ATOMIC
+  accu2 = accu2 + 1
+  !$OMP END ATOMIC
+
+  !$OMP END PARALLEL
+
+end

scripts/verif_omp/check_required_setup.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# take one argument which is the compiler used
+# return the required IRPF90_FLAGS for the $1 compiler
+
+if [ -z "$1" ]
+then
+    echo "Give the compiler in argument"
+else
+
+$1 --version > /dev/null \
+&& $1 -O0 -fopenmp check_omp.f90 \
+&& ./a.out | tail -n 1
+
+
+# if there is an error or if the compiler is not found
+$1 --version > /dev/null || echo 'compiler not found'
+
+fi

scripts/verif_omp/study_omp.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# list of compilers
+list_comp="ifort gfortran-7 gfortran-8 gfortran-9"
+
+# file to store the results
+FILE=results.dat
+
+touch $FILE
+rm $FILE
+
+# Comments
+echo "1: omp_set_max_active_levels(5)" >> $FILE
+echo "2: omp_set_nested(.True.)" >> $FILE
+echo "3: 1 + 2" >> $FILE
+echo "" >> $FILE
+echo "1 2 3" >> $FILE
+
+# loop on the comp
+for comp in $list_comp
+do
+	$comp --version > /dev/null \
+        && $comp -O0 -fopenmp check_omp.f90 \
+	&& echo $(./a.out | grep "Tests:" | cut -d ":" -f2- ) $(echo " : ") $($comp --version | head -n 1) >> $FILE
+
+done
+
+# Display
+cat $FILE
+

scripts/verif_omp/update_comp.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Compiler
+COMP=$1
+
+# Path to file.cfg
+config_PATH="../../config/"
+END="*.cfg"
+CONFIG="/config/"
+
+#LIST=${config_PATH}${COMP}${END} # without ${QP_ROOT}
+LIST=${QP_ROOT}${CONFIG}${COMP}${END}
+
+if [ -z "$1" ]
+then
+    echo "Give the compiler in argument"
+else
+
+    # List of the config files for the compiler
+    #list_files=$(ls ../../config/$comp*.cfg) #does not give the right list
+    list_files=${LIST}
+    echo "Files that will be modified:"
+    echo $list_files
+   
+    # Flags that must be added 
+    FLAGS=$(./check_required_setup.sh $COMP)
+
+    # Add the flags
+    for file in $list_files
+    do
+        echo $file
+        BASE="IRPF90_FLAGS : --ninja"
+        ACTUAL=$(grep "$BASE" $file)
+        
+        # To have only one time each flag	
+	grep " -DSET_MAX_ACT" $file && ${ACTUAL/" -DSET_MAX"/""}
+	grep " -DSET_NESTED" $file && ${ACTUAL/" -DSET_NESTED"/""}
+        SPACE=" "
+
+	NEW=${ACTUAL}${SPACE}${FLAGS}
+
+	# Debug
+        #echo ${NEW}
+
+        sed "s/${ACTUAL}/${NEW}/" $file
+        # -i # to change the files
+    done
+
+fi