From f3e22a81f79fb7389ce2455c7866e06705499da4 Mon Sep 17 00:00:00 2001
From: Anthony Scemama <scemama@irsamc.ups-tlse.fr>
Date: Tue, 18 Sep 2018 00:03:39 +0200
Subject: [PATCH] Improved parallel scaling

---
 plugins/Full_CI_ZMQ/selection.irp.f           |  6 +++--
 .../selection_davidson_slave.irp.f            | 27 ++++++++++++++-----
 plugins/Generators_full/generators.irp.f      |  2 +-
 plugins/Selectors_full/selectors.irp.f        |  2 +-
 4 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/plugins/Full_CI_ZMQ/selection.irp.f b/plugins/Full_CI_ZMQ/selection.irp.f
index 631c96ab..43ada9a0 100644
--- a/plugins/Full_CI_ZMQ/selection.irp.f
+++ b/plugins/Full_CI_ZMQ/selection.irp.f
@@ -332,7 +332,8 @@ subroutine select_singles_and_doubles(i_generator,hole_mask,particle_mask,fock_d
       i = psi_bilinear_matrix_rows(l_a)
       if (nt + exc_degree(i) <= 4) then
         idx = psi_det_sorted_order(psi_bilinear_matrix_order(l_a))
-        if (psi_average_norm_contrib_sorted(idx) < 1.d-12) cycle
+!        if (psi_average_norm_contrib_sorted(idx) < 1.d-12) cycle
+        if (idx > N_det_selectors) cycle
         indices(k) = idx
         k=k+1
       endif
@@ -355,7 +356,8 @@ subroutine select_singles_and_doubles(i_generator,hole_mask,particle_mask,fock_d
         idx = psi_det_sorted_order(                   &
                  psi_bilinear_matrix_order(           &
                  psi_bilinear_matrix_transp_order(l_a)))
-        if (psi_average_norm_contrib_sorted(idx) < 1.d-12) cycle
+!        if (psi_average_norm_contrib_sorted(idx) < 1.d-12) cycle
+        if (idx > N_det_selectors) cycle
         indices(k) = idx
         k=k+1
       endif
diff --git a/plugins/Full_CI_ZMQ/selection_davidson_slave.irp.f b/plugins/Full_CI_ZMQ/selection_davidson_slave.irp.f
index fb214ad9..e778be0c 100644
--- a/plugins/Full_CI_ZMQ/selection_davidson_slave.irp.f
+++ b/plugins/Full_CI_ZMQ/selection_davidson_slave.irp.f
@@ -50,7 +50,8 @@ subroutine run_wf
 
   PROVIDE psi_det psi_coef threshold_generators threshold_selectors state_average_weight mpi_master
   PROVIDE zmq_state N_det_selectors pt2_stoch_istate N_det pt2_e0_denominator
-  PROVIDE N_det_generators N_states N_states_diag
+  PROVIDE N_det_generators N_states N_states_diag psi_energy
+
   IRP_IF MPI
     call MPI_BARRIER(MPI_COMM_WORLD, ierr)
   IRP_ENDIF
@@ -155,6 +156,12 @@ subroutine run_wf
       ! PT2
       ! ---
 
+      IRP_IF MPI
+        call MPI_BARRIER(MPI_COMM_WORLD, ierr)
+        if (ierr /= MPI_SUCCESS) then
+          print *,  irp_here, 'error in barrier'
+        endif
+      IRP_ENDIF
       call wall_time(t0)
       if (zmq_get_psi(zmq_to_qp_run_socket,1) == -1) cycle
       if (zmq_get_N_det_generators (zmq_to_qp_run_socket, 1) == -1) cycle
@@ -177,13 +184,19 @@ subroutine run_wf
 
       call wall_time(t1)
       call write_double(6,(t1-t0),'Broadcast time')
+      IRP_IF MPI
+        call MPI_BARRIER(MPI_COMM_WORLD, ierr)
+        if (ierr /= MPI_SUCCESS) then
+          print *,  irp_here, 'error in barrier'
+        endif
+      IRP_ENDIF
 
-      logical :: lstop
-      lstop = .False.
-      !$OMP PARALLEL PRIVATE(i)
-      i = omp_get_thread_num()
-      call run_pt2_slave(0,i,pt2_e0_denominator)
-      !$OMP END PARALLEL
+      if (.true.) then
+        !$OMP PARALLEL PRIVATE(i)
+        i = omp_get_thread_num()
+        call run_pt2_slave(0,i,pt2_e0_denominator)
+        !$OMP END PARALLEL
+      endif
       print *,  'PT2 done'
       FREE state_average_weight
 
diff --git a/plugins/Generators_full/generators.irp.f b/plugins/Generators_full/generators.irp.f
index dc1c7d77..5b55d1f1 100644
--- a/plugins/Generators_full/generators.irp.f
+++ b/plugins/Generators_full/generators.irp.f
@@ -13,7 +13,7 @@ BEGIN_PROVIDER [ integer, N_det_generators ]
  N_det_generators = N_det
  do i=1,N_det
    norm = norm - psi_average_norm_contrib_sorted(i)
-   if (norm - 1.d-12 < 1.d0 - threshold_generators) then
+   if (norm - 1.d-10 < 1.d0 - threshold_generators) then
      N_det_generators = i
      exit
    endif
diff --git a/plugins/Selectors_full/selectors.irp.f b/plugins/Selectors_full/selectors.irp.f
index 6e198353..ea0e8680 100644
--- a/plugins/Selectors_full/selectors.irp.f
+++ b/plugins/Selectors_full/selectors.irp.f
@@ -13,7 +13,7 @@ BEGIN_PROVIDER [ integer, N_det_selectors]
   norm = 1.d0
   do i=1,N_det
     norm = norm - psi_average_norm_contrib_sorted(i)
-    if (norm - 1.d-12 < 1.d0 - threshold_selectors) then
+    if (norm - 1.d-10 < 1.d0 - threshold_selectors) then
       N_det_selectors = i
       exit
     endif