From eabc940ab2bc4f833536a8049ea84002e4028524 Mon Sep 17 00:00:00 2001
From: Anthony Scemama <scemama@irsamc.ups-tlse.fr>
Date: Thu, 23 Jan 2020 17:30:23 +0100
Subject: [PATCH] Debugging

---
 MOBasis/HF12.ml | 59 ++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 49 insertions(+), 10 deletions(-)

diff --git a/MOBasis/HF12.ml b/MOBasis/HF12.ml
index 5abb7eb..358c513 100644
--- a/MOBasis/HF12.ml
+++ b/MOBasis/HF12.ml
@@ -35,7 +35,7 @@ let array_3_init d1 d2 d3 fx =
   in
   Util.list_range 1 d3
   |> Stream.of_list 
-  |> Farm.run ~f
+  |> Farm.run ~f ~ordered:false
   |> Stream.iter (fun (k,x) ->
         for j=1 to d2 do
             for i=1 to d1 do
@@ -44,7 +44,12 @@ let array_3_init d1 d2 d3 fx =
         done)
   ;
   if Parallel.master then Printf.printf "Broadcast d3\n" ;
-  Parallel.Node.broadcast (lazy result)
+  try Parallel.InterNode.broadcast (lazy result) 
+  with Invalid_argument _ ->
+   begin
+      Printf.eprintf "Array too large.\n%!";
+      failwith "Array too large"
+   end
 
 let array_4_init d1 d2 d3 d4 fx =
   let f (k,l) = 
@@ -67,16 +72,49 @@ let array_4_init d1 d2 d3 d4 fx =
     |> List.map (fun k -> (k,l)) )
   |> List.concat
   |> Stream.of_list 
-  |> Farm.run ~f
+  |> Farm.run ~f ~ordered:false
   |> Stream.iter (fun (k,l,x) ->
+        Printf.printf "Array 4: %d %d %d\n%!" Parallel.rank k l;
         for j=1 to d2 do
           for i=1 to d1 do
             result.{i,j,k,l} <-  x.{i,j}
           done
         done)
   ;
-  if Parallel.master then Printf.printf "Broadcast d4\n" ;
-  Parallel.Node.broadcast (lazy result)
+  if Parallel.master then Printf.printf "Broadcast d4\n%!" ;
+  try Parallel.InterNode.broadcast (lazy result)
+  with Invalid_argument _ ->
+    begin
+      Printf.eprintf "Array too large... splitting.\n%!";
+      let x = 
+        Bigarray.(Array3.create Float64 fortran_layout d1 d2 d3)
+      in
+      for l=1 to d4 do
+        if Parallel.master then
+          begin
+            for k=1 to d3 do
+              for j=1 to d2 do
+                for i=1 to d1 do
+                  x.{i,j,k} <- result.{i,j,k,l} 
+                done
+              done
+            done;
+            ignore @@ Parallel.InterNode.broadcast (lazy x)
+          end
+        else 
+          begin
+            ignore @@ Parallel.InterNode.broadcast (lazy x);
+            for k=1 to d3 do
+              for j=1 to d2 do
+                for i=1 to d1 do
+                  result.{i,j,k,l} <- x.{i,j,k} 
+                done
+              done
+            done
+          end
+      done;
+      result
+    end
 
 let array_5_init d1 d2 d3 d4 d5 fx =
   let f (l,m) =
@@ -101,8 +139,9 @@ let array_5_init d1 d2 d3 d4 d5 fx =
     |> List.map (fun l -> (l,m)) )
   |> List.concat
   |> Stream.of_list 
-  |> Farm.run ~f
+  |> Farm.run ~f ~ordered:false
   |> Stream.iter (fun (l,m,x) ->
+      Printf.printf "Array 5: %d %d\n%!" l m;
       for k=1 to d3 do
         for j=1 to d2 do
           for i=1 to d1 do
@@ -111,9 +150,9 @@ let array_5_init d1 d2 d3 d4 d5 fx =
         done
       done)
   ;
-  if Parallel.master then Printf.printf "Broadcast d5\n" ;
+  if Parallel.master then Printf.printf "Broadcast d5\n%!" ;
   try
-    Parallel.Node.broadcast (lazy result)
+    Parallel.InterNode.broadcast (lazy result)
   with Invalid_argument _ ->
     begin
       Printf.eprintf "Array too large... splitting.\n%!";
@@ -132,11 +171,11 @@ let array_5_init d1 d2 d3 d4 d5 fx =
                 done
               done
             done;
-            ignore @@ Parallel.Node.broadcast (lazy x)
+            ignore @@ Parallel.InterNode.broadcast (lazy x)
           end
         else 
           begin
-            ignore @@ Parallel.Node.broadcast (lazy x);
+            ignore @@ Parallel.InterNode.broadcast (lazy x);
             for l=1 to d4 do
               for k=1 to d3 do
                 for j=1 to d2 do