Memory mapped 4idx

Basis/F12.ml

@@ -370,7 +370,7 @@ let of_basis_parallel f12 basis =
     if Parallel.master then
       Fis.create ~size:n `Dense
     else
-      Fis.create ~size:0 `Dense
+      Fis.create ~size:n `Dense
   in
 
   (*
@@ -389,7 +389,7 @@ let of_basis_parallel f12 basis =
   if Parallel.master then
     Printf.printf
       "Computed %s Integrals in parallel in %f seconds\n%!" Zero_m.name (Unix.gettimeofday () -. t0);
-  Parallel.broadcast (lazy eri_array)
+  Fis.broadcast eri_array
 
 
 

Basis/TwoElectronIntegralsNonSeparable.ml

@@ -279,7 +279,7 @@ module Make(Zero_m : Zero_mType) = struct
       if Parallel.master then
         Fis.create ~size:n `Dense
       else
-        Fis.create ~size:0 `Dense
+        Fis.create ~size:n `Dense
     in
     Farm.run ~ordered:false ~f input_stream
     |> Stream.iter (fun l ->
@@ -289,7 +289,7 @@ module Make(Zero_m : Zero_mType) = struct
     if Parallel.master then
       Printf.printf
         "Computed %s Integrals in parallel in %f seconds\n%!" Zero_m.name (Unix.gettimeofday () -. t0);
-    Parallel.broadcast (lazy eri_array)
+    Fis.broadcast eri_array
 
 
 

Parallel_mpi/Parallel.ml

@@ -23,49 +23,7 @@ let barrier () =
   Mpi.barrier Mpi.comm_world
 
 
-let broadcast_bytes data root comm =
-  let length = Bytes.length data in
-  let lmax = Int32.(to_int max_int) in
-  if length < lmax then
-    Mpi.broadcast data root comm
-    |> ignore
-  else
-    let rec aux start =
-      if length - start < lmax then (
-        let message = Bytes.sub data start (length - start) in
-        Mpi.broadcast message root comm
-        |> ignore;
-        Bytes.blit message 0 data start lmax;
-      ) else (
-        let message = Bytes.sub data start lmax in
-        Mpi.broadcast message root comm
-        |> ignore;
-        Bytes.blit message 0 data start lmax;
-        aux (start + lmax)
-      )
-    in
-    aux 0 
-
-
-let broadcast v root comm =
-  if rank = root then
-    begin
-      let data = Marshal.to_bytes v [Marshal.Closures] in
-      ignore(Mpi.broadcast_int (Bytes.length data) root comm);
-      broadcast_bytes data root comm;
-      v
-    end
-  else
-    begin
-      (* Other processes receive length, allocate buffer, receive data,
-             and unmarshal it. *)
-      let len = Mpi.broadcast_int 0 root comm in
-      let data = Bytes.create len in
-      broadcast_bytes data root comm;
-      Marshal.from_bytes data 0
-    end
-
-let broadcast x =
+let broadcast_generic broadcast x =
   let x = 
     if master then Some (Lazy.force x)
     else None
@@ -75,24 +33,22 @@ let broadcast x =
   | None -> assert false
 
 
+let broadcast x = broadcast_generic Mpi.broadcast x
+
 let broadcast_int x =
   Mpi.broadcast_int x 0 Mpi.comm_world
 
-
 let broadcast_int_array x =
   Mpi.broadcast_int_array x 0 Mpi.comm_world;
   x
 
-
 let broadcast_float x =
   Mpi.broadcast_float x 0 Mpi.comm_world
 
-
 let broadcast_float_array x =
   Mpi.broadcast_float_array x 0 Mpi.comm_world;
   x
 
-
 let broadcast_vec x =
   let a = Lacaml.D.Vec.to_array x  in
   let a = broadcast_float_array a in

Utils/FourIdxStorage.ml

@@ -14,6 +14,7 @@ type storage_t =
 type t =
   {
     size         : int ;
+    filename     : string ;
     four_index   : storage_t ;
   }
 
@@ -127,22 +128,41 @@ let set ~r1 ~r2 ~value =
 let increment ~r1 ~r2 = 
   increment_four_index ~r1 ~r2 
 
-let create ~size sparsity =
+let create ~size ?temp_dir sparsity =
   assert (size < max_index);
+  let filename = Parallel.broadcast (lazy (Filename.temp_file ?temp_dir "4idx." ".tmp")) in
+  at_exit (fun () -> try Sys.remove filename with _ -> ());
   let four_index = 
     match sparsity with
     | `Dense ->
       let result = 
-        Bigarray.Array2.create Float64 Bigarray.fortran_layout (size*size) (size*size)
+        if Parallel.master then
+          begin
+Printf.printf "filename: %s\n%!" filename;
+            let fd = Unix.openfile filename [Unix.O_RDWR] 0o600 in
+            let result = 
+              Unix.map_file fd Float64 Bigarray.fortran_layout  true
+                [| size*size ; size*size |]
+              |> Bigarray.array2_of_genarray
+            in Bigarray.Array2.fill result 0.;
+            Parallel.barrier ();
+            result
+          end
+        else
+          begin
+            Parallel.barrier ();
+            let fd = Unix.openfile filename [Unix.O_RDONLY] 0o600 in
+            Unix.map_file fd Float64 Bigarray.fortran_layout false
+              [| size*size ; size*size |]
+            |> Bigarray.array2_of_genarray
+          end
       in
-      Bigarray.Array2.fill result 0.;
       Dense result
-
     | `Sparse -> 
       let result = Hashtbl.create (size*size+13) in
       Sparse result
   in
-  { size ; four_index }
+  { size ; filename ; four_index }
 
 
 let size t = t.size
@@ -264,7 +284,41 @@ let to_list data =
 
 
 
+let broadcast t = 
+t
+(*
+  let size =
+    Parallel.broadcast (lazy t.size)
+  in
+  let bufsize = size * size * size in
+  let stream = to_stream t in
+  let rec iterate () =
+    let buffer =
+      Parallel.broadcast (lazy (
+        if Stream.peek stream = None then None else
+        Some (Array.init bufsize (fun _ -> 
+           try Some (Stream.next stream)
+           with _ -> None
+         ))
+      ) )
+    in
+    match buffer with
+    | None -> ()
+    | Some buffer ->
+      begin
+        if not Parallel.master then
+          Array.iter (fun x ->
+            match x with
+            | Some {i_r1 ; j_r2 ; k_r1 ; l_r2 ; value} ->
+                  set_phys t i_r1 j_r2 k_r1 l_r2 value
+            | None -> () ) buffer;
+        iterate ()
+      end
+  in iterate ();
+  t
+*)
 
+     
 
 let four_index_transform coef source =
 
@@ -362,6 +416,6 @@ let four_index_transform coef source =
      Array.iter (fun (alpha, beta, gamma, delta, x) ->
       set_chem destination alpha beta gamma delta x) l);
 
-  if Parallel.master then Printf.eprintf "\n";
-  Parallel.broadcast (lazy destination) 
+  if Parallel.master then Printf.eprintf "\n%!";
+  broadcast destination
 

Utils/FourIdxStorage.mli

@@ -20,7 +20,7 @@ type element = (** Element for the stream *)
   value: float
 }
 
-val create : size:int -> [< `Dense | `Sparse ] -> t
+val create : size:int -> ?temp_dir:string -> [< `Dense | `Sparse ] -> t
 (** If [`Dense] is chosen, internally the data is stored as a 4-dimensional
     [Bigarray]. Else, it is stored as a hash table.
 *)
@@ -68,3 +68,5 @@ val of_file : size:int -> sparsity:[< `Dense | `Sparse ]
   -> Scanf.Scanning.file_name -> t
 (** Read from a text file with format ["%d %d %d %d %f"]. *)
 
+val broadcast : t -> t
+(** Broadcast the data to the distributed processes. *)