From 33387b6c24806e794942b6ae65c6bf2b6531384a Mon Sep 17 00:00:00 2001 From: Anthony Scemama Date: Fri, 29 Mar 2019 17:38:19 +0100 Subject: [PATCH] Memory mapped 4idx --- Basis/F12.ml | 4 +- Basis/TwoElectronIntegralsNonSeparable.ml | 4 +- Parallel_mpi/Parallel.ml | 50 +---------------- Utils/FourIdxStorage.ml | 68 ++++++++++++++++++++--- Utils/FourIdxStorage.mli | 4 +- 5 files changed, 71 insertions(+), 59 deletions(-) diff --git a/Basis/F12.ml b/Basis/F12.ml index 4f6e6e5..2540ea8 100644 --- a/Basis/F12.ml +++ b/Basis/F12.ml @@ -370,7 +370,7 @@ let of_basis_parallel f12 basis = if Parallel.master then Fis.create ~size:n `Dense else - Fis.create ~size:0 `Dense + Fis.create ~size:n `Dense in (* @@ -389,7 +389,7 @@ let of_basis_parallel f12 basis = if Parallel.master then Printf.printf "Computed %s Integrals in parallel in %f seconds\n%!" Zero_m.name (Unix.gettimeofday () -. t0); - Parallel.broadcast (lazy eri_array) + Fis.broadcast eri_array diff --git a/Basis/TwoElectronIntegralsNonSeparable.ml b/Basis/TwoElectronIntegralsNonSeparable.ml index 0b58a80..39b12b8 100644 --- a/Basis/TwoElectronIntegralsNonSeparable.ml +++ b/Basis/TwoElectronIntegralsNonSeparable.ml @@ -279,7 +279,7 @@ module Make(Zero_m : Zero_mType) = struct if Parallel.master then Fis.create ~size:n `Dense else - Fis.create ~size:0 `Dense + Fis.create ~size:n `Dense in Farm.run ~ordered:false ~f input_stream |> Stream.iter (fun l -> @@ -289,7 +289,7 @@ module Make(Zero_m : Zero_mType) = struct if Parallel.master then Printf.printf "Computed %s Integrals in parallel in %f seconds\n%!" Zero_m.name (Unix.gettimeofday () -. t0); - Parallel.broadcast (lazy eri_array) + Fis.broadcast eri_array diff --git a/Parallel_mpi/Parallel.ml b/Parallel_mpi/Parallel.ml index 2f58c5f..ec9c76b 100644 --- a/Parallel_mpi/Parallel.ml +++ b/Parallel_mpi/Parallel.ml @@ -23,49 +23,7 @@ let barrier () = Mpi.barrier Mpi.comm_world -let broadcast_bytes data root comm = - let length = Bytes.length data in - let lmax = Int32.(to_int max_int) in - if length < lmax then - Mpi.broadcast data root comm - |> ignore - else - let rec aux start = - if length - start < lmax then ( - let message = Bytes.sub data start (length - start) in - Mpi.broadcast message root comm - |> ignore; - Bytes.blit message 0 data start lmax; - ) else ( - let message = Bytes.sub data start lmax in - Mpi.broadcast message root comm - |> ignore; - Bytes.blit message 0 data start lmax; - aux (start + lmax) - ) - in - aux 0 - - -let broadcast v root comm = - if rank = root then - begin - let data = Marshal.to_bytes v [Marshal.Closures] in - ignore(Mpi.broadcast_int (Bytes.length data) root comm); - broadcast_bytes data root comm; - v - end - else - begin - (* Other processes receive length, allocate buffer, receive data, - and unmarshal it. *) - let len = Mpi.broadcast_int 0 root comm in - let data = Bytes.create len in - broadcast_bytes data root comm; - Marshal.from_bytes data 0 - end - -let broadcast x = +let broadcast_generic broadcast x = let x = if master then Some (Lazy.force x) else None @@ -75,24 +33,22 @@ let broadcast x = | None -> assert false +let broadcast x = broadcast_generic Mpi.broadcast x + let broadcast_int x = Mpi.broadcast_int x 0 Mpi.comm_world - let broadcast_int_array x = Mpi.broadcast_int_array x 0 Mpi.comm_world; x - let broadcast_float x = Mpi.broadcast_float x 0 Mpi.comm_world - let broadcast_float_array x = Mpi.broadcast_float_array x 0 Mpi.comm_world; x - let broadcast_vec x = let a = Lacaml.D.Vec.to_array x in let a = broadcast_float_array a in diff --git a/Utils/FourIdxStorage.ml b/Utils/FourIdxStorage.ml index b08f1bc..b1e45a2 100644 --- a/Utils/FourIdxStorage.ml +++ b/Utils/FourIdxStorage.ml @@ -14,6 +14,7 @@ type storage_t = type t = { size : int ; + filename : string ; four_index : storage_t ; } @@ -127,22 +128,41 @@ let set ~r1 ~r2 ~value = let increment ~r1 ~r2 = increment_four_index ~r1 ~r2 -let create ~size sparsity = +let create ~size ?temp_dir sparsity = assert (size < max_index); + let filename = Parallel.broadcast (lazy (Filename.temp_file ?temp_dir "4idx." ".tmp")) in + at_exit (fun () -> try Sys.remove filename with _ -> ()); let four_index = match sparsity with | `Dense -> let result = - Bigarray.Array2.create Float64 Bigarray.fortran_layout (size*size) (size*size) + if Parallel.master then + begin +Printf.printf "filename: %s\n%!" filename; + let fd = Unix.openfile filename [Unix.O_RDWR] 0o600 in + let result = + Unix.map_file fd Float64 Bigarray.fortran_layout true + [| size*size ; size*size |] + |> Bigarray.array2_of_genarray + in Bigarray.Array2.fill result 0.; + Parallel.barrier (); + result + end + else + begin + Parallel.barrier (); + let fd = Unix.openfile filename [Unix.O_RDONLY] 0o600 in + Unix.map_file fd Float64 Bigarray.fortran_layout false + [| size*size ; size*size |] + |> Bigarray.array2_of_genarray + end in - Bigarray.Array2.fill result 0.; Dense result - | `Sparse -> let result = Hashtbl.create (size*size+13) in Sparse result in - { size ; four_index } + { size ; filename ; four_index } let size t = t.size @@ -264,7 +284,41 @@ let to_list data = +let broadcast t = +t +(* + let size = + Parallel.broadcast (lazy t.size) + in + let bufsize = size * size * size in + let stream = to_stream t in + let rec iterate () = + let buffer = + Parallel.broadcast (lazy ( + if Stream.peek stream = None then None else + Some (Array.init bufsize (fun _ -> + try Some (Stream.next stream) + with _ -> None + )) + ) ) + in + match buffer with + | None -> () + | Some buffer -> + begin + if not Parallel.master then + Array.iter (fun x -> + match x with + | Some {i_r1 ; j_r2 ; k_r1 ; l_r2 ; value} -> + set_phys t i_r1 j_r2 k_r1 l_r2 value + | None -> () ) buffer; + iterate () + end + in iterate (); + t +*) + let four_index_transform coef source = @@ -362,6 +416,6 @@ let four_index_transform coef source = Array.iter (fun (alpha, beta, gamma, delta, x) -> set_chem destination alpha beta gamma delta x) l); - if Parallel.master then Printf.eprintf "\n"; - Parallel.broadcast (lazy destination) + if Parallel.master then Printf.eprintf "\n%!"; + broadcast destination diff --git a/Utils/FourIdxStorage.mli b/Utils/FourIdxStorage.mli index 28beef6..2df9ccd 100644 --- a/Utils/FourIdxStorage.mli +++ b/Utils/FourIdxStorage.mli @@ -20,7 +20,7 @@ type element = (** Element for the stream *) value: float } -val create : size:int -> [< `Dense | `Sparse ] -> t +val create : size:int -> ?temp_dir:string -> [< `Dense | `Sparse ] -> t (** If [`Dense] is chosen, internally the data is stored as a 4-dimensional [Bigarray]. Else, it is stored as a hash table. *) @@ -68,3 +68,5 @@ val of_file : size:int -> sparsity:[< `Dense | `Sparse ] -> Scanf.Scanning.file_name -> t (** Read from a text file with format ["%d %d %d %d %f"]. *) +val broadcast : t -> t +(** Broadcast the data to the distributed processes. *)