Binary output

configure.sh

@@ -41,6 +41,7 @@ export LD_LIBRARY_PATH="\${QMCCHEM_PATH}/lib:\${LD_LIBRARY_PATH}"
 export LIBRARY_PATH="\${QMCCHEM_PATH}/lib:\${LIBRARY_PATH}"
 export QMCCHEM_MPIRUN="mpirun"
 export QMCCHEM_MPIRUN_FLAGS=""
+#export QMCCHEM_IO="B"
 export C_INCLUDE_PATH="\${QMCCHEM_PATH}/include:\${C_INCLUDE_PATH}"
 #export QMCCHEM_NIC=ib0
 source \${QMCCHEM_PATH}/irpf90/bin/irpman

ocaml/Block.ml

@@ -51,6 +51,26 @@ let of_string s =
 
 
 
+let to_short_string b =
+    Printf.sprintf "%s # %s %d %d"
+      (Property.to_string     b.property)
+      (Compute_node.to_string b.compute_node)
+      b.pid
+      (Block_id.to_int        b.block_id)
+
+
+let to_string b =
+    Printf.sprintf "%s %s # %s %s %s %d"
+      (Sample.to_string       b.value )
+      (Weight.to_float        b.weight       |> string_of_float)
+      (Property.to_string     b.property)
+      (Compute_node.to_string b.compute_node)
+      (string_of_int          b.pid)
+      (Block_id.to_int        b.block_id)
+
+
+
+
 let zero =
   bytes_of_int 0
 
@@ -83,22 +103,22 @@ let read_bytes b =
      Then, read the next m bytes and return a tuple containing the decoded data and the rest.
   *)
   let l = Bytes.length b in
-  let m =
+  if l < 8 then
-    Bytes.get_int64_le b 0
+     failwith "Zero-sized bytes"
-    |> Int64.to_int
-  in
-  let nl = l-m-8 in
-  if nl > 0 then
-    (Bytes.sub b 8 m, Some (Bytes.sub b (8+m) nl))
   else
-    (Bytes.sub b 8 m, None)
+     let m =
+       Bytes.get_int64_le b 0
+       |> Int64.to_int
+     in
+     let nl = l-m-8 in
+     if nl > 0 then
+       (Bytes.sub b 8 m, Some (Bytes.sub b (8+m) nl))
+     else
+       (Bytes.sub b 8 m, None)
 
 
 
 let of_bytes b =
-  let b, _rest =
-    read_bytes b
-  in
   let rec loop accu s =
     match read_bytes s with
     | data, None -> List.rev (data :: accu)
@@ -106,7 +126,8 @@ let of_bytes b =
   in
   let result =
     match loop [] b with
-    | value :: weight :: property :: compute_node :: pid :: block_id :: [] ->
+    | property :: value :: weight :: pid :: block_id :: compute_node :: [] ->
+          Some
           { property = Property.of_bytes property;
             value    = Sample.of_bytes value;
             weight   = Weight.of_bytes weight;
@@ -114,24 +135,17 @@ let of_bytes b =
             block_id     = Block_id.of_bytes block_id;
             compute_node = Compute_node.of_bytes compute_node;
           }
-    | _ -> assert false
+    | _ -> None
   in
   result
 
 
-let to_string b =
+let of_string_or_bytes s =
   if Qmcchem_config.binary_io then
-    to_bytes b
+     Bytes.of_string s
-    |> Bytes.to_string
+     |> of_bytes
   else
-    Printf.sprintf "%s %s # %s %s %s %d"
+     of_string s
-      (Sample.to_string       b.value )
-      (Weight.to_float        b.weight       |> string_of_float)
-      (Property.to_string     b.property)
-      (Compute_node.to_string b.compute_node)
-      (string_of_int          b.pid)
-      (Block_id.to_int        b.block_id)
-
 
 let dir_name = lazy(
   let ezfio_filename =
@@ -191,26 +205,67 @@ let update_raw_data ?(locked=true) () =
        | Some x -> transform (x::new_list) tail
   in
 
-  let result =
+  if Qmcchem_config.binary_io then
-    let rec aux ic accu =
+    begin
-      let l =
+      let result =
-        try
+        let rec aux buf accu =
-          Some (input_line ic)
+          (* Read one block *)
-        with
+          let item, rest =
-        | End_of_file -> None
+            read_bytes buf
+          in
+          match of_bytes item with
+          | None   -> []
+          | Some item ->
+                  match rest with
+                  | None -> List.rev (item::accu)
+                  | Some rest -> (aux [@tailcall]) rest (item::accu)
+        in
+        List.concat_map (fun filename ->
+          let ic = open_in filename in
+          let length = in_channel_length ic in
+          let result =
+            if length > 0 then 
+                let buf = Bytes.create length in
+                really_input ic buf 0 length;
+                aux buf []
+            else []
+          in
+          close_in ic;
+          result ) files
       in
-      match l with
+      result
-      | None -> List.rev accu
+    end
-      | Some l -> (aux [@tailcall]) ic (l::accu)
+  else
-    in
+    begin
-    List.concat_map (fun filename ->
+      let result =
-      let ic = open_in filename in
+        let rec aux ic accu =
-      let result = aux ic [] in
+          let l =
-      close_in ic;
+            try
-      result ) files
+              Some (input_line ic)
-    |> transform []
+            with
-  in
+            | End_of_file -> None
-  result
+          in
+          match l with
+          | None -> List.rev accu
+          | Some l -> (aux [@tailcall]) ic (l::accu)
+          in
+          List.concat_map (fun filename ->
+            let ic = open_in filename in
+            let result = aux ic [] in
+            close_in ic;
+            result ) files
+        |> transform []
+      in
+      result
+    end
+
+
+let to_string_or_bytes b =
+  if Qmcchem_config.binary_io then
+    to_bytes b
+    |> Bytes.to_string
+  else
+    to_string b
 
 
 let raw_data ?(locked=true) () =

ocaml/Qmcchem_config.ml

@@ -115,5 +115,8 @@ let ip_address = lazy (
 
 
 
-let binary_io = false
+let binary_io =
+  try
+    Sys.getenv "QMCCHEM_IO" = "B"
+  with Not_found -> false
 

ocaml/Qmcchem_dataserver.ml

@@ -757,7 +757,7 @@ let run ?(daemon=true) ezfio_filename =
               match wall with
               | Some wall ->
                 begin
-                  output_string !block_channel (Block.to_string wall);
+                  output_string !block_channel (Block.to_string_or_bytes wall);
                   if not Qmcchem_config.binary_io then
                     output_char   !block_channel '\n';
                 end
@@ -767,10 +767,10 @@ let run ?(daemon=true) ezfio_filename =
             begin
               if (status = Status.Running) then
                 touch_worker b.Block.compute_node b.Block.pid ;
-              output_string !block_channel (Block.to_string b);
+              output_string !block_channel (Block.to_string_or_bytes b);
               if not Qmcchem_config.binary_io then
                 output_char   !block_channel '\n';
-              recv_log (Block.to_string b)
+              recv_log (Block.to_short_string b)
             end
           | Message.Test
           | Message.GetWalkers _

ocaml/Random_variable.ml

@@ -740,8 +740,9 @@ let compress_files () =
             *)
     in
     List.iter (fun x ->
-      output_string block_channel (Block.to_string x);
+      output_string block_channel (Block.to_string_or_bytes x);
-      output_char   block_channel '\n';
+      if not Qmcchem_config.binary_io then
+        output_char   block_channel '\n';
     ) l.data
   ) properties ;
   close_out block_channel;