From 27af80c04eeab255629b77abf44f7d4153270c94 Mon Sep 17 00:00:00 2001 From: Olivier Parcollet Date: Wed, 21 May 2014 14:33:07 +0200 Subject: [PATCH] Serialization boost/h5 and clean parameters - parameters : clean, add back serialization. Clean whitespace - serialization : depending on the version of the hdf5 lib, uses h5 or boost. - TODO : test it on a machine with new hdf5. --- pytriqs/parameters/parameters_desc.py | 3 +- triqs/parameters/_field.cpp | 16 +--- triqs/parameters/_field.hpp | 119 +++++++++++++------------- triqs/parameters/parameters.cpp | 53 +++++++++--- triqs/parameters/parameters.hpp | 60 ++++++++++--- triqs/utility/boost_serialization.hpp | 74 ++++++++++++++++ triqs/utility/formatted_output.hpp | 54 ------------ triqs/utility/serialization.hpp | 63 +++----------- 8 files changed, 237 insertions(+), 205 deletions(-) create mode 100644 triqs/utility/boost_serialization.hpp delete mode 100644 triqs/utility/formatted_output.hpp diff --git a/pytriqs/parameters/parameters_desc.py b/pytriqs/parameters/parameters_desc.py index dfd26120..8c2c551e 100644 --- a/pytriqs/parameters/parameters_desc.py +++ b/pytriqs/parameters/parameters_desc.py @@ -3,7 +3,6 @@ from wrap_generator import * # The module module = module_(full_name = "pytriqs.parameters.parameters", doc = "TO BE WRITTEN") module.add_include("") -module.add_include("") module.add_using("namespace triqs::params") # one class @@ -20,7 +19,7 @@ g = class_( #add a constructor #g.add_constructor(doc = "DOC of constructor", args = []) -g.add_method(py_name = "help", calling_pattern = "auto result = triqs::utility::print_formatted(self_c.generate_help())", signature = "std::string()", doc = "help") +g.add_method(py_name = "help", calling_pattern = "auto result = self_c.help()", signature = "std::string()", doc = "help") # add getitem/setitem ... g.add_getitem(signature = "PyObject *(const char * key)", diff --git a/triqs/parameters/_field.cpp b/triqs/parameters/_field.cpp index 5fab8b49..fa46eefb 100644 --- a/triqs/parameters/_field.cpp +++ b/triqs/parameters/_field.cpp @@ -45,18 +45,10 @@ namespace params { if (obj.has_type()) { return lex_cast_from_string(obj); } -#define TRANSFORM_TYPE(T) \ - if (obj.has_type()) return extract(obj) - TRANSFORM_TYPE(int); - // TRANSFORM_TYPE(unsigned int); - TRANSFORM_TYPE(long); - // TRANSFORM_TYPE(unsigned long); - TRANSFORM_TYPE(short); - // TRANSFORM_TYPE(unsigned short); - TRANSFORM_TYPE(long long); -// TRANSFORM_TYPE(unsigned long long); -// TRANSFORM_TYPE(float); -#undef TRANSFORM_TYPE + try { + if (obj.has_type()) return extract(obj); + } + catch(...){} TRIQS_RUNTIME_ERROR << "extraction of " << obj.name() << " impossible : type mismatch. Got " << obj.type_name() << ", while I am supposed to extract a double"; } diff --git a/triqs/parameters/_field.hpp b/triqs/parameters/_field.hpp index a52ff236..5594a4f4 100644 --- a/triqs/parameters/_field.hpp +++ b/triqs/parameters/_field.hpp @@ -19,32 +19,28 @@ * ******************************************************************************/ #pragma once -#include +#include +#ifdef TRIQS_WITH_PYTHON_SUPPORT +#include +#endif #include #include #include #include #include #include - -//#include - -#include -#include +// serialization will use hdf5 with a failsafe to boost for old hdf5 libs. +#include namespace triqs { namespace params { - using triqs::get_triqs_hdf5_data_scheme; + using triqs::get_triqs_hdf5_data_scheme; // bring it into name resolution template std::ostream &operator<<(std::ostream &out, std::vector const &v) { out << "["; - if (v.size() > 3) - out << v[0] << ", ..., " << v[v.size()-1]; - else { - int c = 0; - for (auto const &x : v) out << (c++ ? ", " : "") << x; - } + int c = 0; + for (auto const &x : v) out << (c++ ? ", " : "") << x; return out << "]"; } @@ -52,21 +48,13 @@ namespace params { #define TRIQS_UTIL_OPAQUE_OBJECT_PREDEFINED_CAST \ (int)(long)(long long)(unsigned int)(unsigned long)(unsigned long long)(double)(bool)(std::string) - // a trait to compute the type actually stored in the opaque object. - // T except for integers, which are all stored as long - template struct storage_t_impl { - using type = T; - }; + // the type actually stored + template struct storage_t_impl : std::conditional::value, long, T>{}; template using storage_t = typename storage_t_impl>::type; - template struct storage_t_impl::value>> { - using type = long; - }; - // differentiate value and view ? template struct storage_t_impl::value>> { using type = arrays::array; }; - template<> struct storage_t_impl {using type = std::string;}; template<> struct storage_t_impl {using type = std::string;}; @@ -80,14 +68,14 @@ namespace params { virtual _data *clone() const = 0; virtual void h5_write_(h5::group, std::string const &) const = 0; virtual void h5_read_(h5::group, std::string const &) = 0; - - //virtual std::string serialize() const = 0; - //virtual void deserialize(std::string const &) = 0; - + virtual std::string serialize() const = 0; + virtual void deserialize(std::string const &) = 0; virtual std::ostream &print(std::ostream &out) const = 0; +#ifdef TRIQS_WITH_PYTHON_SUPPORT virtual bool from_python_convertible(PyObject *) const = 0; virtual void set_from_python(PyObject *) = 0; virtual PyObject *to_python() const = 0; +#endif }; template struct _data_impl : _data { @@ -101,19 +89,18 @@ namespace params { virtual _data *clone() const override { return new _data_impl(*this); } virtual void h5_write_(h5::group f, std::string const &name) const override { h5_write(f, name, x); } virtual void h5_read_(h5::group f, std::string const &name) override { h5_read(f, name, x); } - - /* virtual std::string serialize() const override { return triqs::serialize(x); } virtual void deserialize(std::string const &s) override { x = triqs::deserialize(s); }; - */ virtual std::ostream &print(std::ostream &out) const override { return out << x; } +#ifdef TRIQS_WITH_PYTHON_SUPPORT virtual bool from_python_convertible(PyObject *ob) const override { return py_tools::py_converter::is_convertible(ob, true); } virtual void set_from_python(PyObject *ob) override { x = py_tools::py_converter::py2c(ob); } virtual PyObject *to_python() const override { return py_tools::py_converter::c2py(x); } +#endif }; std::type_index index; @@ -121,18 +108,28 @@ namespace params { static std::map type_names; std::string name_; // for_error_messages bool modified = false; - + bool no_default_value = false; + // only parameters will construct _field template - _field(T obj, std::string n, bool modification_required) - : index(typeid(storage_t)), p(new _data_impl>{std::move(obj)}), name_(n), modified(!modification_required) { + _field(T obj, std::string n, bool without_default_value) + : index(typeid(storage_t)), p(new _data_impl>{std::move(obj)}), name_(n), no_default_value(without_default_value) { type_names.insert({index, get_triqs_hdf5_data_scheme(storage_t{})}); } + + std::string type_name() const { return type_name(index); } + static std::string type_name(std::type_index i); + + const void *get() const { return (p ? p->get() : nullptr); } + template friend T extract(_field const &ob); + template bool has_type() const { return index == typeid(T); } public: - //_field() : index(typeid(void)) {} // BREAKS invariant : only used for BOOST serialization... +#ifdef TRIQS_SERIALIZATION_WITH_BOOST + _field() : index(typeid(void)) {} // BREAKS invariant : only used for BOOST serialization. +#endif _field(_field &&c) = default; - _field(_field const &x) : index(x.index), p(x.p ? x.p->clone() : nullptr), name_(x.name_), modified(x.modified) {} // regular type + _field(_field const &x) : index(x.index), p(x.p ? x.p->clone() : nullptr), name_(x.name_), modified(x.modified), no_default_value(x.no_default_value) {} // regular type _field &operator=(_field &&c) = default; _field &operator=(_field const &x) { return operator=(_field(x)); } @@ -147,33 +144,38 @@ namespace params { return *this; } - // rewrite a few cases for convenience ... + // rewrite a few cases for practical convenience ... _field &operator=(int rhs) { // a special case where we can correct : int -> double if (index == typeid(double)) return operator=(double(rhs)); - return operator=(long(rhs));// beware infinite loop! + return operator=(long(rhs));// beware infinite loop! Works because int != long ... Clean this ... } - - // special treatment for const char *: fall back to string _field &operator=(const char *rhs) { return operator=(std::string(rhs)); } - // for subgroups only : implemented after parameters. Check type at runtime + // In the case where the data is itself a parameters (used to implement subgroups) + // we suplement _field with a few parameters methods + // Implemented after parameters (we need the type). + // Strict check type at runtime. template _field &add_field(T &&... x); _field &operator[](const char * key); _field const &operator[](const char * key) const; friend bool is_parameter(_field const & f); _field& add_group(std::string const& key, std::string const& doc); - std::string type_name() const { return type_name(index); } - + /// Name of the field std::string const &name() const { return name_; } - bool is_modified() const { return modified; } - const void *get() const { return (p ? p->get() : nullptr); } - template bool has_type() const { return index == typeid(T); } - static std::string type_name(std::type_index i); + /// Has the field been modified + bool is_modified() const { return modified; } + + /// Is a modification required for this field + bool is_modification_required() const { return no_default_value && (!modified); } + +#ifdef TRIQS_WITH_PYTHON_SUPPORT + /// Convertions python <-> C++ bool from_python_convertible(PyObject *ob) const { return p->from_python_convertible(ob); } void set_from_python(PyObject *ob) { p->set_from_python(ob); } PyObject *to_python() const { return p->to_python(); } +#endif // implemented later, since it needs the extract function ... #define CAST_OPERATOR(r, data, T) operator T() const; @@ -181,20 +183,17 @@ namespace params { #undef CAST_OPERATOR // ----- Boost serialisation -/* - template - void save(Archive & ar, const unsigned int version) const { - std::string s = p->serialize(); - ar << TRIQS_MAKE_NVP("seria_str", s); - } - template - void load(Archive & ar, const unsigned int version) { - std::string s; - ar >> TRIQS_MAKE_NVP("seria_str", s); - p->deserialize(s); - } - BOOST_SERIALIZATION_SPLIT_MEMBER(); -*/ + + template void save(Archive &ar, const unsigned int version) const { + std::string s = p->serialize(); + ar << TRIQS_MAKE_NVP("seria_str", s); + } + template void load(Archive &ar, const unsigned int version) { + std::string s; + ar >> TRIQS_MAKE_NVP("seria_str", s); + p->deserialize(s); + } + BOOST_SERIALIZATION_SPLIT_MEMBER(); friend std::ostream &operator<<(std::ostream &out, _field const &ob) { return ob.p->print(out); } diff --git a/triqs/parameters/parameters.cpp b/triqs/parameters/parameters.cpp index 758f0f7d..417600a5 100644 --- a/triqs/parameters/parameters.cpp +++ b/triqs/parameters/parameters.cpp @@ -1,5 +1,4 @@ #include -#include #include #include "./parameters.hpp" #include @@ -29,6 +28,8 @@ namespace params { A(4); } } +#undef A +#undef R parameters::_data_t::iterator parameters::find(std::string const& key) { return std::find_if(_data.begin(), _data.end(), [&key](_data_elem const& x) { return x.key == key; }); @@ -76,9 +77,9 @@ namespace params { void parameters::update(parameters const& other) { for (auto const& kfd : other._data) { auto it = find(kfd.key); - if (it !=_data.end()) { + if (it !=_data.end()) { // already there if (it->f.index != kfd.f.index) TRIQS_RUNTIME_ERROR << "Index mismatch in merging parameters"; - if (kfd.doc.size()>0) it->doc = kfd.doc; + if (kfd.doc.size() > 0) it->doc = kfd.doc; // update doc iif there is a doc ... } else _data.push_back(kfd); } @@ -86,45 +87,73 @@ namespace params { //----------------------------------------------------------------------- - std::vector> parameters::generate_help() const { + std::vector> parameters::generate_help(bool with_header) const { std::vector> str; - str.push_back({"parameter:", "type:", "value:", "description:"}); + if (with_header) str.push_back({"parameter:", "type:", "value:", "description:"}); for (auto const& s : _data) - if (!s.f.is_modified()) str.push_back({s.key, s.f.type_name(), "-", s.doc}); + if (s.f.is_modification_required()) str.push_back({s.key, s.f.type_name(), "-", s.doc}); for (auto const& s : _data) { if (is_parameter(s.f)) continue; // no subgroup - if (s.f.is_modified()) { + if (!s.f.is_modification_required()) { std::ostringstream val; val << s.f; auto sv = val.str(); boost::algorithm::trim(sv); int size_max = 30; + std::replace( sv.begin(), sv.end(), '\n', ','); // replace all '\n' with ' ' if (sv.size()>size_max) { auto s2 = std::string(size_max+5,'.'); std::copy(sv.begin(), sv.begin()+size_max/2, s2.begin()); std::copy(sv.end()-size_max/2, sv.end(), s2.end()-size_max/2); sv = s2; } - std::replace( sv.begin(), sv.end(), '\n', ','); // replace all 'x' to 'y' str.push_back({s.key, s.f.type_name(), sv, s.doc}); } } // all sub groups after for (auto const& s : _data) { if (!is_parameter(s.f)) continue; - auto p = dynamic_cast<_field::_data_impl*>(s.f.p.get()); - auto str2 = p->x.generate_help(); + auto* p = dynamic_cast<_field::_data_impl*>(s.f.p.get()); + auto str2 = p->x.generate_help(false); for (auto & x : str2) x[0] = " " + x[0]; str2.insert(str2.begin(), {"Sub-group : ", s.key, " ", ""}); str.insert(str.end(), str2.begin(), str2.end()); } return str; } + //----------------------------------------------------------------------- + + // + std::string print_formatted(std::vector> const &out){ + std::vector max_len; + std::ostringstream str; + + for(auto const &s : out){ + max_len.resize(out[0].size(),0); + for(std::size_t i=0; imax_len[i]) max_len[i]=s[i].length(); + } + for(auto const &s : out){ + for(std::size_t i=0; i struct no_default {}; + /** - * Class for storing program parameters. - * Parameters can be added to and extracted from the parameter object using the element access operator []. - * Each element is stored by means of an object of type _field, which also stores the original type (all - * integral types are collapsed to long and char* is collapsed to std::string). - * When accessing elements, a typecheck is performed. Typecasts are allowed and are similar to the C++ rules - * for casts. If the lvalue has type double, a cast from any integral type is allowed. If the lvalue has - * arithmetic type, a boost::lexical_cast is attempted. + * DOC to be written. + * + * Provide a form for program parameters. + * + * Fields of the form can be added with add_field method (only in C++), with a name, a type, a doc, and optionally a default value. + * + * They can be retrieved from and put into the form, using the [] operator, in C++ and python. + * + * The data is stored with a strict type checking at runtime, with some type collapse (all integers into a long, + * all string, const char * into a std::string, etc). + * + * Basic type cast to basic types are provided. + * * The class is boost-serializable and implements hdf5 I/O operations. */ class parameters { @@ -56,51 +64,70 @@ namespace params { public: parameters(); - /// calls can be chained for multiple parameters + /// Add a field with a default value template parameters& add_field(std::string const& key, T&& x, std::string const& doc) { insert(key, _field{std::forward(x),key, false}, doc); return *this; } - // add a field without a default value + // Add a field without a default value template parameters& add_field(std::string const& key, no_default, std::string const& doc) { insert(key, _field{T{}, key, true}, doc); return *this; } + /// Add a subgroup (a field which it itself a parameter). parameters& add_group(std::string const& key, std::string const& doc) { insert(key, _field{parameters{}, key, false}, doc); return *this; } + /// Sort with the key. By default the list of fields is ordered as in the order of their addition to the parameter. void sort_by_key(); - bool has_key(std::string const& k) const; + /// Does the form have the key ? + bool has_key(std::string const& key) const; - /// Access the parameter key, which must be present (or it throws an exception). + /** + * Access the parameter key. + * Key must be a valid key or a TRIQS_RUNTIME_ERROR is thrown. + */ _field& operator[](const char * key); + + /** + * Access the parameter key. + * Key must be a valid key or a TRIQS_RUNTIME_ERROR is thrown. + */ _field const& operator[](const char * key) const; /// generate help in form of a table of strings containing a list of required and optional parameters - std::vector> generate_help() const; + std::vector> generate_help(bool with_header = true) const; - friend std::string get_triqs_hdf5_data_scheme(parameters) { return ""; } + /// help as a string + std::string help() const; + + /// hdf5 + friend std::string get_triqs_hdf5_data_scheme(parameters const &) { return ""; } friend void h5_write(h5::group F, std::string const& subgroup_name, parameters const& p); friend void h5_read(h5::group F, std::string const& subgroup_name, parameters& p); + + /// Ostream friend std::ostream& operator<<(std::ostream& out, parameters const& p); /** * Update with another parameter set. * If a key is present in other and not in this, add parameter to this. - * If a key is present in both, overwrite parameter in this without any check (Python-like behaviour). + * If a key is present in both, overwrite parameter with a strict type checking. */ void update(parameters const&); }; + /// Another form of update, more pythonic inline parameters operator+(parameters p1, parameters const& p2) { p1.update(p2); return p1; } + // can only be implemented after complete declaration of parameters template _field& _field::add_field(T&&... x) { auto* pp = dynamic_cast<_data_impl*>(p.get()); @@ -110,4 +137,9 @@ namespace params { } } } + +#ifdef TRIQS_WITH_PYTHON_SUPPORT +// include the generated python converter #include +#endif + diff --git a/triqs/utility/boost_serialization.hpp b/triqs/utility/boost_serialization.hpp new file mode 100644 index 00000000..7fb547af --- /dev/null +++ b/triqs/utility/boost_serialization.hpp @@ -0,0 +1,74 @@ +/******************************************************************************* + * + * TRIQS: a Toolbox for Research in Interacting Quantum Systems + * + * Copyright (C) 2012 by M. Ferrero, O. Parcollet + * + * TRIQS is free software: you can redistribute it and/or modify it under the + * terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * TRIQS is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * TRIQS. If not, see . + * + ******************************************************************************/ +#ifndef TRIQS_SERIAL_H +#define TRIQS_SERIAL_H +#include +#include +//#include +//#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triqs { + + // serialize obj into an std::string + template std::string serialize (T const & obj) { + std::string serial_str; + boost::iostreams::back_insert_device inserter(serial_str); + boost::iostreams::stream > s(inserter); + //boost::archive::binary_oarchive oa(s); + boost::archive::text_oarchive oa(s); + oa << obj; + s.flush(); + return serial_str; + } + + template T deserialize (std::string const & serial_str) { + T obj; + // wrap buffer inside a stream and deserialize serial_str into obj + boost::iostreams::basic_array_source device(serial_str.data(), serial_str.size()); + boost::iostreams::stream > s(device); + //boost::archive::binary_iarchive ia(s); + boost::archive::text_iarchive ia(s); + ia >> obj; + return obj; + } + + template void deserialize_into_view (std::string const & serial_str, T & x) { + typename regular_type_if_exists_else_type::type obj; + // wrap buffer inside a stream and deserialize serial_str into obj + boost::iostreams::basic_array_source device(serial_str.data(), serial_str.size()); + boost::iostreams::stream > s(device); + boost::archive::text_iarchive ia(s); + //boost::archive::binary_iarchive ia(s); + ia >> obj; + x= typename view_type_if_exists_else_type::type (obj); + } +} +#endif + diff --git a/triqs/utility/formatted_output.hpp b/triqs/utility/formatted_output.hpp deleted file mode 100644 index b85368e6..00000000 --- a/triqs/utility/formatted_output.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/******************************************************************************* - * - * TRIQS: a Toolbox for Research in Interacting Quantum Systems - * - * Copyright (C) 2013 by H. Hafermann - * - * TRIQS is free software: you can redistribute it and/or modify it under the - * terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * TRIQS is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * TRIQS. If not, see . - * - ******************************************************************************/ -#ifndef TRIQS_FORMATTED_OUTPUT_H -#define TRIQS_FORMATTED_OUTPUT_H - -#include -#include -#include -#include -#include - -namespace triqs { namespace utility { - - inline std::string print_formatted(std::vector> const &out){ - std::vector max_len; - std::ostringstream str; - - for(auto const &s : out){ - max_len.resize(out[0].size(),0); - for(std::size_t i=0; imax_len[i]) max_len[i]=s[i].length(); - } - for(auto const &s : out){ - for(std::size_t i=0; i. * ******************************************************************************/ -#ifndef TRIQS_SERIAL_H -#define TRIQS_SERIAL_H -#include -#include -//#include -//#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#pragma once + +#if defined(TRIQS_SERIALIZATION_USE_BOOST) or not H5_VERSION_GE(1,8,9) +#include "./boost_serialization.hpp" + +#define TRIQS_SERIALIZATION_WITH_BOOST +#else +#include "../h5/serialization.hpp" +#define TRIQS_SERIALIZATION_WITH_HDF5 namespace triqs { - - // serialize obj into an std::string - template std::string serialize (T const & obj) { - std::string serial_str; - boost::iostreams::back_insert_device inserter(serial_str); - boost::iostreams::stream > s(inserter); - //boost::archive::binary_oarchive oa(s); - boost::archive::text_oarchive oa(s); - oa << obj; - s.flush(); - return serial_str; - } - - template T deserialize (std::string const & serial_str) { - T obj; - // wrap buffer inside a stream and deserialize serial_str into obj - boost::iostreams::basic_array_source device(serial_str.data(), serial_str.size()); - boost::iostreams::stream > s(device); - //boost::archive::binary_iarchive ia(s); - boost::archive::text_iarchive ia(s); - ia >> obj; - return obj; - } - - template void deserialize_into_view (std::string const & serial_str, T & x) { - typename regular_type_if_exists_else_type::type obj; - // wrap buffer inside a stream and deserialize serial_str into obj - boost::iostreams::basic_array_source device(serial_str.data(), serial_str.size()); - boost::iostreams::stream > s(device); - boost::archive::text_iarchive ia(s); - //boost::archive::binary_iarchive ia(s); - ia >> obj; - x= typename view_type_if_exists_else_type::type (obj); - } + using h5::serialize; + using h5::deserialize; } #endif -