|
[yoda-svn] r430 - in trunk: . include/YODA srcblackhole at projects.hepforge.org blackhole at projects.hepforge.orgThu Dec 8 18:55:03 GMT 2011
Author: hoeth Date: Thu Dec 8 18:55:03 2011 New Revision: 430 Log: ReaderYODA can now parse Histo1D and Profile1D flat files Added: trunk/src/ReaderYODA.cc Modified: trunk/ChangeLog trunk/include/YODA/ReaderYODA.h trunk/src/Makefile.am Modified: trunk/ChangeLog ============================================================================== --- trunk/ChangeLog Thu Dec 8 18:51:29 2011 (r429) +++ trunk/ChangeLog Thu Dec 8 18:55:03 2011 (r430) @@ -1,9 +1,17 @@ +2011-12-08 Hendrik Hoeth <hendrik.hoeth at cern.ch> + + * ReaderYODA can now parse Histo1D and Profile1D flat files + 2011-12-08 Andy Buckley <andy.buckley at cern.ch> * Adding a Utils::ndarray object and using it to implement a general Scatter<N> system, with generalised Point<N> and Error<N> to boot. +2011-12-07 Hendrik Hoeth <hendrik.hoeth at cern.ch> + + * Lots of cleanup + 2011-12-07 Andy Buckley <andy.buckley at cern.ch> * Mapping the Dbn1D and Dbn2D classes into Python. Modified: trunk/include/YODA/ReaderYODA.h ============================================================================== --- trunk/include/YODA/ReaderYODA.h Thu Dec 8 18:51:29 2011 (r429) +++ trunk/include/YODA/ReaderYODA.h Thu Dec 8 18:55:03 2011 (r430) @@ -8,14 +8,22 @@ #include "YODA/AnalysisObject.h" #include "YODA/Reader.h" +#include <YODA/Histo1D.h> +#include <YODA/Profile1D.h> +#include <boost/spirit/include/qi.hpp> +#include <boost/spirit/include/phoenix_operator.hpp> +#include <boost/fusion/include/adapt_struct.hpp> namespace YODA { + using namespace boost::spirit; + using namespace boost::phoenix; /// @brief Persistency reader from YODA flat text data format. class ReaderYODA : public Reader { public: + /// Singleton creation function static Reader& create() { static ReaderYODA _instance; return _instance; @@ -30,11 +38,307 @@ void _readDoc(std::istream& stream, std::vector<AnalysisObject*>& aos); private: + + void cleanup() { + _histo1d.bins.clear(); + _histo1d.dbn_tot.reset(); + _histo1d.dbn_uflow.reset(); + _histo1d.dbn_oflow.reset(); + + _profile1d.bins.clear(); + _profile1d.dbn_tot.reset(); + _profile1d.dbn_uflow.reset(); + _profile1d.dbn_oflow.reset(); + + _annotations.clear(); + } + + public: + + /// Private constructor, since it's a singleton. ReaderYODA() { } + + + /// Here comes everything we need for the parser + + + /// Structs for the Histo1D parser + /// The data for Dbn1D + struct histo1ddbn { + double sumW; + double sumW2; + double sumWX; + double sumWX2; + unsigned long numFills; + }; + + /// A Histo1D bin + struct histo1dbin { + double lowedge; + double highedge; + histo1ddbn dbn; + }; + + /// Structs for the Profile1D parser + /// The data for Dbn2D (except sumWXY) + struct profile1ddbn { + double sumW; + double sumW2; + double sumWX; + double sumWX2; + double sumWY; + double sumWY2; + unsigned long numFills; + }; + + /// A Profile1D bin + struct profile1dbin { + double lowedge; + double highedge; + profile1ddbn dbn; + }; + + /// Structs for the key-value pair parser (annotations) + struct keyval { + std::string key; + std::string val; + }; + + + /// Annotations (common for all data types) + static std::map<std::string, std::string> _annotations; + + + /// All information for creating a Histo1D + struct histo1d { + std::vector<YODA::HistoBin1D> bins; + YODA::Dbn1D dbn_tot; + YODA::Dbn1D dbn_uflow; + YODA::Dbn1D dbn_oflow; + }; + static histo1d _histo1d; + + + /// All information for creating a Profile1D + struct profile1d { + std::vector<YODA::ProfileBin1D> bins; + YODA::Dbn2D dbn_tot; + YODA::Dbn2D dbn_uflow; + YODA::Dbn2D dbn_oflow; + }; + static profile1d _profile1d; + + + /// Functions to call from the parser + /// Filling the dbn_tot + struct filltotaldbn { + void operator()(const histo1ddbn dbn, qi::unused_type, qi::unused_type) const { + _histo1d.dbn_tot = YODA::Dbn1D(dbn.numFills, dbn.sumW, dbn.sumW2, dbn.sumWX, dbn.sumWX2); + } + void operator()(const profile1ddbn dbn, qi::unused_type, qi::unused_type) const { + _profile1d.dbn_tot = YODA::Dbn2D(dbn.numFills, dbn.sumW, dbn.sumW2, dbn.sumWX, dbn.sumWX2, dbn.sumWY, dbn.sumWY2, 0.0); + } + }; + + + /// Filling the underflow + struct filluflowdbn { + void operator()(const histo1ddbn dbn, qi::unused_type, qi::unused_type) const { + _histo1d.dbn_uflow = YODA::Dbn1D(dbn.numFills, dbn.sumW, dbn.sumW2, dbn.sumWX, dbn.sumWX2); + } + void operator()(const profile1ddbn dbn, qi::unused_type, qi::unused_type) const { + _profile1d.dbn_uflow = YODA::Dbn2D(dbn.numFills, dbn.sumW, dbn.sumW2, dbn.sumWX, dbn.sumWX2, dbn.sumWY, dbn.sumWY2, 0.0); + } + }; + + + /// Filling the overflow + struct filloflowdbn { + void operator()(const histo1ddbn dbn, qi::unused_type, qi::unused_type) const { + _histo1d.dbn_oflow = YODA::Dbn1D(dbn.numFills, dbn.sumW, dbn.sumW2, dbn.sumWX, dbn.sumWX2); + } + void operator()(const profile1ddbn dbn, qi::unused_type, qi::unused_type) const { + _profile1d.dbn_oflow = YODA::Dbn2D(dbn.numFills, dbn.sumW, dbn.sumW2, dbn.sumWX, dbn.sumWX2, dbn.sumWY, dbn.sumWY2, 0.0); + } + }; + + + /// Filling a bin + struct fillbin { + void operator()(const histo1dbin b, qi::unused_type, qi::unused_type) const { + YODA::HistoBin1D bin(b.lowedge, b.highedge, YODA::Dbn1D(b.dbn.numFills, b.dbn.sumW, b.dbn.sumW2, b.dbn.sumWX, b.dbn.sumWX2)); + _histo1d.bins.push_back(bin); + } + void operator()(const profile1dbin b, qi::unused_type, qi::unused_type) const { + YODA::ProfileBin1D bin(b.lowedge, b.highedge, YODA::Dbn2D(b.dbn.numFills, b.dbn.sumW, b.dbn.sumW2, b.dbn.sumWX, b.dbn.sumWX2, b.dbn.sumWY, b.dbn.sumWY2, 0.0)); + _profile1d.bins.push_back(bin); + } + }; + + + /// Filling the annotations map + struct fillkeyval { + void operator()(const keyval m, qi::unused_type, qi::unused_type) const { + _annotations[m.key] = m.val; + } + }; + + + /// A helper grammar for determining in which context we are. + /// bgroup and egroup are dynamic parsers. We add and remove + /// the "# BEGIN SOMETHING" and "# END SOMETHING" strings + /// dynamically. + static qi::symbols<char, int> bgroup; + static qi::symbols<char, int> egroup; + template <typename Iterator> + struct group_grammar : qi::grammar<Iterator, int()> + { + group_grammar() : group_grammar::base_type(start) { + start = begin | end; + begin = qi::eps [_val = 0] >> + qi::lit("# BEGIN ") >> + bgroup [_val += _1]; + end = qi::eps [_val = 0] >> + qi::lit("# END ") >> + egroup [_val += _1]; + } + qi::rule<Iterator, int()> start, begin, end; + }; + + + /// The actual grammar for parsing the lines of a flat data file. + template <typename Iterator, typename Skipper> + struct yoda_grammar : qi::grammar<Iterator, Skipper> + { + + yoda_grammar() : yoda_grammar::base_type(line) { + + /// A line can be anything. Note that we need + /// to specify the long lines first, because the + /// first match wins. + /// In brackets we specify the functions that are + /// called in case the rule matches. + line = Profile1Dbin[fillbin()] | + Profile1Dtotal[filltotaldbn()] | + Profile1Duflow[filluflowdbn()] | + Profile1Doflow[filloflowdbn()] | + Histo1Dbin[fillbin()] | + Histo1Dtotal[filltotaldbn()] | + Histo1Duflow[filluflowdbn()] | + Histo1Doflow[filloflowdbn()] | + keyvaluepair[fillkeyval()] | + comment; + + /// Match the strings "Underflow", "Overflow" and "Total" + underflow = qi::lit("Underflow"); + overflow = qi::lit("Overflow"); + total = qi::lit("Total"); + + /// Histo1D + /// Regular bins, total statistics, underflow or overflow. + Histo1Dbin %= double_ >> double_ >> Histo1Ddbn; + Histo1Dtotal %= total >> total >> Histo1Ddbn; + Histo1Duflow %= underflow >> underflow >> Histo1Ddbn; + Histo1Doflow %= overflow >> overflow >> Histo1Ddbn; + Histo1Ddbn = double_ >> double_ >> double_ >> double_ >> ulong_; + + // Histo2D + + // Profile1D + /// Regular bins, total statistics, underflow or overflow. + Profile1Dbin %= double_ >> double_ >> Profile1Ddbn; + Profile1Dtotal %= total >> total >> Profile1Ddbn; + Profile1Duflow %= underflow >> underflow >> Profile1Ddbn; + Profile1Doflow %= overflow >> overflow >> Profile1Ddbn; + Profile1Ddbn = double_ >> double_ >> double_ >> double_ >> double_ >> double_ >> ulong_; + + // Profile2D + + // Scatter1D + + // Scatter2D + + // Scatter3D + + + /// Annotations. + /// The key is anyting up to the first equal sign, but + /// keys can't start with a number or a minus sign. The + /// value is anything after the equal sign. + key = !qi::char_("0-9-") >> *~qi::char_("="); + value = *~qi::char_("\n"); + keyvaluepair %= key >> "=" >> value; + + /// Lines starting with a "#" are comments. + comment = qi::lit("#") >> *~qi::char_("\n"); + } + + /// In the rules, the first template argument is the Iterator for the string, + /// the second one is the return type, and the last one is a "Skipper" for + /// ignoring whitespace. Note that the key/value pair doesn't ignore whitespace. + /// Most of the return values match our structs (like keyval, histo1dbin, etc.). + /// Those are used to directly fill the corresponding structs. + qi::rule<Iterator, Skipper> line, total, underflow, overflow, comment; + qi::rule<Iterator, std::string()> key, value; + qi::rule<Iterator, keyval(), Skipper> keyvaluepair; + + qi::rule<Iterator, histo1dbin(), Skipper> Histo1Dbin; + qi::rule<Iterator, histo1ddbn(), Skipper> Histo1Ddbn, Histo1Dtotal, Histo1Duflow, Histo1Doflow; + + qi::rule<Iterator, profile1dbin(), Skipper> Profile1Dbin; + qi::rule<Iterator, profile1ddbn(), Skipper> Profile1Ddbn, Profile1Dtotal, Profile1Duflow, Profile1Doflow; + }; + }; +} // end of YODA namespace + + +/// Now we need to make boost aware of the structs we want to +/// fill directly from the parser. Boost wants this magic in +/// the global scope, that's why we have it outside the namespace. + +BOOST_FUSION_ADAPT_STRUCT( + YODA::ReaderYODA::histo1ddbn, + (double, sumW) + (double, sumW2) + (double, sumWX) + (double, sumWX2) + (unsigned long, numFills) +) + +BOOST_FUSION_ADAPT_STRUCT( + YODA::ReaderYODA::histo1dbin, + (double, lowedge) + (double, highedge) + (YODA::ReaderYODA::histo1ddbn, dbn) +) + +BOOST_FUSION_ADAPT_STRUCT( + YODA::ReaderYODA::profile1ddbn, + (double, sumW) + (double, sumW2) + (double, sumWX) + (double, sumWX2) + (double, sumWY) + (double, sumWY2) + (unsigned long, numFills) +) + +BOOST_FUSION_ADAPT_STRUCT( + YODA::ReaderYODA::profile1dbin, + (double, lowedge) + (double, highedge) + (YODA::ReaderYODA::profile1ddbn, dbn) +) + +BOOST_FUSION_ADAPT_STRUCT( + YODA::ReaderYODA::keyval, + (std::string, key) + (std::string, val) +) + -} #endif Modified: trunk/src/Makefile.am ============================================================================== --- trunk/src/Makefile.am Thu Dec 8 18:51:29 2011 (r429) +++ trunk/src/Makefile.am Thu Dec 8 18:55:03 2011 (r430) @@ -15,7 +15,8 @@ WriterAIDA.cc \ WriterYODA.cc \ Reader.cc \ - ReaderAIDA.cc + ReaderAIDA.cc \ + ReaderYODA.cc libYODA_la_LIBADD = tinyxml/libtinyxml.la libYODA_la_CPPFLAGS = $(AM_CPPFLAGS) -DTIXML_USE_STL Added: trunk/src/ReaderYODA.cc ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ trunk/src/ReaderYODA.cc Thu Dec 8 18:55:03 2011 (r430) @@ -0,0 +1,116 @@ +// -*- C++ -*- +// +// This file is part of YODA -- Yet more Objects for Data Analysis +// Copyright (C) 2008-2011 The YODA collaboration (see AUTHORS for details) +// +#include "YODA/ReaderYODA.h" +#include "YODA/Utils/StringUtils.h" +#include "YODA/Exceptions.h" + +#include <iostream> +using namespace std; + +namespace YODA { + + qi::symbols<char, int> ReaderYODA::bgroup; + qi::symbols<char, int> ReaderYODA::egroup; + ReaderYODA::histo1d ReaderYODA::_histo1d; + ReaderYODA::profile1d ReaderYODA::_profile1d; + std::map<std::string, std::string> ReaderYODA::_annotations; + + + void ReaderYODA::_readDoc(std::istream& stream, vector<AnalysisObject*>& aos) { + + // These are the context groups we know. We need + // that map to dynamically change the parser depending + // on what we read in. + std::map<int, std::string> groups; + groups[1] = "YODA_HISTO1D"; + groups[2] = "YODA_HISTO2D"; + groups[3] = "YODA_PROFILE1D"; + groups[4] = "YODA_PROFILE2D"; + groups[5] = "YODA_SCATTER1D"; + groups[6] = "YODA_SCATTER2D"; + groups[7] = "YODA_SCATTER3D"; + + // Initialize the group parser + std::pair <int, std::string> pis; // To make boost's foreach happy + foreach(pis, groups) { + bgroup.add(pis.second, pis.first); + } + + // The grammars for content (yoda) and context (group) + yoda_grammar<std::string::iterator, ascii::space_type> yoda_parser; + group_grammar<std::string::iterator> group_parser; + + // Now loop over all lines of the input file + int context = 0; + bool contextchange = false; + std::string s; + while (getline(stream, s)) { + // First check if we found a "# BEGIN ..." or "# END ..." line. + // This marks a context change. + int newcontext = 0; + if (qi::parse(s.begin(), s.end(), group_parser, newcontext)) { + context = newcontext; + if (context > 0) { + // We are inside a group now, so we are looking for the corresponding + // END and ignore all BEGINs + bgroup.clear(); + egroup.add(groups[context], -context); + } + if (context < 0) { + // We are outside a group, so we are looking for any BEGIN and ignore + // all ENDs + egroup.remove(groups[-context]); + foreach(pis, groups) { + bgroup.add(pis.second, pis.first); + } + contextchange = true; + } + } + + // Depending on the context, we either want to parse the line as data, + // or to write out what we parsed so far (when leaving a group). + switch (context) { + case 1: // we are inside YODA_HISTO1D + case 2: // we are inside YODA_HISTO2D + case 3: // we are inside YODA_PROFILE1D + case 4: // we are inside YODA_PROFILE2D + case 5: // we are inside YODA_SCATTER1D + case 6: // we are inside YODA_SCATTER2D + case 7: // we are inside YODA_SCATTER3D + if (! qi::phrase_parse(s.begin(), s.end(), yoda_parser, ascii::space) ) { + std::cerr << "failed parsing this line:\n" << s << std::endl; + } + break; + case -1: // we left YODA_HISTO1D + if (contextchange) { + YODA::Histo1D* h = new YODA::Histo1D(_histo1d.bins, _histo1d.dbn_tot, _histo1d.dbn_uflow, _histo1d.dbn_oflow); + std::pair <std::string, std::string> pss; // to make boost's foreach happy + foreach (pss, _annotations) { + h->setAnnotation(pss.first, pss.second); + } + aos.push_back(h); + cleanup(); + contextchange = false; + } + break; + case -3: // we left YODA_PROFILE1D + if (contextchange) { + YODA::Profile1D* h = new YODA::Profile1D(_profile1d.bins, _profile1d.dbn_tot, _profile1d.dbn_uflow, _profile1d.dbn_oflow); + std::pair <std::string, std::string> pss; // to make boost's foreach happy + foreach (pss, _annotations) { + h->setAnnotation(pss.first, pss.second); + } + aos.push_back(h); + cleanup(); + contextchange = false; + } + break; + } + } + } + + +}
More information about the yoda-svn mailing list |