|
[Rivet-svn] r4205 - in trunk: . bin doc pyext pyext/rivetblackhole at projects.hepforge.org blackhole at projects.hepforge.orgThu Mar 7 11:35:19 GMT 2013
Author: buckley Date: Thu Mar 7 11:35:18 2013 New Revision: 4205 Log: Creating spiresbib, util, and plotinfo rivet python module submodules: this eliminates lighthisto and the standalone spiresbib modules. Util contains convenience functions for Python version testing, clean ElementTree import, and process renaming, for primary use by the rivet-* scripts. bin/: Lots of tidying/pruning of messy/defunct scripts. Added: trunk/pyext/rivet/plotinfo.py trunk/pyext/rivet/spiresbib.py - copied unchanged from r4203, trunk/pyext/spiresbib.py trunk/pyext/rivet/util.py Deleted: trunk/pyext/ez_setup.py trunk/pyext/lighthisto.py trunk/pyext/spiresbib.py Modified: trunk/ChangeLog trunk/bin/aida2root trunk/bin/make-plots trunk/bin/rivet trunk/bin/rivet-cmphistos trunk/bin/rivet-findid trunk/bin/rivet-merge-CDF_2012_NOTE10874 trunk/bin/rivet-mergeruns trunk/bin/rivet-mkanalysis trunk/bin/rivet-mkhtml trunk/bin/rivet-rescale trunk/bin/rivet-updateanalyses trunk/bin/rivet-which trunk/bin/rivetgrid trunk/bin/root2flat trunk/doc/get-spires-bib trunk/doc/mk-analysis-latex trunk/pyext/Makefile.am trunk/pyext/rivet/Makefile.am trunk/pyext/rivet/__init__.py trunk/pyext/setup.py.in Modified: trunk/ChangeLog ============================================================================== --- trunk/ChangeLog Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/ChangeLog Thu Mar 7 11:35:18 2013 (r4205) @@ -1,5 +1,13 @@ 2013-03-07 Andy Buckley <andy.buckley at cern.ch> + * bin/: Lots of tidying/pruning of messy/defunct scripts. + + * Creating spiresbib, util, and plotinfo rivet python module + submodules: this eliminates lighthisto and the standalone + spiresbib modules. Util contains convenience functions for Python + version testing, clean ElementTree import, and process renaming, + for primary use by the rivet-* scripts. + * Removing defunct scripts that have been replaced/obsoleted by YODA. 2013-03-06 Andy Buckley <andy.buckley at cern.ch> Modified: trunk/bin/aida2root ============================================================================== --- trunk/bin/aida2root Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/aida2root Thu Mar 7 11:35:18 2013 (r4205) @@ -15,13 +15,10 @@ import ROOT ROOT.PyConfig.IgnoreCommandLineOptions = True -import sys -if sys.version_info[:3] < (2,4,0): - print "rivet scripts require Python version >= 2.4.0... exiting" - sys.exit(1) - - -import os, array +import rivet, sys, os, array +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) +rivet.util.import_ET() try: from ROOT import ROOT, TGraphAsymmErrors, TGraph2DErrors, TH1F, TH2F, TFile @@ -42,20 +39,8 @@ sys.stderr.write("You can test if PyROOT is properly set up by calling 'import ROOT' at the interactive Python shell\n") sys.exit(1) -## Try to load faster but non-standard cElementTree module -try: - import xml.etree.cElementTree as ET -except ImportError: - try: - import cElementTree as ET - except ImportError: - try: - import xml.etree.ElementTree as ET - except: - sys.stderr.write("Can't load the ElementTree XML parser: please install it!\n") - sys.exit(1) - +# TODO: replace with YODA (and move this script to YODA) class Histo: def __init__(self, nDim): Modified: trunk/bin/make-plots ============================================================================== --- trunk/bin/make-plots Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/make-plots Thu Mar 7 11:35:18 2013 (r4205) @@ -19,11 +19,21 @@ ## $Revision$ ## +## Check the Python version import sys if sys.version_info[:3] < (2,4,0): - print "rivet scripts require Python version >= 2.4.0... exiting" + print "make-plots requires Python version >= 2.4.0... exiting" sys.exit(1) +## Try to rename the process on Linux +try: + import ctypes + libc = ctypes.cdll.LoadLibrary('libc.so.6') + libc.prctl(15, 'make-plots', 0, 0, 0) +except Exception, e: + pass + + import os, logging, re import tempfile import getopt Modified: trunk/bin/rivet ============================================================================== --- trunk/bin/rivet Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet Thu Mar 7 11:35:18 2013 (r4205) @@ -17,34 +17,11 @@ metadata files """ -import sys -if sys.version_info[:3] < (2, 4, 0): - print "rivet scripts require Python version >= 2.4.0... exiting" - sys.exit(1) - - -import os, time -import logging, signal - -## Try to rename the process on Linux -try: - import ctypes - libc = ctypes.cdll.LoadLibrary('libc.so.6') - libc.prctl(15, 'rivet', 0, 0, 0) -except Exception: - pass - -## Try to use Psyco optimiser -try: - import psyco - psyco.full() -except ImportError: - pass - +## Try to bootstrap the Python path for rivet loading +# TODO: Move this intorivet.__init__.py? +import os, sys PROGPATH = sys.argv[0] PROGNAME = os.path.basename(PROGPATH) - -## Try to bootstrap the Python path import commands try: modname = sys.modules[__name__].__file__ @@ -55,17 +32,10 @@ except: pass - -## Try importing rivet -try: - import rivet - rivet.check_python_version() -except Exception, e: - sys.stderr.write(PROGNAME + " requires the 'rivet' Python module but it cannot be loaded:\n") - sys.stderr.write(str(e)+'\n') - sys.stderr.write("Try re-running with the -v switch to enable more debugging output \n") - sys.exit(1) - +import rivet +rivet.util.check_python_version() +rivet.util.set_process_name("rivet") +import time, logging, signal ## Parse command line options from optparse import OptionParser, OptionGroup Modified: trunk/bin/rivet-cmphistos ============================================================================== --- trunk/bin/rivet-cmphistos Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-cmphistos Thu Mar 7 11:35:18 2013 (r4205) @@ -10,14 +10,13 @@ section. ''' -import sys, os -if sys.version_info[:3] < (2, 4, 0): - print 'rivet scripts require Python version >= 2.4.0... exiting' - sys.exit(1) +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) class Plot(dict): - ## A tiny Plot object to help writing out the head in the .dat file + "A tiny Plot object to help writing out the head in the .dat file" def __repr__(self): return "# BEGIN PLOT\n" + "\n".join("%s=%s" % (k,v) for k,v in self.iteritems()) + "\n# END PLOT\n\n" @@ -32,7 +31,7 @@ def getCommandLineOptions(): - ## Parse command line options + "Parse command line options" from optparse import OptionParser, OptionGroup parser = OptionParser(usage=__doc__) @@ -99,24 +98,20 @@ return parser -def initPlotparser(filelist): - from lighthisto import PlotParser - - ## Add standard locations and the input files' dirs to the PLOTINFO search paths - import rivet +def initPlotParser(filelist): + "Add standard locations and the input files' dirs to the PLOTINFO search paths" opts.PLOTINFODIR += rivet.getAnalysisPlotPaths() for infile in filelist: adir = os.path.abspath(os.path.split(infile)[0]) if not adir in opts.PLOTINFODIR: opts.PLOTINFODIR.append(adir) - - return PlotParser(opts.PLOTINFODIR, opts.CONFIGFILES) + return rivet.PlotParser(opts.PLOTINFODIR, opts.CONFIGFILES) def getHistos(filelist): - ## Loop over all input files. Only use the first occurrence of any REF-histogram - ## and the first occurrence in each MC file for every MC-histogram. + """Loop over all input files. Only use the first occurrence of any REF-histogram + and the first occurrence in each MC file for every MC-histogram.""" refhistos = {} mchistos = {} mchistolist = [] @@ -138,8 +133,7 @@ def getRefdata(refhistos): - ## Find all Rivet reference data files - import rivet + "Find all Rivet reference data files" rivet_data_dirs = rivet.getAnalysisRefPaths() dirlist = [] for d in rivet_data_dirs: @@ -156,9 +150,9 @@ def parseArgs(args): - ## Look at the argument list and split it at colons, in order to separate - ## the file names from the plotting options. Store the file names and - ## file specific plotting options. + """Look at the argument list and split it at colons, in order to separate + the file names from the plotting options. Store the file names and + file specific plotting options.""" filelist = [] plotoptions = {} for a in args: @@ -180,8 +174,7 @@ def setStyle(ao, style): - ## Set default plot styles (color and line width) - # colors borrowed from Google Ngrams + """Set default plot styles (color and line width) colors borrowed from Google Ngrams""" LINECOLORS = ['{[HTML]{ee3311}}', # red (Google uses 'dc3912') '{[HTML]{3366cc}}', # blue '{[HTML]{109618}}', # green @@ -207,7 +200,7 @@ def setOptions(ao, options): - ## Set arbitrary annotations + "Set arbitrary annotations" for opt in options: key = opt.split('=', 1)[0] val = opt.split('=', 1)[1] @@ -215,7 +208,7 @@ def mkoutdir(outdir): - ## Function to make output directories + "Function to make output directories" if not os.path.exists(outdir): try: os.makedirs(outdir) @@ -228,7 +221,7 @@ def writeOutput(output, h): - ## Choose output file name and dir + "Choose output file name and dir" if opts.HIER_OUTPUT: outdir = os.path.dirname(os.path.join(opts.OUTDIR, h[1:])) outfile = '%s.dat' % os.path.basename(h) @@ -243,19 +236,15 @@ #-------------------------------------------------------------------------------------------- -#-------------------------------------------------------------------------------------------- + if __name__ == '__main__': import os import yoda ## Try to rename the process on Linux - try: - import ctypes - libc = ctypes.cdll.LoadLibrary('libc.so.6') - libc.prctl(15, 'compare-histos', 0, 0, 0) - except Exception: - pass + print __file__ + rivet.util.set_process_name(__name__) ## Try to use Psyco optimiser try: @@ -264,7 +253,6 @@ except ImportError: pass - ## Command line parsing parser = getCommandLineOptions() opts, args = parser.parse_args() @@ -290,7 +278,6 @@ ## A list of all analysis objects to be plotted anaobjects = [] - ## Plot object for the PLOT section in the .dat file plot = Plot() plot['Legend'] = '1' @@ -311,7 +298,6 @@ ## order of the files on the command line drawonly = '' - ## Check if we have reference data for the histogram if refhistos.has_key('/REF' + h): refdata = refhistos['/REF' + h] @@ -367,4 +353,3 @@ ## Write everything into a file writeOutput(output, h) - Modified: trunk/bin/rivet-findid ============================================================================== --- trunk/bin/rivet-findid Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-findid Thu Mar 7 11:35:18 2013 (r4205) @@ -13,19 +13,10 @@ - SPIRES: [S]nnnnnnn - Inspire: [I]nnnnnn[n]""" -import sys,re - -try: - import xml.etree.cElementTree as ET -except ImportError: - try: - import cElementTree as ET - except ImportError: - try: - import xml.etree.ElementTree as ET - except: - sys.stderr.write("Can't load the ElementTree XML parser\n") - sys.exit(1) +import rivet, sys, os, re +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) +rivet.util.import_ET() def main(): Modified: trunk/bin/rivet-merge-CDF_2012_NOTE10874 ============================================================================== --- trunk/bin/rivet-merge-CDF_2012_NOTE10874 Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-merge-CDF_2012_NOTE10874 Thu Mar 7 11:35:18 2013 (r4205) @@ -7,32 +7,12 @@ energy ratio plots in the CDF_2012_NOTE10874 analysis. """ -import sys -if sys.version_info[:3] < (2,4,0): - print "rivet scripts require Python version >= 2.4.0... exiting" - sys.exit(1) - -import os, logging -from lighthisto import * - -## Try to load faster but non-standard cElementTree module -try: - import xml.etree.cElementTree as ET -except ImportError: - try: - import cElementTree as ET - except ImportError: - try: - import xml.etree.ElementTree as ET - except: - sys.stderr.write("Can't load the ElementTree XML parser: please install it!\n") - sys.exit(1) - - -########################################################## +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) -def CreateHistoList(histolist1, histolist2, EnergyRatioIndex): +def createHistoList(histolist1, histolist2, EnergyRatioIndex): newHistoList = [] for m in range(3): histo1= histolist1[m] @@ -69,7 +49,8 @@ ######################################################## if __name__ == "__main__": - plotparser = PlotParser() #will this work? Check. + import logging + plotparser = rivet.PlotParser() histos = {} @@ -85,7 +66,7 @@ tree = ET.parse(aidafile) except: logging.error("%s can not be parsed as XML" % aidafile) - sys.exit(1) + exit(1) for dps in tree.findall("dataPointSet"): useThis = True @@ -109,9 +90,9 @@ for energy, hs in histos.items(): sorted(hs, key=lambda hist: hist.name) - MyNewHistoList = CreateHistoList(histos['1960'], histos['300'], 4) - MyNewHistoList += CreateHistoList(histos['900'], histos['300'], 5) - MyNewHistoList += CreateHistoList(histos['1960'], histos['900'], 6) + MyNewHistoList = createHistoList(histos['1960'], histos['300'], 4) + MyNewHistoList += createHistoList(histos['900'], histos['300'], 5) + MyNewHistoList += createHistoList(histos['1960'], histos['900'], 6) outfile = "RatioPlots.dat" out = open(outfile, "w") Modified: trunk/bin/rivet-mergeruns ============================================================================== --- trunk/bin/rivet-mergeruns Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-mergeruns Thu Mar 7 11:35:18 2013 (r4205) @@ -1,5 +1,7 @@ #! /usr/bin/env python +# TODO: Rewrite or remove -- this is not a "proper" statistical run merger but a hard-coded binwise histo patching hack + """%prog Script for merging parts of multiple histo files made with different run params @@ -7,11 +9,10 @@ analysis. TODO: + * rewrite to use YODA histogram objects and formats * take merge specs from a conf file instead of hard-coding * generalise to more generic merge ranges (i.e. not just sqrts & pT) * improve cmd line interface - * rationalise all histogramming formats... remove AIDA! - * use lighthisto (then YODA) Usage example: $ uemerge hwpp/hpp-1800-{030.aida:1800:30,090.aida:1800:90} > hpp-hists.dat @@ -21,12 +22,11 @@ $ make_plot.py --pdf *.dat """ -import sys -if sys.version_info[:3] < (2,4,0): - print "rivet scripts require Python version >= 2.4.0... exiting" - sys.exit(1) +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) -import os, copy, re +import copy, re from math import sqrt @@ -39,239 +39,6 @@ return total / float(num) -## TODO: replace with lighthisto -class Histo: - def __init__(self): - self.clear() - - def clear(self): - self._bins = [] - self.path = None - self.name = None - self.title = None - self.xtitle = None - self.ytitle = None - - def __cmp__(self, other): - """Sort by $path/$name string""" - return self.fullPath() > other.fullPath() - - def __str__(self): - out = "Histogram '%s' with %d bins\n" % (self.fullPath(), self.numBins()) - if self.title: - out += "Title: %s\n" % self.title - if self.xtitle: - out += "XLabel: %s\n" % self.xtitle - if self.ytitle: - out += "YLabel: %s\n" % self.ytitle - out += "\n".join([str(b) for b in self.getBins()]) - return out - - def fullPath(self): - return os.path.join(self.path, self.name) - - def asFlat(self): - #global headerprefix - headerprefix = "" - global opts - out = "# BEGIN HISTOGRAM %s\n" % self.fullPath() - out += headerprefix + "AidaPath=%s\n" % self.fullPath() - if self.title: - out += headerprefix + "Title=%s\n" % self.title - if self.xtitle: - out += headerprefix + "XLabel=%s\n" % self.xtitle - if self.ytitle: - out += headerprefix + "YLabel=%s\n" % self.ytitle - try: - out += "## Area: %s\n" % self.area() - except: - out += "## Area: UNKNOWN (invalid bin details)" - out += "## Num bins: %d\n" % self.numBins() -# if opts.GNUPLOT: -# out += "## xval \tyval \txlow \txhigh \tylow \tyhigh\n" -# else: - #out += "## xlow \txhigh \tyval \tyerrminus\tyerrplus\n" - out += "\n".join([b.asFlat() for b in self.getBins()]) - out += "\n# END HISTOGRAM" - return out - - def numBins(self): - return len(self._bins) - - def getBins(self): - return sorted(self._bins) - - def setBins(self, bins): - self._bins = bins - return self - - def setBin(self, index, bin): - self._bins[index] = bin - return self - - def addBin(self, bin): - self._bins.append(bin) - return self - - def getBin(self, index): - self._bins.sort() - return self.getBins()[index] - - bins = property(getBins, setBins) - - def area(self): - return sum([bin.area() for bin in self.bins]) - - def __iter__(self): - return iter(self.getBins()) - - def __len__(self): - return len(self._bins) - - def __getitem__(self, index): - return self.getBin(index) - - -## TODO: replace with lighthisto -class Bin: - """A simple container for a binned value with an error.""" - def __init__(self, xlow=None, xhigh=None, yval=0, yerrplus=0, yerrminus=0, focus=None): - self.xlow = xlow - self.xhigh= xhigh - self.yval = yval - self.yerrplus = yerrplus - self.yerrminus = yerrminus - self.focus= focus - - def clear(self): - #self.xlow = None - #self.xhigh= None - self.yval = 0 - self.yerrplus = 0 - self.yerrminus = 0 - self.focus= None - - def __str__(self): - meanyerr = None - try: - meanyerr = mean(self.yerrplus, self.yerrminus) - except: - pass - out = "%s to %s: %s +- %s" % (str(self.xlow), str(self.xhigh), str(self.yval), str(meanyerr)) - return out - - def asFlat(self): - global opts -# if opts.GNUPLOT: -# out = "%e\t%e\t%e\t%e\t%e\t%e" % (self.getBinCenter(), self.yval, -# self.xlow, self.xhigh, -# self.yval-self.yerrminus, self.yval+self.yerrplus) -# else: - out = "%e\t%e\t%e\t%e" % (self.xlow, self.xhigh, self.yval, 0.5*(self.yerrminus+self.yerrplus)) - return out - - def __cmp__(self, other): - """Sort by mean x value (yeah, I know...)""" - rtn = True - lhnone = (self.xlow is None or self.xhigh is None) - rhnone = (other.xlow is None or other.xhigh is None) - somenones = lhnone or rhnone - if somenones: - if lhnone == rhnone: - return 0 - elif lhnone: - return -1 - else: - return 1; - else: - return cmp(self.xlow + self.xhigh, other.xlow + other.xhigh) - - def getXRange(self): - return (self.xlow, self.xhigh) - - def setXRange(self, xlow, xhigh): - self.xlow = xlow - self.xhigh = xhigh - return self - - def getBinCenter(self): - """Geometric middle of the bin range.""" - return self.xlow + .5*(self.xhigh - self.xlow) - - def getFocus(self): - """Mean x-value of the bin.""" - if self.focus is not None: - return (self.xlow + self.xhigh)/2.0 - else: - return self.focus - - def area(self): - return self.yval * (self.xhigh - self.xlow) - - def getYErr(self): - """Get mean of +ve and -ve y-errors.""" - return (self.yerrplus + self.yerrminus)/2.0 - - def setYErr(self, yerr): - """Set both +ve and -ve y-errors simultaneously.""" - self.yerrplus = yerr - self.yerrminus = yerr - return self - - - -## Try to load faster but non-standard cElementTree module -try: - import xml.etree.cElementTree as ET -except ImportError: - try: - import cElementTree as ET - except ImportError: - try: - import xml.etree.ElementTree as ET - except: - sys.stderr.write("Can't load the ElementTree XML parser: please install it!\n") - sys.exit(1) - - -## TODO: replace with lighthisto -def mkHistoFromDPS(dps): - """Make a mini histo representation from an AIDA dataPointSet tag.""" - myhist = Histo() - myhist.name = dps.get("name") - myhist.title = dps.get("title") - myhist.path = dps.get("path") - dims = dps.findall("dimension") - for d in dims: - if d.get("dim") == "0": - myhist.xtitle = d.get("title") - elif d.get("dim") == "1": - myhist.ytitle = d.get("title") - points = dps.findall("dataPoint") - numbins = len(points) - for binnum, point in enumerate(points): - bin = Bin() - for d, m in enumerate(point.findall("measurement")): - val = float(m.get("value")) - down = float(m.get("errorMinus")) - up = float(m.get("errorPlus")) - if d == 0: - low = val - down - high = val + up - bin.setXRange(low, high) - elif d == 1: - bin.yval = val - bin.yerrplus = up - bin.yerrminus = down - myhist.addBin(bin) - return myhist - - - -############################################# - - - def fillAbove(desthisto, sourcehistosbyptmin): for i, b in enumerate(desthisto.getBins()): ## Fill bins with pT-ordered histos (so that 'highest always wins') @@ -279,8 +46,6 @@ newb = h.getBin(i) if newb.xlow >= float(ptmin): desthisto.setBin(i, newb) - ##### This logging line breaks the output!!!!!!! - ####logging.debug("Copied: %s / %s" % (str(b), str(histosS[hpath][sqrts].getBin(i))) ) def mergeByPt(hpath, sqrts): global inhistos @@ -334,13 +99,11 @@ ## Prefix used in dat file headers headerprefix = "# " - ## Check args if len(args) < 1: logging.error("Must specify at least one AIDA histogram file") sys.exit(1) - ## Get histos inhistos = {} weights = {} Modified: trunk/bin/rivet-mkanalysis ============================================================================== --- trunk/bin/rivet-mkanalysis Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-mkanalysis Thu Mar 7 11:35:18 2013 (r4205) @@ -9,12 +9,10 @@ directory. """ -import sys -if sys.version_info[:3] < (2,4,0): - print "rivet scripts require Python version >= 2.4.0... exiting" - sys.exit(1) - -import logging, os +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) +import logging ## Handle command line @@ -33,6 +31,7 @@ logging.basicConfig(format="%(msg)s", level=opts.LOGLEVEL) ANANAMES = args + ## Work out installation paths ANAROOT = os.path.abspath(opts.SRCROOT or os.getcwd()) if not os.access(ANAROOT, os.W_OK): @@ -105,9 +104,8 @@ bibkey, bibtex = None, None if STDANA: try: - import spiresbib logging.debug("Getting Inspire/SPIRES biblio data for '%s'" % ANANAME) - bibkey, bibtex = spiresbib.get_bibtex_from_repo(INSPIRE_SPIRES, ANAINSPIREID) + bibkey, bibtex = rivet.spiresbib.get_bibtex_from_repo(INSPIRE_SPIRES, ANAINSPIREID) except Exception, e: logging.error("Inspire/SPIRES oops: %s" % e) if bibkey and bibtex: Modified: trunk/bin/rivet-mkhtml ============================================================================== --- trunk/bin/rivet-mkhtml Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-mkhtml Thu Mar 7 11:35:18 2013 (r4205) @@ -14,20 +14,11 @@ You can overwrite an existing output directory. """ -import sys, os -if sys.version_info[:3] < (2,4,0): - sys.stderr.write("rivet scripts require Python version >= 2.4.0... exiting\n") - sys.exit(1) +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) -import traceback -try: - import rivet -except ImportError: - traceback.print_exc(file=sys.stderr) - sys.stderr.write("rivet is broken... exiting\n") - sys.exit(1) - -import sys, os, glob, shutil +import glob, shutil from subprocess import Popen, PIPE Modified: trunk/bin/rivet-rescale ============================================================================== --- trunk/bin/rivet-rescale Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-rescale Thu Mar 7 11:35:18 2013 (r4205) @@ -1,5 +1,7 @@ #!/usr/bin/env python +# TODO: Rewrite or remove + """\ %prog [-r <REFDATAPATH>] [-O <observable-file>] [-b <bindef> [-b ...]] <AIDAFILE> [<OUTFILE>] @@ -45,56 +47,13 @@ data file is only applied between 2 < x < 5 GeV. """ -import sys -if sys.version_info[:3] < (2,4,0): - print "rivet scripts require Python version >= 2.4.0... exiting" - sys.exit(1) - -import os, re, logging -from lighthisto import Histo - -# try: -# from IPython.Shell import IPShellEmbed -# ipshell = IPShellEmbed([]) -# except: -# logging.info("Ipython shell not available.") - - -## Try to load faster but non-standard cElementTree module -try: - import xml.etree.cElementTree as ET -except ImportError: - try: - import cElementTree as ET - except ImportError: - try: - import xml.etree.ElementTree as ET - except: - sys.stderr.write("Can't load the ElementTree XML parser: please install it!\n") - sys.exit(1) - - -def getHistosFromAIDA(aidafile): - '''Get a dictionary of histograms indexed by name.''' - if not re.match(r'.*\.aida$', aidafile): - logging.debug("Error: input file '%s' is not an AIDA file" % aidafile) - aidafilepath = os.path.abspath(aidafile) - if not os.access(aidafilepath, os.R_OK): - logging.debug("Error: cannot read from %s" % aidafile) - - histos = {} - tree = ET.parse(aidafilepath) - for dps in tree.findall("dataPointSet"): - ## Get this histogram's path name - dpsname = os.path.join(dps.get("path"), dps.get("name")) - ## Is it a data histo? - h = Histo.fromDPS(dps) - h.isdata = dpsname.upper().startswith("/REF") - if h.isdata: - dpsname = dpsname.replace("/REF", "") - histos[dpsname] = h - return histos +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) +import re, logging +# TODO: convert to YODA +import yoda def getRefHistos(refpaths, analyses): """ Modified: trunk/bin/rivet-updateanalyses ============================================================================== --- trunk/bin/rivet-updateanalyses Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-updateanalyses Thu Mar 7 11:35:18 2013 (r4205) @@ -1,8 +1,6 @@ #! /usr/bin/env python -import os, sys, cmd, logging, re - -usage = """%prog [--unvalidated] [--all] [--repo=URL] [--dest=DEST] [--dryrun] +"""%prog [--unvalidated] [--all] [--repo=URL] [--dest=DEST] [--dryrun] Grab analysis files from the Rivet trunk repository and install them into a local directory. You wil have to build them after downloading, using @@ -18,6 +16,12 @@ * Run rivet-buildplugin on the downloaded files. """ +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) +import cmd, logging, re + + def getFileList(url): logging.debug("Getting Rivet file list from '%s'" % url) import urllib2 @@ -84,7 +88,7 @@ if __name__ == '__main__': ## Parse command line options from optparse import OptionParser - parser = OptionParser(usage=usage) + parser = OptionParser(usage=__doc__) parser.add_option("--repo", help="Base URL of online sets repository (%default)", metavar="URL", dest="URL", default="http://svn.hepforge.org/rivet/trunk") parser.add_option("--dest", help="Directory to install to (%default)", metavar="DEST", Modified: trunk/bin/rivet-which ============================================================================== --- trunk/bin/rivet-which Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivet-which Thu Mar 7 11:35:18 2013 (r4205) @@ -10,12 +10,14 @@ * Add partial name / regex searching? (Requires extending the Rivet library) """ +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) + import sys, os, optparse op = optparse.OptionParser() ops, args = op.parse_args() -import rivet - # print rivet.findAnalysisPlotFile() # print rivet.findAnalysisLibFile() # print rivet.findAnalysisInfoFile() Modified: trunk/bin/rivetgrid ============================================================================== --- trunk/bin/rivetgrid Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/rivetgrid Thu Mar 7 11:35:18 2013 (r4205) @@ -13,12 +13,9 @@ * Allow control over the submission system """ -import sys -if sys.version_info[:3] < (2,4,0): - print "rivet scripts require Python version >= 2.4.0... exiting" - sys.exit(1) - -import os, logging +import rivet +rivet.util.check_python_version() +import os, sys, logging import glob, tempfile, commands ## Parse command line options Modified: trunk/bin/root2flat ============================================================================== --- trunk/bin/root2flat Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/bin/root2flat Thu Mar 7 11:35:18 2013 (r4205) @@ -14,13 +14,10 @@ Use --help to get information for all options. """ -import sys -if sys.version_info[:3] < (2,4,0): - print "rivet scripts require Python version >= 2.4.0... exiting" - sys.exit(1) - - -import os, optparse, logging, ROOT +import rivet, sys, os +rivet.util.check_python_version() +rivet.util.set_process_name(os.path.basename(__file__)) +import optparse, logging, ROOT ## Parse options Modified: trunk/doc/get-spires-bib ============================================================================== --- trunk/doc/get-spires-bib Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/doc/get-spires-bib Thu Mar 7 11:35:18 2013 (r4205) @@ -5,40 +5,29 @@ Get SPIRES BibTeX entries for analysis info files """ -## Change dlopen status to GLOBAL for Rivet lib -import sys -try: - import ctypes - sys.setdlopenflags(sys.getdlopenflags() | ctypes.RTLD_GLOBAL) -except: - import dl - sys.setdlopenflags(sys.getdlopenflags() | dl.RTLD_GLOBAL) -import rivet - +## Parse command line options +from optparse import OptionParser +parser = OptionParser(usage=usage) +opts, args = parser.parse_args() -if __name__ == '__main__': - ## Parse command line options - from optparse import OptionParser - parser = OptionParser(usage=usage) - opts, args = parser.parse_args() +import rivet - ## If not args supplied, use all analyses - if not args: - args = rivet.AnalysisLoader.analysisNames() +## If not args supplied, use all analyses +if not args: + args = rivet.AnalysisLoader.analysisNames() - ## Make individual bibinfo files - import spiresbib - for aname in args: - ana = rivet.AnalysisLoader.getAnalysis(aname) - if not ana: - sys.exit(1) - sid = ana.spiresId() - print "Getting SPIRES biblio data for '%s'" % aname - key, bibtex = spiresbib.get_bibtex_from_spires(sid) - if key and bibtex: - f = open(aname+".bib.info", "w") - f.write("BibKey: %s\n" % key) - f.write("BibTeX: '%s'\n" % bibtex) - f.close() - else: - print "No BibTeX available for analysis '%s'" % aname +## Make individual bibinfo files +for aname in args: + ana = rivet.AnalysisLoader.getAnalysis(aname) + if not ana: + exit(1) + sid = ana.spiresId() + print "Getting SPIRES bib data for '%s'" % aname + key, bibtex = rivet.spiresbib.get_bibtex_from_spires(sid) + if key and bibtex: + f = open(aname+".bib.info", "w") + f.write("BibKey: %s\n" % key) + f.write("BibTeX: '%s'\n" % bibtex) + f.close() + else: + print "No BibTeX available for analysis '%s'" % aname Modified: trunk/doc/mk-analysis-latex ============================================================================== --- trunk/doc/mk-analysis-latex Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/doc/mk-analysis-latex Thu Mar 7 11:35:18 2013 (r4205) @@ -29,20 +29,9 @@ except: pass anadirs = glob.glob(os.path.join(os.getcwd(), "..", "src", "Analyses", ".libs")) -#print anadirs os.environ["RIVET_ANALYSIS_PATH"] = ":".join(anadirs) -## Change dlopen status to GLOBAL for Rivet lib -try: - import ctypes - sys.setdlopenflags(sys.getdlopenflags() | ctypes.RTLD_GLOBAL) -except: - import dl - sys.setdlopenflags(sys.getdlopenflags() | dl.RTLD_GLOBAL) -import rivet - - def texify(s): "Insert required TeX escapes" t = s \ @@ -63,6 +52,7 @@ ## Build analysis pages +import rivet all_analyses = rivet.AnalysisLoader.analysisNames() pages = { "LEP and SLC":[], "Tevatron":[], "LHC":[], "SPS":[], "HERA":[], "RHIC":[], "Monte Carlo":[], "Example":[], "Misc.":[] } ## Use list(...) ctor for 2.3 compatibility @@ -178,8 +168,7 @@ hpaths = [ao.path for ao in aos] paths_titles = {} if hpaths: - import lighthisto - pp = lighthisto.PlotParser([os.path.join("..", "data", "plotinfo")]) + pp = rivet.PlotParser([os.path.join("..", "data", "plotinfo")]) for hp in hpaths: attrs = pp.getHeaders(hp) if attrs.has_key("Title"): Modified: trunk/pyext/Makefile.am ============================================================================== --- trunk/pyext/Makefile.am Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/pyext/Makefile.am Thu Mar 7 11:35:18 2013 (r4205) @@ -1,5 +1,3 @@ -EXTRA_DIST = ez_setup.py lighthisto.py spiresbib.py - if ENABLE_PYEXT SUBDIRS = rivet . @@ -13,8 +11,6 @@ ## TODO: Really want DESTDIR here? uninstall-local: rm -rf $(DESTDIR)$(RIVET_PYTHONPATH)/rivet - rm -rf $(DESTDIR)$(RIVET_PYTHONPATH)/lighthisto.py* - rm -rf $(DESTDIR)$(RIVET_PYTHONPATH)/spiresbib.py* rm -rf $(DESTDIR)$(RIVET_PYTHONPATH)/Rivet-*.egg-info clean-local: Modified: trunk/pyext/rivet/Makefile.am ============================================================================== --- trunk/pyext/rivet/Makefile.am Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/pyext/rivet/Makefile.am Thu Mar 7 11:35:18 2013 (r4205) @@ -1,4 +1,4 @@ -EXTRA_DIST = __init__.py rivetwrap.i +EXTRA_DIST = __init__.py util.py plotinfo.py spiresbib.py rivetwrap.i all-local: rivetwrap_wrap.cc rivetwrap.py @true Modified: trunk/pyext/rivet/__init__.py ============================================================================== --- trunk/pyext/rivet/__init__.py Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/pyext/rivet/__init__.py Thu Mar 7 11:35:18 2013 (r4205) @@ -1,5 +1,19 @@ "Python interface to the Rivet MC validation system" +# ## Try to bootstrap the Python path for rivet loading +# import os, sys +# PROGPATH = sys.argv[0] +# PROGNAME = os.path.basename(PROGPATH) +# import commands +# try: +# modname = sys.modules[__name__].__file__ +# binpath = os.path.dirname(modname) +# rivetconfigpath = os.path.join(binpath, "rivet-config") +# rivetpypath = commands.getoutput(rivetconfigpath + " --pythonpath") +# sys.path.append(rivetpypath) +# except: +# pass + ## Change dlopen status to GLOBAL for Rivet lib import sys try: @@ -13,12 +27,18 @@ del sys ## Import SWIG-generated wrapper core -from rivetwrap import * +from .rivetwrap import * + +## Import plot info helper +from .plotinfo import * -## Provide an extra Python-only function used to enforce the Rivet scripts' minimal Python version -def check_python_version(): - import sys - req_version = (2,4,0) - if sys.version_info[:3] < req_version: - sys.stderr.write( "rivet scripts require Python version >= %s... exiting\n" % ".".join(req_version) ) - sys.exit(1) +## Import submodules into the visible namespace +import spiresbib, util + +## Try to use Psyco optimiser if available +# TODO: or PyPy, Cython, Weave? +try: + import psyco + psyco.full() +except Exception, e: + pass Added: trunk/pyext/rivet/plotinfo.py ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ trunk/pyext/rivet/plotinfo.py Thu Mar 7 11:35:18 2013 (r4205) @@ -0,0 +1,190 @@ +import os, re + +class PlotParser(object): +#class PlotStyler(object) or class PlotInfo(object): + """Reads Rivet's .plot files and determines which attributes to apply to each histo path.""" + + pat_begin_block = re.compile('^#+ BEGIN ([A-Z0-9_]+) ?(\S+)?') + pat_end_block = re.compile('^#+ END ([A-Z0-9_]+)') + pat_comment = re.compile('^#|^\s*$') + pat_property = re.compile('^(\w+?)=(.*)$') + pat_path_property = re.compile('^(\S+?)::(\w+?)=(.*)$') + pat_paths = {} + + def __init__(self, plotpaths=None, addfiles=[]): + """ + Parameters + ---------- + plotpaths : list of str, optional + The directories to search for .plot files. + The default is to call :command:`rivet-config --datadir` to get + the directory where the .plot files can be found. + + Raises + ------ + ValueError + If `plotpaths` is not specified and calling + :command:`rivet-config` fails. + """ + self.addfiles = addfiles + + self.plotpaths = plotpaths + if not self.plotpaths: + try: + import rivet + self.plotpaths = rivet.getAnalysisPlotPaths() + except Exception, e: + sys.stderr.write("Failed to load Rivet analysis plot paths: %s\n" % e) + raise ValueError("No plot paths given and the rivet module could not be loaded!") + + + def getSection(self, section, hpath): + """Get a section for a histogram from a .plot file. + + Parameters + ---------- + section : ('PLOT'|'SPECIAL'|'HISTOGRAM') + The section that should be extracted. + hpath : str + The histogram path, i.e. /AnalysisID/HistogramID . + + Todo + ---- + Caching! + At the moment the result of the lookup is not cached so every + call requires a file to be searched for and opened. + """ + if section not in ['PLOT', 'SPECIAL', 'HISTOGRAM']: + raise ValueError("Can't parse section \'%s\'" % section) + + ## Decompose the histo path and remove the /REF prefix if necessary + parts = hpath.split("/") + if parts[1] == "REF": + del parts[1] + hpath = "/".join(parts) + if len(parts) != 3: + raise ValueError("hpath '%s' has wrong number of parts (%i) -- should be 3" % (hpath, len(parts))) + + ## Assemble the list of headers from any matching plotinfo paths and additional style files + base = parts[1] + ".plot" + ret = {'PLOT': {}, 'SPECIAL': None, 'HISTOGRAM': {}} + for pidir in self.plotpaths: + plotfile = os.path.join(pidir, base) + self.readHeadersFromFile(plotfile, ret, section, hpath) + ## Don't break here: we can collect settings from multiple .plot files + # TODO: So the *last* path wins? Hmm... reverse the loop order? + for extrafile in self.addfiles: + self.readHeadersFromFile(extrafile, ret, section, hpath) + return ret[section] + + + def readHeadersFromFile(self, plotfile, ret, section, hpath): + """Get a section for a histogram from a .plot file.""" + if os.access(plotfile, os.R_OK): + startreading = False + f = open(plotfile) + for line in f: + m = self.pat_begin_block.match(line) + if m: + tag, pathpat = m.group(1,2) + # pathpat could be a regex + if not self.pat_paths.has_key(pathpat): + self.pat_paths[pathpat] = re.compile(pathpat) + if tag == section: + if self.pat_paths[pathpat].match(hpath): + startreading = True + if section in ['SPECIAL']: + ret[section] = '' + continue + if not startreading: + continue + if self.isEndMarker(line, section): + startreading = False + continue + elif self.isComment(line): + continue + if section in ['PLOT', 'HISTOGRAM']: + vm = self.pat_property.match(line) + if vm: + prop, value = vm.group(1,2) + #print prop, value + ret[section][prop] = value + elif section in ['SPECIAL']: + ret[section] += line + f.close() + + + def getHeaders(self, hpath): + """Get the plot headers for histogram hpath. + + This returns the PLOT section. + + Parameters + ---------- + hpath : str + The histogram path, i.e. /AnalysisID/HistogramID . + + Returns + ------- + plot_section : dict + The dictionary usually contains the 'Title', 'XLabel' and + 'YLabel' properties of the respective plot. + + See also + -------- + :meth:`getSection` + """ + return self.getSection('PLOT', hpath) + + + def getSpecial(self, hpath): + """Get a SPECIAL section for histogram hpath. + + The SPECIAL section is only available in a few analyses. + + Parameters + ---------- + hpath : str + Histogram path. Must have the form /AnalysisID/HistogramID . + + See also + -------- + :meth:`getSection` + """ + return self.getSection('SPECIAL', hpath) + + + def getHistogramOptions(self, hpath): + """Get a HISTOGRAM section for histogram hpath. + + The HISTOGRAM section is only available in a few analyses. + + Parameters + ---------- + hpath : str + Histogram path. Must have the form /AnalysisID/HistogramID . + + See also + -------- + :meth:`getSection` + """ + return self.getSection('HISTOGRAM', hpath) + + + def isEndMarker(self, line, blockname): + m = self.pat_end_block.match(line) + return m and m.group(1) == blockname + + + def isComment(self, line): + return self.pat_comment.match(line) is not None + + + def updateHistoHeaders(self, hist): + headers = self.getHeaders(hist.histopath) + if headers.has_key("Title"): + hist.title = headers["Title"] + if headers.has_key("XLabel"): + hist.xlabel = headers["XLabel"] + if headers.has_key("YLabel"): + hist.ylabel = headers["YLabel"] Copied: trunk/pyext/rivet/spiresbib.py (from r4203, trunk/pyext/spiresbib.py) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ trunk/pyext/rivet/spiresbib.py Thu Mar 7 11:35:18 2013 (r4205, copy of r4203, trunk/pyext/spiresbib.py) @@ -0,0 +1,87 @@ +#! /usr/bin/env python + +import logging +import urllib2 +import re + +usage = """%prog <spiresid> [<spiresid2> ...] + +Given Inspire and SPIRES paper IDs, fetch the corresponding BibTeX db entry from +the SPIRES Web interface and write it to stdout. Prefix the code with I or S +appropriately. +""" + +def fetch_bibtex(iscode, refid): + if iscode.upper() == "I": + url = "http://inspire-hep.net/record/%s/export/hx" % str(refid) + logging.debug("Downloading Inspire BibTeX from %s" % url) + elif iscode.upper() == "S": + url = "http://inspire-hep.net/search?p=find+key+%s&of=hx" % str(refid) + logging.debug("Downloading SPIRES BibTeX from %s" % url) + hreq = urllib2.urlopen(url) + bibtexhtml = hreq.read() + hreq.close() + #logging.debug(bibtexhtml) + return bibtexhtml + + +def extract_bibtex(html): + ## Extract BibTeX block from HTML + re_spiresbibtex = re.compile(r'<pre>(.*?)</pre>', re.MULTILINE | re.DOTALL) + m = re_spiresbibtex.search(html) + if m is None: + return None, None + bib = m.group(1).strip() + + ## Get BibTeX key + re_bibtexkey = re.compile(r'^@.+?{(.+?),$', re.MULTILINE) + m = re_bibtexkey.search(bib) + if m is None: + return None, bib + key = m.group(1) + + ## Return key and BibTeX + return key, bib + + +def get_bibtex_from_repo(iscode, refid): + html = fetch_bibtex(iscode, refid) + key, bibtex = extract_bibtex(html) + return key, bibtex + + +def get_bibtexs_from_repos(iscodes_refids): + bibdb = {} + for iscode, refid in iscodes_refids: + key, bibtex = get_bibtex_from_repo(iscode, refid) + if key and bibtex: + bibdb[refid] = (key, bibtex) + return bibdb + + +if __name__ == '__main__': + ## Parse command line options + from optparse import OptionParser + parser = OptionParser(usage=usage) + opts, args = parser.parse_args() + + ## Make individual bibinfo files + for arg in args: + iscode = arg[0] + refid = arg[1:] + key, bibtex = get_bibtex_from_repo(iscode, refid) + import sys + f = sys.stdout + f.write("BibKey: %s\n" % key) + f.write("BibTeX: '%s'\n" % bibtex) + + # ## Build ref db + # bibdb = get_bibtexs_from_spires(args) + # for sid, (key, bibtex) in bibdb.iteritems(): + # print key, "=>\n", bibtex + + # ## Pickle ref db9151176 + # import cPickle as pickle + # fpkl = open("spiresbib.pkl", "w")repo + # pickle.dump(bibdb) + # fpkl.close() Added: trunk/pyext/rivet/util.py ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ trunk/pyext/rivet/util.py Thu Mar 7 11:35:18 2013 (r4205) @@ -0,0 +1,31 @@ +"Python utility functions for use by Rivet scripts (and anyone else who wants to)" + +def check_python_version(req_version=(2,4,0)): + "Enforce the Rivet scripts' minimal Python version requirement" + import sys + if sys.version_info[:3] < req_version: + sys.stderr.write( "Python version >= %s is required... exiting\n" % ".".join(req_version) ) + sys.exit(1) + +def set_process_name(name): + "Try to rename the process on Linux so it doesn't appear as 'python <scriptpath>'" + try: + import ctypes + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc.prctl(15, name, 0, 0, 0) + except Exception: + pass + +def import_ET(): + "Try to import the ElementTree XML parser, which has many historical import signatures" + try: + import xml.etree.cElementTree as ET + except ImportError: + try: + import cElementTree as ET + except ImportError: + try: + import xml.etree.ElementTree as ET + except: + raise ImportError("Can't load the ElementTree XML parser (any of three historical ways)") + Modified: trunk/pyext/setup.py.in ============================================================================== --- trunk/pyext/setup.py.in Thu Mar 7 09:57:16 2013 (r4204) +++ trunk/pyext/setup.py.in Thu Mar 7 11:35:18 2013 (r4205) @@ -33,7 +33,7 @@ version = '@PACKAGE_VERSION@', ext_package = 'rivet', ext_modules = [ext], - py_modules = ['lighthisto', 'spiresbib', 'rivet.__init__', 'rivet.rivetwrap'], + py_modules = ['rivet.rivetwrap', 'rivet.__init__', 'rivet.plotinfo', 'rivet.util', 'rivet.spiresbib'], author = ['Andy Buckley'], author_email = 'andy.buckley at cern.ch', url = 'http://projects.hepforge.org/rivet/',
More information about the Rivet-svn mailing list |