ported HEPPY_PR2_7_2_X from CMS

Author: cbernet

BuildFile.xml

@@ -17,7 +17,7 @@ def process(self, event):
         self.tree.fill('test_variable', event.input.var1)
         self.tree.tree.Fill()
 
-    def write(self):
+    def write(self, setup):
         self.rootfile.Write()
         self.rootfile.Close()
 

analyzers/SimpleTreeProducer.py

@@ -1 +0,0 @@
-#Automatically created by SCRAM

framework/__init__.py

@@ -32,8 +32,12 @@ def __init__(self, cfg_ana, cfg_comp, looperName ):
         self.cfg_ana = cfg_ana
         self.cfg_comp = cfg_comp
         self.looperName = looperName
-        self.dirName = '/'.join( [self.looperName, self.name] )
-        os.mkdir( self.dirName )
+	if hasattr(cfg_ana,"nosubdir") and cfg_ana.nosubdir:
+       	    self.dirName = self.looperName
+	else:
+            self.dirName = '/'.join( [self.looperName, self.name] )
+            os.mkdir( self.dirName )
+
 
         # this is the main logger corresponding to the looper.
         # each analyzer could also declare its own logger
@@ -62,7 +66,7 @@ def process(self, event ):
         print self.cfg_ana.name
 
 
-    def write(self):
+    def write(self, setup):
         """Called by Looper.write, for all analyzers.
         Just overload it if you have histograms to write."""
         self.counters.write( self.dirName )

framework/analyzer.py

@@ -3,39 +3,51 @@
 import shutil
 
 from heppy.framework.chain import Chain
+from heppy.utils.testtree import create_tree
+
+testfname = 'test_tree.root'
 
 class ChainTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.file = '../../test/test_tree.root'
-        self.chain = Chain(self.file, 'test_tree')
+        self.chain = Chain(testfname, 'test_tree')
 
     def test_file(self):
-        self.assertTrue(os.path.isfile(self.file))
+        '''Test that the test file exists'''
+        self.assertTrue(os.path.isfile(testfname))
 
-    def test_guess_name(self):
+    def test_wrong_filename(self):
         self.assertRaises(ValueError,
-                          Chain, 'self.file')
+                          Chain, 'non_existing_file.root')
+
+    def test_guess_treename(self):
+        chain = Chain(testfname)
+        self.assertEqual(len(self.chain), 100)        
 
     def test_load_1(self):
+        '''Test that the chain has the correct number of entries'''
         self.assertEqual(len(self.chain), 100)
 
     def test_load_2(self):
-        tmpfile = self.file.replace('test_tree', 'test_tree_2_tmp')
-        shutil.copyfile(self.file, tmpfile)
-        chain = Chain(self.file.replace('.root', '*.root'), 'test_tree')
+        '''Test chaining of two files.'''
+        tmpfile = testfname.replace('test_tree', 'test_tree_2_tmp')
+        shutil.copyfile(testfname, tmpfile)
+        chain = Chain(testfname.replace('.root', '*.root'), 'test_tree')
         self.assertEqual(len(chain), 200)
         os.remove(tmpfile)
 
     def test_iterate(self):
+        '''Test iteration'''
         for ev in self.chain:
             pass
         self.assertTrue(True)
 
     def test_get(self):
+        '''Test direct event access'''
         event = self.chain[2]
         self.assertEqual(event.var1, 2.)
 
 
 if __name__ == '__main__':
+    create_tree(testfname)
     unittest.main()

framework/chain_test.py

@@ -135,10 +135,11 @@ def __init__(self, name, files, tree_name=None, triggers=None, **kwargs):
         self.dataset_entries = 0
         self.isData = False
         self.isMC = False
+        self.isEmbed = False
 
 class DataComponent( Component ):
 
-    def __init__(self, name, files, intLumi, triggers, json=None):
+    def __init__(self, name, files, intLumi=None, triggers=[], json=None):
         super(DataComponent, self).__init__(name, files, triggers)
         self.isData = True
         self.intLumi = intLumi
@@ -154,10 +155,9 @@ def getWeight( self, intLumi = None):
 
 
 class MCComponent( Component ):
-    def __init__(self, name, files, triggers, xSection,
-                 nGenEvents,
-                 # vertexWeight,tauEffWeight, muEffWeight,
-                 effCorrFactor, **kwargs ):
+    def __init__(self, name, files, triggers=[], xSection=1,
+                 nGenEvents=None,
+                 effCorrFactor=None, **kwargs ):
         super( MCComponent, self).__init__( name = name,
                                             files = files,
                                             triggers = triggers, **kwargs )
@@ -195,35 +195,3 @@ def __str__(self):
         return '\n'.join([comp, sequence, services])
 
 
-if __name__ == '__main__':
-
-    from heppy.framework.chain import Chain as Events
-    from heppy.analyzers.Printer import Printer
-
-    class Ana1(object):
-        pass
-    ana1 = Analyzer(
-        Ana1,
-        toto = '1',
-        tata = 'a'
-        )
-    ana2 = Analyzer(
-        Printer,
-        'instance1'
-        )
-    sequence = Sequence( [ana1, ana2] )
-
-    DYJets = MCComponent(
-        name = 'DYJets',
-        files ='blah_mc.root',
-        xSection = 3048.,
-        nGenEvents = 34915945,
-        triggers = ['HLT_MC'],
-        vertexWeight = 1.,
-        effCorrFactor = 1 )
-    selectedComponents = [DYJets]
-    sequence = [ana1, ana2]
-    config = Config( components = selectedComponents,
-                     sequence = sequence, 
-                     events_class = Events )
-    print config

framework/config.py

@@ -45,6 +45,7 @@ class Ana1(object):
         from heppy.framework.chain import Chain as Events
         config = Config( components = [comp1],
                          sequence = [ana1], 
+                         services = [],
                          events_class = Events )
 
 

framework/config_test.py

@@ -4,12 +4,11 @@
 import os
 import sys
 import imp
-import copy
 import logging
 import pprint
-from platform import platform 
+from math import ceil
 from event import Event
-
+import timeit
 
 class Setup(object):
     '''The Looper creates a Setup object to hold information relevant during 
@@ -32,7 +31,7 @@ def __init__(self, config, services):
         services: dictionary of services indexed by service name.
         The service name has the form classObject_instanceLabel 
         as in this example: 
-        heppy.framework.services.tfile.TFileService_myhists
+        <base_heppy_path>.framework.services.tfile.TFileService_myhists
         To find out about the service name of a given service, 
         load your configuration file in python, and print the service. 
         '''
@@ -51,7 +50,7 @@ class Looper(object):
     def __init__( self, name,
                   config, 
                   nEvents=None,
-                  firstEvent=0, nPrint=0 ):
+                  firstEvent=0, nPrint=0, timeReport=False ):
         """Handles the processing of an event sample.
         An Analyzer is built for each Config.Analyzer present
         in sequence. The Looper can then be used to process an event,
@@ -79,20 +78,33 @@ def __init__( self, name,
         self.nEvents = nEvents
         self.firstEvent = firstEvent
         self.nPrint = int(nPrint)
+        self.timeReport = [ {'time':0.0,'events':0} for a in self.analyzers ] if timeReport else False
         tree_name = None
         if( hasattr(self.cfg_comp, 'tree_name') ):
             tree_name = self.cfg_comp.tree_name
         if len(self.cfg_comp.files)==0:
             errmsg = 'please provide at least an input file in the files attribute of this component\n' + str(self.cfg_comp)
             raise ValueError( errmsg )
         self.events = config.events_class(self.cfg_comp.files, tree_name)
+        if hasattr(self.cfg_comp, 'fineSplit'):
+            fineSplitIndex, fineSplitFactor = self.cfg_comp.fineSplit
+            if fineSplitFactor > 1:
+                if len(self.cfg_comp.files) != 1:
+                    raise RuntimeError, "Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (self.cfg_comp.name, self.cfg_comp.files)
+                totevents = min(len(self.events),int(nEvents)) if (nEvents and int(nEvents) not in [-1,0]) else len(self.events)
+                self.nEvents = int(ceil(totevents/float(fineSplitFactor)))
+                self.firstEvent = firstEvent + fineSplitIndex * self.nEvents
+                #print "For component %s will process %d events starting from the %d one" % (self.cfg_comp.name, self.nEvents, self.firstEvent)
         # self.event is set in self.process
         self.event = None
         services = dict()
         for cfg_serv in config.services:
             service = self._build(cfg_serv)
             services[cfg_serv.name] = service
-        self.setup = Setup( copy.deepcopy(config), services)
+        # would like to provide a copy of the config to the setup,
+        # so that analyzers cannot modify the config of other analyzers. 
+        # but cannot copy the autofill config.
+        self.setup = Setup(config, services)
 
     def _build(self, cfg):
         theClass = cfg.class_object
@@ -102,14 +114,16 @@ def _build(self, cfg):
     def _prepareOutput(self, name):
         index = 0
         tmpname = name
-        while True:
+        while True and index < 2000:
             try:
                 # print 'mkdir', self.name
                 os.mkdir( tmpname )
                 break
             except OSError:
                 index += 1
                 tmpname = '%s_%d' % (name, index)
+        if index == 2000:
+              raise ValueError( "More than 2000 output folder with same name or 2000 attempts failed, please clean-up, change name or check permissions")
         return tmpname
 
 
@@ -141,8 +155,18 @@ def loop(self):
                 # if iEv == nEvents:
                 #     break
                 if iEv%100 ==0:
-                    print 'event', iEv
+                    # print 'event', iEv
+                    if not hasattr(self,'start_time'):
+                        print 'event', iEv
+                        self.start_time = timeit.default_timer()
+                        self.start_time_event = iEv
+                    else:
+                        print 'event %d (%.1f ev/s)' % (iEv, (iEv-self.start_time_event)/float(timeit.default_timer() - self.start_time))
+
                 self.process( iEv )
+                if iEv<self.nPrint:
+                    print self.event
+
         except UserWarning:
             print 'Stopped loop following a UserWarning exception'
         for analyzer in self.analyzers:
@@ -162,10 +186,16 @@ def process(self, iEv ):
         """
         self.event = Event(iEv, self.events[iEv], self.setup)
         self.iEvent = iEv
-        for analyzer in self.analyzers:
+        for i,analyzer in enumerate(self.analyzers):
             if not analyzer.beginLoopCalled:
                 analyzer.beginLoop()
-            if analyzer.process( self.event ) == False:
+            start = timeit.default_timer()
+            ret = analyzer.process( self.event )
+            if self.timeReport:
+                self.timeReport[i]['events'] += 1
+                if self.timeReport[i]['events'] > 0:
+                    self.timeReport[i]['time'] += timeit.default_timer() - start
+            if ret == False:
                 return (False, analyzer.name)
         if iEv<self.nPrint:
             self.logger.info( self.event.__str__() )
@@ -177,8 +207,17 @@ def write(self):
         See Analyzer.Write for more information.
         """
         for analyzer in self.analyzers:
-            analyzer.write()
+            analyzer.write(self.setup)
         self.setup.close() 
+
+        if self.timeReport:
+            allev = max([x['events'] for x in self.timeReport])
+            print "\n      ---- TimeReport (all times in ms; first evt is skipped) ---- "
+            print "%9s   %9s    %9s   %9s   %s" % ("processed","all evts","time/proc", " time/all", "analyer")
+            print "%9s   %9s    %9s   %9s   %s" % ("---------","--------","---------", "---------", "-------------")
+            for ana,rep in zip(self.analyzers,self.timeReport):
+                print "%9d   %9d   %10.2f  %10.2f   %s" % ( rep['events'], allev, 1000*rep['time']/(rep['events']-1) if rep['events']>1 else 0, 1000*rep['time']/(allev-1) if allev > 1 else 0, ana.name)
+            print ""
         pass
 
 
@@ -187,16 +226,22 @@ def write(self):
     import pickle
     import sys
     import os
-
-    cfgFileName = sys.argv[1]
-    pckfile = open( cfgFileName, 'r' )
-    config = pickle.load( pckfile )
-    comp = config.components[0]
-    events_class = config.events_class
-    looper = Looper( 'Loop', comp,
-                     config.sequence,
-                     config.services,
-                     events_class, 
-                     nPrint = 5)
+    if len(sys.argv) == 2 :
+        cfgFileName = sys.argv[1]
+        pckfile = open( cfgFileName, 'r' )
+        config = pickle.load( pckfile )
+        comp = config.components[0]
+        events_class = config.events_class
+    elif len(sys.argv) == 3 :
+        cfgFileName = sys.argv[1]
+        file = open( cfgFileName, 'r' )
+        cfg = imp.load_source( 'cfg', cfgFileName, file)
+        compFileName = sys.argv[2]
+        pckfile = open( compFileName, 'r' )
+        comp = pickle.load( pckfile )
+        cfg.config.components=[comp]
+        events_class = cfg.config.events_class
+
+    looper = Looper( 'Loop', cfg.config,nPrint = 5)
     looper.loop()
     looper.write()

framework/eventsalbers.py

@@ -6,6 +6,7 @@ export PYTHONPATH=$PWD/..:$PYTHONPATH
 # set up executable directory
 cd $HEPPY/bin
 ln -sf ../scripts/*.py .
+ln -sf ../scripts/heppy .
 chmod +x * 
 cd .. > /dev/null