Merge pull request #144 from lgray/topic_fix143

Fallback to JME treatment of JEC files in case there are overlapping bins

coffea/lookup_tools/txt_converters.py

@@ -1,6 +1,8 @@
 from ..util import awkward
 from ..util import numpy as np
 import os
+import sys
+import warnings
 try:
     import cStringIO as io
 except ImportError:
@@ -13,10 +15,12 @@
 def _parse_jme_formatted_file(jmeFilePath, interpolatedFunc=False, parmsFromColumns=False, jme_f=None):
     if jme_f is None:
         fopen = open
+        fmode = 'rt'
         if '.gz' in jmeFilePath:
             import gzip
             fopen = gzip.open
-        jme_f = fopen(jmeFilePath, 'rt')
+            fmode = 'r' if sys.platform.startswith('win') else fmode
+        jme_f = fopen(jmeFilePath, fmode)
     layoutstr = jme_f.readline().strip().strip('{}')
 
     name = jmeFilePath.split('/')[-1].split('.')[0]
@@ -107,14 +111,23 @@ def _build_standard_jme_lookup(name, layout, pars, nBinnedVars, nBinColumns,
         if i == 0:
             binMins = np.unique(pars[columns[0]])
             binMaxs = np.unique(pars[columns[1]])
-            bins[layout[i + offset_name]] = np.union1d(binMins, binMaxs)
+            if np.all(binMins[1:] == binMaxs[:-1]):
+                bins[layout[i + offset_name]] = np.union1d(binMins, binMaxs)
+            else:
+                warnings.warn('binning for file for %s is malformed in variable %s' % (name, layout[i + offset_name]))
+                bins[layout[i + offset_name]] = np.union1d(binMins, binMaxs[-1:])
         else:
             counts = np.zeros(0, dtype=np.int)
             allBins = np.zeros(0, dtype=np.double)
             for binMin in bins[bin_order[0]][:-1]:
                 binMins = np.unique(pars[np.where(pars[columns[0]] == binMin)][columns[i + offset_col]])
                 binMaxs = np.unique(pars[np.where(pars[columns[0]] == binMin)][columns[i + offset_col + 1]])
-                theBins = np.union1d(binMins, binMaxs)
+                theBins = None
+                if np.all(binMins[1:] == binMaxs[:-1]):
+                    theBins = np.union1d(binMins, binMaxs)
+                else:
+                    warnings.warn('binning for file for %s is malformed in variable %s' % (name, layout[i + offset_name]))
+                    theBins = np.union1d(binMins, binMaxs[-1:])
                 allBins = np.append(allBins, theBins)
                 counts = np.append(counts, theBins.size)
             bins[layout[i + offset_name]] = awkward.JaggedArray.fromcounts(counts, allBins)
@@ -139,14 +152,18 @@ def _build_standard_jme_lookup(name, layout, pars, nBinnedVars, nBinColumns,
         if not interpolatedFunc:
             clamp_mins[layout[i + offset_name]] = awkward.JaggedArray.fromcounts(jagged_counts, np.atleast_1d(pars[columns[i + offset_col]]))
             clamp_maxs[layout[i + offset_name]] = awkward.JaggedArray.fromcounts(jagged_counts, np.atleast_1d(pars[columns[i + offset_col + 1]]))
+            assert(clamp_mins[layout[i + offset_name]].valid())
+            assert(clamp_maxs[layout[i + offset_name]].valid())
             offset_col += 1
 
     # now get the parameters, which we will look up with the clamped values
     parms = []
     parm_order = []
     offset_col = 2 * nBinnedVars + 1 + int(not interpolatedFunc) * 2 * nEvalVars
     for i in range(nParms):
-        parms.append(awkward.JaggedArray.fromcounts(jagged_counts, pars[columns[i + offset_col]]))
+        jag = awkward.JaggedArray.fromcounts(jagged_counts, pars[columns[i + offset_col]])
+        assert(jag.valid())
+        parms.append(jag)
         parm_order.append('p%i' % (i))
 
     wrapped_up = {}
@@ -190,10 +207,12 @@ def convert_junc_txt_file(juncFilePath):
     components = []
     basename = os.path.basename(juncFilePath).split('.')[0]
     fopen = open
+    fmode = 'rt'
     if '.gz' in juncFilePath:
         import gzip
         fopen = gzip.open
-    with fopen(juncFilePath, 'rt') as uncfile:
+        fmode = 'r' if sys.platform.startswith('win') else fmode
+    with fopen(juncFilePath, fmode) as uncfile:
         for line in uncfile:
             if line.startswith('#'):
                 continue
@@ -253,10 +272,12 @@ def convert_junc_txt_component(juncFilePath, uncFile):
 
 def convert_effective_area_file(eaFilePath):
     fopen = open
+    fmode = 'rt'
     if '.gz' in eaFilePath:
         import gzip
         fopen = gzip.open
-    ea_f = fopen(eaFilePath, 'rt')
+        fmode = 'r' if sys.platform.startswith('win') else fmode
+    ea_f = fopen(eaFilePath, fmode)
     layoutstr = ea_f.readline().strip().strip('{}')
     ea_f.close()
 

coffea/version.py

@@ -30,7 +30,7 @@
 
 import re
 
-__version__ = "0.6.4"
+__version__ = "0.6.5"
 version = __version__
 version_info = tuple(re.split(r"[-\.]", __version__))
 

tests/test_jetmet_tools.py

@@ -10,9 +10,6 @@
 
 from dummy_distributions import dummy_jagged_eta_pt, dummy_four_momenta
 
-if sys.platform.startswith("win"):
-    pytest.skip("skipping tests that only function in linux", allow_module_level=True)
-
 
 def jetmet_evaluator():
     from coffea.lookup_tools import extractor

tests/test_lookup_tools.py

@@ -137,6 +137,7 @@ def test_jec_txt_scalefactors():
     extractor = lookup_tools.extractor()
     extractor.add_weight_sets([
         "testJEC * tests/samples/Fall17_17Nov2017_V32_MC_L2Relative_AK4PFPuppi.jec.txt",
+        "* * tests/samples/Summer16_07Aug2017_V11_L1fix_MC_L2Relative_AK4PFchs.jec.txt.gz",
         "* * tests/samples/Fall17_17Nov2017_V32_MC_Uncertainty_AK4PFPuppi.junc.txt",
         "* * tests/samples/Autumn18_V8_MC_UncertaintySources_AK4PFchs.junc.txt",
         "* * tests/samples/Spring16_25nsV10_MC_SF_AK4PFPuppi.jersf.txt"
@@ -150,6 +151,10 @@ def test_jec_txt_scalefactors():
     jec_out = evaluator['testJECFall17_17Nov2017_V32_MC_L2Relative_AK4PFPuppi'](test_eta,test_pt)
 
     print(evaluator['testJECFall17_17Nov2017_V32_MC_L2Relative_AK4PFPuppi'])
+
+    jec_out = evaluator['Summer16_07Aug2017_V11_L1fix_MC_L2Relative_AK4PFchs'](test_eta,test_pt)
+
+    print(evaluator['Summer16_07Aug2017_V11_L1fix_MC_L2Relative_AK4PFchs'])
 
     jersf = evaluator['Spring16_25nsV10_MC_SF_AK4PFPuppi']