Merge pull request #155 from lgray/spark_py27

Enable spark executor in python 2.7

.travis.yml

@@ -18,7 +18,6 @@ addons:
     - python-setuptools
     - pandoc
 script:
-- pip list
 - python setup.py flake8
 - coverage run --source=coffea/ setup.py pytest
 before_install:
@@ -31,12 +30,12 @@ install:
 - pip -q install codecov
 - PY_MAJ=`python -c 'import sys; print(sys.version_info[0])'`
 - PY_MIN=`python -c 'import sys; print(sys.version_info[1])'`
-- if [ $PY_MAJ -eq 3 ]; then pip -q install pandas pyarrow cloudpickle lz4 jinja2; fi
-- if [ $PY_MAJ -eq 3 ]; then pip -q install https://github.com/Parsl/parsl/zipball/master; fi
-- if [ $PY_MAJ -eq 3 ] && [ $PY_MIN -ge 6 ]; then  pip -q install pyspark; fi
 - pip -q install numpy scipy numba tqdm matplotlib pytest pytest-runner flake8 --upgrade
+- pip -q install pandas pyarrow cloudpickle lz4 jinja2 pyspark --upgrade
+- if [ $PY_MAJ -eq 3 ]; then pip -q install https://github.com/Parsl/parsl/zipball/master; fi
 - pip -q install ${AWKWARD}
 - pip -q install uproot 
+- pip list
 branches:
   only:
   - master

coffea/processor/spark/detail.py

@@ -4,7 +4,11 @@
 import pyspark.sql
 import pyspark.sql.functions as fn
 from pyarrow.compat import guid
-from collections.abc import Sequence
+
+try:
+    from collections.abc import Sequence
+except ImportError:
+    from collections import Sequence
 
 from ..executor import futures_handler
 

coffea/processor/spark/spark_executor.py

@@ -3,7 +3,7 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from tqdm import tqdm
-import _pickle as pkl
+import pickle as pkl
 import lz4.frame as lz4f
 import numpy as np
 import pandas as pd

coffea/version.py

@@ -30,7 +30,7 @@
 
 import re
 
-__version__ = "0.6.7"
+__version__ = "0.6.8"
 version = __version__
 version_info = tuple(re.split(r"[-\.]", __version__))
 

tests/test_spark.py

@@ -66,7 +66,7 @@ def test_spark_hist_adders():
     pyspark = pytest.importorskip("pyspark", minversion="2.4.1")
 
     import pandas as pd
-    import _pickle as pkl
+    import pickle as pkl
     import lz4.frame as lz4f
 
     from coffea.util import numpy as np