Merge pull request #23 from eth-cscs/release-0.3.0

Release 0.3.0

.travis.yml

@@ -10,8 +10,12 @@ addons:
     - libpython3.4-dev
     - python3-numpy
     - swig
+    - libmpich-dev
+    - mpich
 install:
 - pip install -r requirements.txt
+- pip install -r requirements/backend-mpi.txt
+- pip install -r requirements/backend-spark.txt
 script:
 - make test
 deploy:

Makefile

@@ -4,34 +4,45 @@ MAKEDIRS=$(shell find examples -name Makefile -exec dirname {} \;)
 whl_file = abcpy-${VERSION}-py3-none-any.whl
 
 .DEFAULT: help
-.PHONY: help clean doc doctest exampletest package test uninstall unittest install reinstall $(MAKEDIRS)
+.PHONY: help clean doc doctest exampletest package test uninstall unittest unittest_mpi install reinstall $(MAKEDIRS)
 
 help:
-	@echo Targets are: clean, doc, doctest, exampletest, package, uninstall, unittest, test
+	@echo Targets are: clean, doc, doctest, exampletest, package, uninstall, unittest, unittest_mpi	, test
 
 clean:
 	find . -name "*.pyc" -type f -delete
 	find . -name "__pycache__" -delete
 	find . -name ".#*" -delete
 	find . -name "#*#" -delete
 
-test: unittest exampletest doctest
+$(MAKEDIRS):
+	make -C $@
 
-unittest:
-	python3 -m unittest discover -s tests -v -p "*_tests.py" || (echo "Error in unit tests."; exit 1)
+# testing
 
+test: unittest unittest_mpi exampletest exampletest_mpi doctest
 
-$(MAKEDIRS):
-	make -C $@
+unittest:
+	echo "Running standard unit tests.."
+	python3 -m unittest discover -s tests -v -p "*_tests.py" || (echo "Error in standard unit tests."; exit 1)
 
-doctest:
-	make -C doc html || (echo "Error in documentation generator."; exit 1)
+unittest_mpi:
+	echo "Running MPI backend unit tests.."
+	mpirun -np 2 python3 -m unittest discover -s tests -v -p "backend_tests_mpi.py" || (echo "Error in MPI unit tests."; exit 1)
 
 exampletest: $(MAKEDIRS)
+	echo "Testing standard examples.."
 	python3 -m unittest discover -s examples -v -p "*.py" || (echo "Error in example tests."; exit 1)
 
+exampletest_mpi:
+	echo "Testing MPI backend examples.."
+	mpirun -np 2 python3 -m unittest -v examples/backends/mpi/pmcabc_gaussian.py || (echo "Error in MPI example tests."; exit 1)
+
+doctest:
+	make -C doc html || (echo "Error in documentation generator."; exit 1)
+
 coveragetest:
-	command -v coverage >/dev/null 2>&1 || { echo >&2 "Python package 'coverage' has to be installed. Please, run 'pip3 install coverage'."; exit;}
+	command -v coverage >/dev/null 2>&1 || { echo >&2 "Python package 'coverage' has to been installed. Please, run 'pip3 install coverage'."; exit;}
 	@- $(foreach TEST, $(UNITTESTS), \
 		echo === Testing code coverage: $(TEST); \
 		python3 -m unittest $(TEST); \

VERSION

@@ -1 +1 @@
-0.2.2
+0.3

abcpy/backends/__init__.py

@@ -0,0 +1,14 @@
+from abcpy.backends.base import *
+
+
+def BackendMPI(*args,**kwargs):
+    from abcpy.backends.mpi import BackendMPI
+    return BackendMPI(*args,**kwargs)
+
+def BackendMPITestHelper(*args,**kwargs):
+    from abcpy.backends.mpi import BackendMPITestHelper 
+    return BackendMPITestHelper(*args,**kwargs)
+
+def BackendSpark(*args,**kwargs):
+    from  abcpy.backends.spark import BackendSpark
+    return BackendSpark(*args,**kwargs)

abcpy/backends.py → abcpy/backends/base.py

@@ -226,148 +226,3 @@ def __init__(self, object):
     def value(self):
         return self.object
 
-
-
-
-class BackendSpark(Backend):
-    """
-    A parallelization backend for Apache Spark. It is essetially a wrapper for
-    the required Spark functionality.
-    """
-
-    def __init__(self, sparkContext, parallelism=4):
-        """
-        Initialize the backend with an existing and configured SparkContext.
-
-        Parameters
-        ----------
-        sparkContext: pyspark.SparkContext
-            an existing and fully configured PySpark context
-        parallelism: int
-            defines on how many workers a distributed dataset can be distributed
-        """
-        self.sc = sparkContext
-        self.parallelism = parallelism
-
-
-    def parallelize(self, python_list):
-        """
-        This is a wrapper of pyspark.SparkContext.parallelize().
-
-        Parameters
-        ----------
-        list: Python list
-            list that is distributed on the workers
-        
-        Returns
-        -------
-        PDSSpark class (parallel data set)
-            A reference object that represents the parallelized list
-        """
-
-        rdd = self.sc.parallelize(python_list, self.parallelism)
-        pds = PDSSpark(rdd)
-        return pds
-
-
-    def broadcast(self, object):
-        """
-        This is a wrapper for pyspark.SparkContext.broadcast().
-
-        Parameters
-        ----------
-        object: Python object
-            An abitrary object that should be available on all workers
-        Returns
-        -------
-        BDSSpark class (broadcast data set)
-            A reference to the broadcasted object
-        """
-
-        bcv = self.sc.broadcast(object)
-        bds = BDSSpark(bcv)
-        return bds
-
-
-    def map(self, func, pds):
-        """
-        This is a wrapper for pyspark.rdd.map()
-
-        Parameters
-        ----------
-        func: Python func
-            A function that can be applied to every element of the pds
-        pds: PDSSpark class
-            A parallel data set to which func should be applied
-        Returns
-        -------
-        PDSSpark class
-            a new parallel data set that contains the result of the map
-        """
-
-        rdd = pds.rdd.map(func)
-        new_pds = PDSSpark(rdd)
-        return new_pds
-
-
-    def collect(self, pds):
-        """
-        A wrapper for pyspark.rdd.collect()
-
-        Parameters
-        ----------
-        pds: PDSSpark class
-            a parallel data set
-        Returns
-        -------
-        Python list
-            all elements of pds as a list
-        """
-
-        python_list = pds.rdd.collect()
-        return python_list
-
-
-
-class PDSSpark(PDS):
-    """
-    This is a wrapper for Apache Spark RDDs.
-    """
-
-    def __init__(self, rdd):
-        """
-        Returns
-        -------
-        rdd: pyspark.rdd
-            initialize with an Spark RDD
-        """
-
-        self.rdd = rdd
-
-
-
-class BDSSpark(BDS):
-    """
-    This is a wrapper for Apache Spark Broadcast variables.
-    """
-
-    def __init__(self, bcv):
-        """
-        Parameters
-        ----------
-        bcv: pyspark.broadcast.Broadcast
-            Initialize with a Spark broadcast variable
-        """
-
-        self.bcv = bcv
-
-
-    def value(self):
-        """
-        Returns
-        -------
-        object
-            returns the referenced object that was broadcasted.
-        """
-
-        return self.bcv.value