Open PR for Streaming Partitioning

.gitignore

@@ -52,3 +52,4 @@ pylint_report.txt
 pep8_report.txt
 pep257_report.txt
 .cache/
+khmer/_oxli/*.cpp

MANIFEST.in

@@ -2,6 +2,7 @@ include ChangeLog ez_setup.py IDEAS Makefile README.rst setup.cfg
 include versioneer.py MANIFEST.in CITATION CONTRIBUTING.md Doxyfile.in
 include LICENSE TODO .ycm_extra_conf.py
 recursive-include lib *.hh *.cc [Mm]akefile* get_version.py
+recursive-include khmer *.hh *.cc *.cpp
 recursive-include third-party *.cc *.1 *.xsl README* sample* words* *.sh *.c
 recursive-include third-party manual* [Mm]akefile* *.pl *.dsp CHANGES *.txt *.h
 recursive-include third-party ChangeLog FAQ INDEX configure *.xsl

Makefile

@@ -38,12 +38,13 @@
 # `SHELL=bash` Will break Titus's laptop, so don't use BASH-isms like
 # `[[` conditional expressions.
 CPPSOURCES=$(wildcard lib/*.cc lib/*.hh khmer/_khmer.cc) setup.py
+CYSOURCES=$(wildcard khmer/_oxli/*.pyx khmer/_oxli/*.pxd)
 PYSOURCES=$(filter-out khmer/_version.py, \
 	  $(wildcard khmer/*.py scripts/*.py oxli/*.py) )
-SOURCES=$(PYSOURCES) $(CPPSOURCES) setup.py
+SOURCES=$(PYSOURCES) $(CPPSOURCES) $(CYSOURCES) setup.py
 
 DEVPKGS=pep8==1.6.2 diff_cover autopep8 pylint coverage gcovr pytest \
-	pydocstyle screed pyenchant
+	pydocstyle screed pyenchant cython
 GCOVRURL=git+https://github.com/nschum/gcovr.git@never-executed-branches
 
 VERSION=$(shell ./setup.py version | grep Version | awk '{print $$4}' \
@@ -83,6 +84,7 @@ endif
 MODEXT=$(shell python -c \
        "import sysconfig;print(sysconfig.get_config_var('SO'))")
 EXTENSION_MODULE = khmer/_khmer$(MODEXT)
+CYTHON_MODULE = khmer/_oxli/_oxli$(MODEXT)
 
 PYLINT_TEMPLATE="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}"
 
@@ -102,11 +104,14 @@ install-dependencies:
 	pip install --upgrade --requirement doc/requirements.txt
 
 ## sharedobj   : build khmer shared object file
-sharedobj: $(EXTENSION_MODULE)
+sharedobj: $(EXTENSION_MODULE) $(CYTHON_MODULE)
 
 $(EXTENSION_MODULE): $(CPPSOURCES)
 	./setup.py build_ext --inplace
 
+$(CYTHON_MODULE): $(CPPSOURCES) $(CYSOURCES)
+	./setup.py build_ext --inplace
+
 coverage-debug: $(CPPSOURCES)
 	export CFLAGS="-pg -fprofile-arcs -ftest-coverage -O0"; ./setup.py \
 		build_ext --debug --inplace --libraries gcov
@@ -127,8 +132,9 @@ clean: FORCE
 	cd lib && $(MAKE) clean || true
 	cd tests && rm -rf khmertest_* || true
 	rm -f $(EXTENSION_MODULE)
+	rm -f $(CYTHON_MODULE)
 	rm -f khmer/*.pyc lib/*.pyc scripts/*.pyc tests/*.pyc oxli/*.pyc \
-		sandbox/*.pyc khmer/__pycache__/* sandbox/__pycache__/*
+		sandbox/*.pyc khmer/__pycache__/* sandbox/__pycache__/* khmer/_oxli/*.cpp
 	./setup.py clean --all || true
 	rm -f coverage-debug
 	rm -Rf .coverage

khmer/__init__.py

@@ -69,6 +69,8 @@
 # tests/test_read_parsers.py,scripts/{filter-abund-single,load-graph}.py
 # scripts/{abundance-dist-single,load-into-counting}.py
 
+from khmer import _oxli
+
 import sys
 
 from struct import pack, unpack

khmer/_khmer.cc

@@ -42,7 +42,7 @@ Contact: [email protected]
 
 // Must be first.
 #include <Python.h>
-
+#include "_khmer.hh"
 #include <iostream>
 
 #include "khmer.hh"
@@ -270,12 +270,6 @@ namespace khmer
 namespace python
 {
 
-typedef struct {
-    PyObject_HEAD
-    //! Pointer to the low-level genomic read object.
-    read_parsers:: Read *   read;
-} khmer_Read_Object;
-
 
 static
 void
@@ -388,23 +382,6 @@ static PyTypeObject khmer_Read_Type = {
 // ReadPairIterator -- return pairs of Read objects
 //
 
-
-typedef struct {
-    PyObject_HEAD
-    //! Pointer to the low-level parser object.
-    read_parsers:: IParser *  parser;
-} khmer_ReadParser_Object;
-
-
-typedef struct {
-    PyObject_HEAD
-    //! Pointer to Python parser object for reference counting purposes.
-    PyObject *  parent;
-    //! Persistent value of pair mode across invocations.
-    int pair_mode;
-} khmer_ReadPairIterator_Object;
-
-
 static
 void
 _ReadParser_dealloc(khmer_ReadParser_Object * obj)
@@ -785,11 +762,6 @@ _PyObject_to_khmer_ReadParser( PyObject * py_object )
     return ((python:: khmer_ReadParser_Object *)py_object)->parser;
 }
 
-typedef struct {
-    PyObject_HEAD
-    pre_partition_info *   PrePartitionInfo;
-} khmer_PrePartitionInfo_Object;
-
 static
 void
 khmer_PrePartitionInfo_dealloc(khmer_PrePartitionInfo_Object * obj)
@@ -827,11 +799,6 @@ static PyTypeObject khmer_PrePartitionInfo_Type = {
 /***********************************************************************/
 /***********************************************************************/
 
-typedef struct {
-    PyObject_HEAD
-    SeenSet * hashes;
-    WordLength ksize;
-} khmer_HashSet_Object;
 
 static khmer_HashSet_Object * create_HashSet_Object(SeenSet * h, WordLength k);
 
@@ -886,12 +853,6 @@ static PyObject* khmer_HashSet_new(PyTypeObject * type, PyObject * args,
 
 /***********************************************************************/
 
-typedef struct {
-    PyObject_HEAD
-    khmer_HashSet_Object * parent;
-    SeenSet::iterator * it;
-} _HashSet_iterobj;
-
 static
 void
 _HashSet_iter_dealloc(_HashSet_iterobj * obj)
@@ -1190,16 +1151,6 @@ static khmer_HashSet_Object * create_HashSet_Object(SeenSet * h, WordLength k)
 
 /***********************************************************************/
 
-typedef struct {
-    PyObject_HEAD
-    Hashtable * hashtable;
-} khmer_KHashtable_Object;
-
-typedef struct {
-    PyObject_HEAD
-    SubsetPartition * subset;
-} khmer_KSubsetPartition_Object;
-
 static void khmer_subset_dealloc(khmer_KSubsetPartition_Object * obj);
 
 static PyTypeObject khmer_KSubsetPartition_Type = {
@@ -1226,10 +1177,7 @@ static PyTypeObject khmer_KSubsetPartition_Type = {
     "subset object",                       /* tp_doc */
 };
 
-typedef struct {
-    khmer_KHashtable_Object khashtable;
-    Hashbits * hashbits;
-} khmer_KHashbits_Object;
+
 
 static void khmer_hashbits_dealloc(khmer_KHashbits_Object * obj);
 static PyObject* khmer_hashbits_new(PyTypeObject * type, PyObject * args,
@@ -3219,16 +3167,6 @@ CPYCHECKER_TYPE_OBJECT_FOR_TYPEDEF("khmer_KHashtable_Object")
 // KCountingHash object
 //
 
-typedef struct {
-    khmer_KHashtable_Object khashtable;
-    CountingHash * counting;
-} khmer_KCountingHash_Object;
-
-typedef struct {
-    PyObject_HEAD
-    ReadAligner * aligner;
-} khmer_ReadAligner_Object;
-
 static void khmer_counting_dealloc(khmer_KCountingHash_Object * obj);
 
 static
@@ -4039,11 +3977,6 @@ static PyMethodDef khmer_subset_methods[] = {
     {NULL, NULL, 0, NULL}           /* sentinel */
 };
 
-typedef struct {
-    PyObject_HEAD
-    LabelHash * labelhash;
-} khmer_KGraphLabels_Object;
-
 static PyObject * khmer_graphlabels_new(PyTypeObject * type, PyObject *args,
                                         PyObject *kwds);
 
@@ -4852,11 +4785,6 @@ static void khmer_subset_dealloc(khmer_KSubsetPartition_Object * obj)
 // KHLLCounter object
 //
 
-typedef struct {
-    PyObject_HEAD
-    HLLCounter * hllcounter;
-} khmer_KHLLCounter_Object;
-
 static PyObject* khmer_hllcounter_new(PyTypeObject * type, PyObject * args,
                                       PyObject * kwds)
 {
@@ -5220,11 +5148,6 @@ static PyObject * hllcounter_merge(khmer_KHLLCounter_Object * me,
  ********************************/
 
 
-typedef struct {
-    PyObject_HEAD
-    LinearAssembler * assembler;
-} khmer_KLinearAssembler_Object;
-
 #define is_linearassembler_obj(v)  (Py_TYPE(v) == &khmer_KLinearAssembler_Type)
 
 static void khmer_linearassembler_dealloc(khmer_KLinearAssembler_Object * obj)
@@ -5382,13 +5305,6 @@ static PyTypeObject khmer_KLinearAssembler_Type = {
 };
 
 
-
-typedef struct {
-    PyObject_HEAD
-    SimpleLabeledAssembler * assembler;
-} khmer_KSimpleLabeledAssembler_Object;
-
-
 static void khmer_simplelabeledassembler_dealloc(khmer_KLinearAssembler_Object * obj)
 {
     delete obj->assembler;
@@ -5533,11 +5449,6 @@ static PyTypeObject khmer_KSimpleLabeledAssembler_Type = {
  ********************************/
 
 
-typedef struct {
-    PyObject_HEAD
-    JunctionCountAssembler * assembler;
-} khmer_KJunctionCountAssembler_Object;
-
 #define is_junctioncountassembler_obj(v)  (Py_TYPE(v) == &khmer_KJunctionCountAssembler_Type)
 
 static void khmer_junctioncountassembler_dealloc(khmer_KJunctionCountAssembler_Object * obj)

khmer/_khmer.hh

@@ -0,0 +1,126 @@
+#include <Python.h>
+
+#include <iostream>
+
+#include "khmer.hh"
+#include "kmer_hash.hh"
+#include "hashtable.hh"
+#include "hashbits.hh"
+#include "counting.hh"
+#include "assembler.hh"
+#include "read_aligner.hh"
+#include "labelhash.hh"
+#include "khmer_exception.hh"
+#include "hllcounter.hh"
+
+
+namespace khmer {
+namespace python {
+
+
+typedef struct {
+    PyObject_HEAD
+    //! Pointer to the low-level genomic read object.
+    read_parsers:: Read *   read;
+} khmer_Read_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    //! Pointer to the low-level parser object.
+    read_parsers:: IParser *  parser;
+} khmer_ReadParser_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    //! Pointer to Python parser object for reference counting purposes.
+    PyObject *  parent;
+    //! Persistent value of pair mode across invocations.
+    int pair_mode;
+} khmer_ReadPairIterator_Object;
+
+}; //python
+
+typedef struct {
+    PyObject_HEAD
+    pre_partition_info *   PrePartitionInfo;
+} khmer_PrePartitionInfo_Object;
+
+
+
+typedef struct {
+    PyObject_HEAD
+    SeenSet * hashes;
+    WordLength ksize;
+} khmer_HashSet_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    khmer_HashSet_Object * parent;
+    SeenSet::iterator * it;
+} _HashSet_iterobj;
+
+
+typedef struct {
+    PyObject_HEAD
+    Hashtable * hashtable;
+} khmer_KHashtable_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    SubsetPartition * subset;
+} khmer_KSubsetPartition_Object;
+
+
+typedef struct {
+    khmer_KHashtable_Object khashtable;
+    Hashbits * hashbits;
+} khmer_KHashbits_Object;
+
+
+typedef struct {
+    khmer_KHashtable_Object khashtable;
+    CountingHash * counting;
+} khmer_KCountingHash_Object;
+
+typedef struct {
+    PyObject_HEAD
+    ReadAligner * aligner;
+} khmer_ReadAligner_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    LabelHash * labelhash;
+} khmer_KGraphLabels_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    HLLCounter * hllcounter;
+} khmer_KHLLCounter_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    LinearAssembler * assembler;
+} khmer_KLinearAssembler_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    SimpleLabeledAssembler * assembler;
+} khmer_KSimpleLabeledAssembler_Object;
+
+
+typedef struct {
+    PyObject_HEAD
+    JunctionCountAssembler * assembler;
+} khmer_KJunctionCountAssembler_Object;
+
+
+
+};