From 18f0cad7e64e7fc5112352742c5d6b9cd296c743 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 2 Nov 2016 14:56:51 -0700 Subject: [PATCH 001/185] Add basic skethes of streaming partitioner --- lib/hashtable.hh | 45 ++++++++++++++++ lib/partitioning.cc | 48 +++++++++++++++++ lib/partitioning.hh | 128 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+) create mode 100644 lib/partitioning.cc create mode 100644 lib/partitioning.hh diff --git a/lib/hashtable.hh b/lib/hashtable.hh index 3e6d6c84d1..2e04911230 100644 --- a/lib/hashtable.hh +++ b/lib/hashtable.hh @@ -83,6 +83,51 @@ struct IParser; namespace khmer { + +inline bool is_prime(uint64_t n) +{ + if (n < 2) { + return false; + } + if (n == 2) { + return true; + } + if (n % 2 == 0) { + return false; + } + for (unsigned long long i=3; i < sqrt(n); i += 2) { + if (n % i == 0) { + return false; + } + } + return true; +} + + +std::vector get_n_primes_near_x(n, x) +{ + std::vector primes; + if (x == 1 && n == 1) { + primes.push_back(1); + return primes; + } + + i = x - 1; + if (i % 2 == 0) { + i--; + } + while (primes.size() != n && i > 0) { + if (is_prime(i)) { + primes.push_back(i); + } + i -= 2; + } + + // might return < n primes if x is too small + return primes; +} + + class Hashtable: public KmerFactory // Base class implementation of a Bloom ht. { diff --git a/lib/partitioning.cc b/lib/partitioning.cc new file mode 100644 index 0000000000..305dbd4530 --- /dev/null +++ b/lib/partitioning.cc @@ -0,0 +1,48 @@ + +void StreamingPartitioner::consume_sequence(const std::string& seq, + uint64_t& n_consumed) +{ + + KmerIterator kmers(seq.c_str(), graph->ksize()); + HashIntoType kmer; + bool kmer_tagged; + unsigned int since = graph->_tag_density / 2 + 1; + + std::set tags; + bool known_component = false; + + // First check if we overlap any tags + kmer = kmers.next(); + tags->insert(kmer); //always tag the first k-mer + + while(!kmers.done()) { + kmer = kmers.next(); + bool is_new_kmer; + is_new_kmer = graph->test_and_set_bits(kmer); + + if (is_new_kmer) { + ++since; + } else { + kmer_tagged = tag_component_map.contains(kmer); + if (kmer_tagged) { + since = 1; + tags->insert(kmer); + known_component = true; + } else { + ++since; + } + } + + if (since >= graph->_tag_density) { + tags.insert(kmer); + since = 1; + } + } + tags.insert(kmer); // always tag the last k-mer + + if (known_component) { + + } else { + + } +} diff --git a/lib/partitioning.hh b/lib/partitioning.hh new file mode 100644 index 0000000000..7c6edf064d --- /dev/null +++ b/lib/partitioning.hh @@ -0,0 +1,128 @@ +/* +This file is part of khmer, https://github.com/dib-lab/khmer/, and is +Copyright (C) 2015-2016, The Regents of the University of California. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the Michigan State University nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +LICENSE (END) + +Contact: khmer-project@idyll.org +*/ +#ifndef PARTITIONING_HH +#define PARTITIONING_HH + +#include + +#include "khmer.hh" +#include "kmer_hash.hh" +#include "hashtable.hh" +#include "counting.hh" +#include "kmer_filters.hh" +#include "traversal.hh" +#include "labelhash.hh" + + +namespace khmer +{ + +template +class GuardedKmerMap { + + public: + + const Hashbits * filter; + std::map data; + + explicit GuardedKmerMap(WordLength ksize, + unsigned short n_tables, + uint64_t max_table_size) { + filter = new Hashbits(ksize, get_n_primes_near_x(n_tables, max_table_size)); + } + + T get(HashIntoType kmer) { + if (filter->get_count(kmer)) { + auto search = data.find(kmer); + if (search != data.end()) { + return search->first; + } + } + + return NULL; + } + + void set(HashIntoType kmer, T item) { + filter->count(kmer); + data[kmer] = item; + } + + bool contains(HashIntoType kmer) { + return get(kmer) != NULL; + } + +}; + + +class Component { + + private: + + static unsigned long long n_created; + std::set tags; + // some container of k-mers + + public: + + const unsigned long long component_id; + + explicit Component(): component_id(n_created) { + n_created++; + } + + void merge(Component& other) { + + } +}; +Component::n_created = 0; + + +class StreamingPartitioner { + + private: + + const Hashtable * graph; + GuardedKmerMap tag_component_map; + + void consume_sequence(const std::string& seq); + +}; + + +} + +#endif From e55b14269daa5d044052d974f9d4027ac729a005 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 3 Nov 2016 10:53:22 -0700 Subject: [PATCH 002/185] Sketch out searching and tag finding, component mapping --- lib/partitioning.cc | 146 ++++++++++++++++++++++++++++++++++++++++---- lib/partitioning.hh | 24 +++++++- 2 files changed, 156 insertions(+), 14 deletions(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 305dbd4530..f762879207 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -1,33 +1,66 @@ +void StreamingPartitioner::map_tags_to_component(std::set tags, + Component * comp) +{ + for (auto tag: tags) { + tag_component_map[tag] = comp; + } +} + + void StreamingPartitioner::consume_sequence(const std::string& seq, uint64_t& n_consumed) { + /* For the following comments, let G be the set of k-mers + * known in the graph before inserting the k-mers R from + * &seq, with / as the difference, + as the union, and & + * as the intersect operator. + */ KmerIterator kmers(seq.c_str(), graph->ksize()); - HashIntoType kmer; - bool kmer_tagged; - unsigned int since = graph->_tag_density / 2 + 1; + unsigned int since = 1; std::set tags; - bool known_component = false; + std::set seen; + std::set intersection; + KmerQueue search_from; + + bool in_known_territory = false; + bool found_tag_in_territory = false; // First check if we overlap any tags - kmer = kmers.next(); + Kmer kmer = kmers.next(); tags->insert(kmer); //always tag the first k-mer + bool is_new_kmer = graph->test_and_set_bits(kmer); while(!kmers.done()) { - kmer = kmers.next(); - bool is_new_kmer; - is_new_kmer = graph->test_and_set_bits(kmer); + bool kmer_tagged = false; if (is_new_kmer) { + // A k-mer from U/G must be searched from for tags, + // as it could be adjacent to a a k-mer in G/U + if (in_known_territory && found_tag_in_territory) { + // If we had found a tag in the U&G component we just + // left, add the component to the seen set. + seen.insert(intersection.begin(), intersection.end()); + } + intersection.clear(); + + search_from.push_back(kmer); + in_known_territory = false; + foud_tag_in_territory = false; ++since; } else { + // Keep track of connected components in U&G: when we exit + // this component, if there is a tag, we will want to add its nodes + // to the seen set, as we do not need to traverse from them in the tag search. + intersection.insert(kmer); + in_known_territory = true; kmer_tagged = tag_component_map.contains(kmer); if (kmer_tagged) { since = 1; tags->insert(kmer); - known_component = true; + found_tag_in_territory = true; } else { ++since; } @@ -37,12 +70,103 @@ void StreamingPartitioner::consume_sequence(const std::string& seq, tags.insert(kmer); since = 1; } + + is_new_kmer = graph->test_and_set_bits(kmer); + kmer = kmers.next(); } tags.insert(kmer); // always tag the last k-mer + if (is_new_kmer || !found_tag_in_territory) { + search_from.push_back(kmer); + } + intersection.clear(); - if (known_component) { - + // Now search for tagged nodes connected to U. + find_connected_tags(search_from, tags, seen); + + // Now resolve components. First, get components from existing tags. + std::set comps; + Component * comp; + for (auto tag: tags) { + if ((comp = tag_component_map.get(kmer)) != NULL) { + comps.insert(comp); + } + } + + if (comps.size() == 0) { + comp = new Component(); } else { + // get the first component + comp = *(comps.begin()); + if (comps.size() == 1) { + // map the new tags to this component + comp->add_tag(tags); + map_tags_to_component(tags, comp); + } else { + // merge the components + comp->merge(comps); + + } + } + // (re)map all the tags to the component + map_tags_to_component(tags, comp); +} + + +void StreamingPartitioner::find_connected_tags(KmerQueue node_q, + std::set& found_tags, + std::set& seen) +{ + + // put a 0 on the breadth queue for each element in the starting node queue + std::queue breadth_q(std::deque(node_q.size(), 0)); + + unsigned int cur_breadth = 0; + const unsigned int max_breadth = (2 * graph->_tag_density) + 1; + + unsigned int total = 0; + unsigned int nfound = 0; + + KmerFilter filter = [&] (const Kmer& n) -> bool { + return set_contains(seen, n); + }; + Traverser traverser(graph, filter); + + while(!node_q.empty()) { + + Kmer node = node_q.front(); + node_q.pop(); + + unsigned int breadth = breadth_q.front(); + breadth_q.pop(); + + // keep track of seen kmers + seen.insert(node); + total++; + + // Found a tag! + if (tag_component_map.contains(node)) { + found_tags.insert(node); + continue; + } + + if (!(breadth >= cur_breadth)) { + throw khmer_exception("Desynchonization between traversal " + "and breadth tracking. Did you forget " + "to pop the node or breadth queue?"); + } + if (breadth > cur_breadth) { + cur_breadth = breadth; + } + + if (breadth >= max_breadth) { + continue; // truncate search @CTB exit? + } + + nfound = traverser.traverse(node, node_q); + for (unsigned int i = 0; i tags; - // some container of k-mers public: const unsigned long long component_id; + std::set tags; explicit Component(): component_id(n_created) { n_created++; } - void merge(Component& other) { + void merge(std::set other_comps) { + for (auto other : other_comps) { + if (other == this) { + continue; + } + this->add_tag(other->tags); + delete other; + } + } + + void add_tag(HashIntoType tag) { + tags.insert(tag); + } + void add_tag(std::set new_tags) { + for (auto tag: new_tags) { + add_tag(tag); + } } }; Component::n_created = 0; @@ -119,6 +134,9 @@ class StreamingPartitioner { GuardedKmerMap tag_component_map; void consume_sequence(const std::string& seq); + void map_tags_to_component(std::set tags, Component * comp); + void find_connected_tags(std::set start_kmers, + SeenSet& found_tags) }; From 12141f9aa8ad4713633f9b27c00192dc610a79a0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 4 Nov 2016 21:52:38 -0700 Subject: [PATCH 003/185] First pass at cythonizing StreamingPartitioner --- khmer/_oxli.pxd | 59 +++++++++ khmer/_oxli.pyx | 31 +++++ lib/hashtable.hh | 4 +- lib/khmer_exception.hh | 8 ++ lib/partitioning.cc | 290 ++++++++++++++++++++++++----------------- lib/partitioning.hh | 68 +++++++--- setup.py | 20 ++- 7 files changed, 333 insertions(+), 147 deletions(-) create mode 100644 khmer/_oxli.pxd create mode 100644 khmer/_oxli.pyx diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd new file mode 100644 index 0000000000..5c2e97bfa0 --- /dev/null +++ b/khmer/_oxli.pxd @@ -0,0 +1,59 @@ +from libcpp cimport bool +from libcpp.string cimport string +from libcpp.vector cimport vector +from libcpp.map cimport map +from libcpp.set cimport set +from libcpp.queue cimport queue +from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr +from libc.stdint cimport uint32_t, uint8_t, uint64_t + + +cdef extern from "khmer.hh" namespace "khmer": + ctypedef unsigned long long int HashIntoType + ctypedef unsigned char WordLength + ctypedef unsigned short int BoundedCounterType + ctypedef queue[Kmer] KmerQueue + + +cdef extern from "hashtable.hh" namespace "khmer": + cdef cppclass CyHashtable "khmer::Hashtable": + CyHashtable(WordLength) + + +cdef extern from "kmer_hash.hh" namespace "khmer": + cdef cppclass Kmer: + HashIntoType kmer_f + HashIntoType kmer_r + HashIntoType kmer_u + + Kmer(HashIntoType, HashIntoType, HashIntoType) + Kmer(string, WordLength) + Kmer() + + bool is_forward() const + + +cdef extern from "partitioning.hh" namespace "khmer": + cdef cppclass CyComponent "khmer::Component": + const uint64_t component_id + set[HashIntoType] tags + + void merge(set[shared_ptr[CyComponent]]) + void add_tag(HashIntoType) + void add_tag(set[HashIntoType]) + uint64_t get_n_tags() + uint64_t get_n_merges() + + ctypedef shared_ptr[CyComponent] ComponentPtr + + +cdef extern from "partitioning.hh" namespace "khmer": + cdef cppclass CyStreamingPartitioner "khmer::StreamingPartitioner": + StreamingPartitioner(weak_ptr[CyHashtable] graph) except +MemoryError + + void consume_sequence(string&) except +MemoryError + void map_tags_to_component(set[HashIntoType], ComponentPtr) + void find_connected_tags(queue[Kmer]&, + set[HashIntoType]&, + set[HashIntoType]&) except +MemoryError + uint64_t get_n_components() diff --git a/khmer/_oxli.pyx b/khmer/_oxli.pyx new file mode 100644 index 0000000000..df4835dde8 --- /dev/null +++ b/khmer/_oxli.pyx @@ -0,0 +1,31 @@ +import cython +from cython.operator cimport dereference as deref, preincrement as inc +from libc.limits cimport UINT_MAX + +cdef class Component: + + cdef ComponentPtr _this + + def __cinit__(self, Component other=None): + if other is not None: + self._this.reset(other._this.get()) + else: + self._this.reset(new CyComponent()) + + property component_id: + def __get__(self): + return deref(self._this).component_id + + property n_merges: + def __get__(self): + return deref(self._this).get_n_merges() + + def __len__(self): + return deref(self._this).get_n_tags() + + def __iter__(self): + it = deref(self._this).tags.begin() + while it != deref(self._this).tags.end(): + yield deref(it) + inc(it) + diff --git a/lib/hashtable.hh b/lib/hashtable.hh index 2e04911230..6ededd14cd 100644 --- a/lib/hashtable.hh +++ b/lib/hashtable.hh @@ -104,7 +104,7 @@ inline bool is_prime(uint64_t n) } -std::vector get_n_primes_near_x(n, x) +inline std::vector get_n_primes_near_x(uint32_t n, uint64_t x) { std::vector primes; if (x == 1 && n == 1) { @@ -112,7 +112,7 @@ std::vector get_n_primes_near_x(n, x) return primes; } - i = x - 1; + uint64_t i = x - 1; if (i % 2 == 0) { i--; } diff --git a/lib/khmer_exception.hh b/lib/khmer_exception.hh index 3e4b40d9f8..38482af622 100644 --- a/lib/khmer_exception.hh +++ b/lib/khmer_exception.hh @@ -86,6 +86,14 @@ public: : khmer_exception(msg) { } }; + +class khmer_ptr_exception : public khmer_exception +{ +public: + explicit khmer_ptr_exception(const std::string& msg) + : khmer_exception(msg) { } +}; + /////// Specialised Exceptions ///// class InvalidStream : public khmer_file_exception diff --git a/lib/partitioning.cc b/lib/partitioning.cc index f762879207..17e6dea5b2 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -1,172 +1,216 @@ +#include +#include +#include +#include +#include + +#include "hashtable.hh" +#include "hashbits.hh" +#include "counting.hh" +#include "partitioning.hh" + +using namespace khmer; + +StreamingPartitioner::StreamingPartitioner(std::weak_ptr& graph) : + graph(graph), _tag_density(DEFAULT_TAG_DENSITY), n_components(0) +{ + this->graph = graph; + + if (auto graphptr = graph.lock()) { + std::vector graph_table_sizes = graphptr->get_tablesizes(); + uint64_t graph_max_table_size = *std::max_element(graph_table_sizes.begin(), + graph_table_sizes.end()); + + // We can guess that, given N k-mers in the graph, there will be + // approximately N / _tag_density tags. If we want to the filter false + // positive rate to be about the same as the graph, we should make its table + // sizes proportional by the number of tags. Here, we use _tag_density-2 + // because we always tag the first and last k-mers in a read. + tag_component_map = std::unique_ptr( + new GuardedKmerCompMap(graphptr->ksize(), + graphptr->n_tables(), + graph_max_table_size / (_tag_density-2))); + } else { + throw khmer_ptr_exception("Hashtable has been deleted."); + } +} + void StreamingPartitioner::map_tags_to_component(std::set tags, - Component * comp) + ComponentPtr& comp) { for (auto tag: tags) { - tag_component_map[tag] = comp; + tag_component_map->set(tag, comp); } } -void StreamingPartitioner::consume_sequence(const std::string& seq, - uint64_t& n_consumed) +void StreamingPartitioner::consume_sequence(const std::string& seq) { /* For the following comments, let G be the set of k-mers * known in the graph before inserting the k-mers R from * &seq, with / as the difference, + as the union, and & * as the intersect operator. */ - - KmerIterator kmers(seq.c_str(), graph->ksize()); - unsigned int since = 1; - - std::set tags; - std::set seen; - std::set intersection; - KmerQueue search_from; - - bool in_known_territory = false; - bool found_tag_in_territory = false; - - // First check if we overlap any tags - Kmer kmer = kmers.next(); - tags->insert(kmer); //always tag the first k-mer - bool is_new_kmer = graph->test_and_set_bits(kmer); - - while(!kmers.done()) { - bool kmer_tagged = false; - - if (is_new_kmer) { - // A k-mer from U/G must be searched from for tags, - // as it could be adjacent to a a k-mer in G/U - if (in_known_territory && found_tag_in_territory) { - // If we had found a tag in the U&G component we just - // left, add the component to the seen set. - seen.insert(intersection.begin(), intersection.end()); + if (auto graphptr = graph.lock()) { + KmerIterator kmers(seq.c_str(), graphptr->ksize()); + unsigned int since = 1; + + std::set tags; + std::set seen; + std::set intersection; + KmerQueue search_from; + + bool in_known_territory = false; + bool found_tag_in_territory = false; + + // First check if we overlap any tags + Kmer kmer = kmers.next(); + tags.insert(kmer); //always tag the first k-mer + bool is_new_kmer = graphptr->test_and_set_bits(kmer); + + while(!kmers.done()) { + bool kmer_tagged = false; + + if (is_new_kmer) { + // A k-mer from U/G must be searched from for tags, + // as it could be adjacent to a a k-mer in G/U + if (in_known_territory && found_tag_in_territory) { + // If we had found a tag in the U&G component we just + // left, add the component to the seen set. + seen.insert(intersection.begin(), intersection.end()); + } + intersection.clear(); + + search_from.push(kmer); + in_known_territory = false; + found_tag_in_territory = false; + ++since; + } else { + // Keep track of connected components in U&G: when we exit + // this component, if there is a tag, we will want to add its nodes + // to the seen set, as we do not need to traverse from them in the tag search. + intersection.insert(kmer); + in_known_territory = true; + kmer_tagged = tag_component_map->contains(kmer); + if (kmer_tagged) { + since = 1; + tags.insert(kmer); + found_tag_in_territory = true; + } else { + ++since; + } } - intersection.clear(); - search_from.push_back(kmer); - in_known_territory = false; - foud_tag_in_territory = false; - ++since; - } else { - // Keep track of connected components in U&G: when we exit - // this component, if there is a tag, we will want to add its nodes - // to the seen set, as we do not need to traverse from them in the tag search. - intersection.insert(kmer); - in_known_territory = true; - kmer_tagged = tag_component_map.contains(kmer); - if (kmer_tagged) { + if (since >= _tag_density) { + tags.insert(kmer); since = 1; - tags->insert(kmer); - found_tag_in_territory = true; - } else { - ++since; } - } - if (since >= graph->_tag_density) { - tags.insert(kmer); - since = 1; + is_new_kmer = graphptr->test_and_set_bits(kmer); + kmer = kmers.next(); } + tags.insert(kmer); // always tag the last k-mer + if (is_new_kmer || !found_tag_in_territory) { + search_from.push(kmer); + } + intersection.clear(); - is_new_kmer = graph->test_and_set_bits(kmer); - kmer = kmers.next(); - } - tags.insert(kmer); // always tag the last k-mer - if (is_new_kmer || !found_tag_in_territory) { - search_from.push_back(kmer); - } - intersection.clear(); - - // Now search for tagged nodes connected to U. - find_connected_tags(search_from, tags, seen); + // Now search for tagged nodes connected to U. + find_connected_tags(search_from, tags, seen); - // Now resolve components. First, get components from existing tags. - std::set comps; - Component * comp; - for (auto tag: tags) { - if ((comp = tag_component_map.get(kmer)) != NULL) { - comps.insert(comp); + // Now resolve components. First, get components from existing tags. + std::set comps; + ComponentPtr comp; + for (auto tag: tags) { + if ((comp = tag_component_map->get(tag)) != NULL) { + comps.insert(comp); + } } - } - if (comps.size() == 0) { - comp = new Component(); - } else { - // get the first component - comp = *(comps.begin()); - if (comps.size() == 1) { - // map the new tags to this component - comp->add_tag(tags); - map_tags_to_component(tags, comp); + if (comps.size() == 0) { + comp = std::make_shared(); + n_components++; } else { - // merge the components - comp->merge(comps); - + // get the first component + comp = *(comps.begin()); + if (comps.size() == 1) { + // map the new tags to this component + comp->add_tag(tags); + map_tags_to_component(tags, comp); + } else { + // merge the components + comp->merge(comps); + n_components -= comps.size() - 1; + } } + // (re)map all the tags to the component + map_tags_to_component(tags, comp); + } else { + throw khmer_ptr_exception("Hashtable has been deleted."); } - // (re)map all the tags to the component - map_tags_to_component(tags, comp); - } -void StreamingPartitioner::find_connected_tags(KmerQueue node_q, +void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, std::set& found_tags, std::set& seen) { - // put a 0 on the breadth queue for each element in the starting node queue - std::queue breadth_q(std::deque(node_q.size(), 0)); + if (auto graphptr = graph.lock()) { - unsigned int cur_breadth = 0; - const unsigned int max_breadth = (2 * graph->_tag_density) + 1; + // put a 0 on the breadth queue for each element in the starting node queue + std::queue breadth_q(std::deque(node_q.size(), 0)); - unsigned int total = 0; - unsigned int nfound = 0; + unsigned int cur_breadth = 0; + const unsigned int max_breadth = (2 * _tag_density) + 1; - KmerFilter filter = [&] (const Kmer& n) -> bool { - return set_contains(seen, n); - }; - Traverser traverser(graph, filter); + unsigned int total = 0; + unsigned int nfound = 0; - while(!node_q.empty()) { + KmerFilter filter = [&] (const Kmer& n) -> bool { + return set_contains(seen, n); + }; + Traverser traverser(graphptr.get(), filter); - Kmer node = node_q.front(); - node_q.pop(); + while(!node_q.empty()) { - unsigned int breadth = breadth_q.front(); - breadth_q.pop(); + Kmer node = node_q.front(); + node_q.pop(); - // keep track of seen kmers - seen.insert(node); - total++; + unsigned int breadth = breadth_q.front(); + breadth_q.pop(); - // Found a tag! - if (tag_component_map.contains(node)) { - found_tags.insert(node); - continue; - } + // keep track of seen kmers + seen.insert(node); + total++; - if (!(breadth >= cur_breadth)) { - throw khmer_exception("Desynchonization between traversal " - "and breadth tracking. Did you forget " - "to pop the node or breadth queue?"); - } - if (breadth > cur_breadth) { - cur_breadth = breadth; - } + // Found a tag! + if (tag_component_map->contains(node)) { + found_tags.insert(node); + continue; + } - if (breadth >= max_breadth) { - continue; // truncate search @CTB exit? - } + if (!(breadth >= cur_breadth)) { + throw khmer_exception("Desynchonization between traversal " + "and breadth tracking. Did you forget " + "to pop the node or breadth queue?"); + } + if (breadth > cur_breadth) { + cur_breadth = breadth; + } - nfound = traverser.traverse(node, node_q); - for (unsigned int i = 0; i= max_breadth) { + continue; // truncate search @CTB exit? + } + + nfound = traverser.traverse(node, node_q); + for (unsigned int i = 0; i +#include #include "khmer.hh" #include "kmer_hash.hh" #include "hashtable.hh" +#include "hashbits.hh" #include "counting.hh" #include "kmer_filters.hh" #include "traversal.hh" -#include "labelhash.hh" namespace khmer @@ -56,20 +57,23 @@ class GuardedKmerMap { public: - const Hashbits * filter; + // Filter should be owned exclusively by GuardedKmerMap + std::unique_ptr filter; std::map data; explicit GuardedKmerMap(WordLength ksize, unsigned short n_tables, - uint64_t max_table_size) { - filter = new Hashbits(ksize, get_n_primes_near_x(n_tables, max_table_size)); + uint64_t max_table_size) + { + std::vector table_sizes = get_n_primes_near_x(n_tables, max_table_size); + filter = std::unique_ptr(new Hashbits(ksize, table_sizes)); } T get(HashIntoType kmer) { if (filter->get_count(kmer)) { auto search = data.find(kmer); if (search != data.end()) { - return search->first; + return search->second; } } @@ -92,24 +96,29 @@ class Component { private: - static unsigned long long n_created; + static uint64_t n_created; + uint64_t n_merges; public: - const unsigned long long component_id; + const uint64_t component_id; std::set tags; - explicit Component(): component_id(n_created) { + explicit Component(): component_id(n_created), n_merges(0) { n_created++; } - void merge(std::set other_comps) { + ~Component() { + tags.clear(); // maybe not necessary? + } + + void merge(std::set> other_comps) { for (auto other : other_comps) { - if (other == this) { + if (other.get() == this) { continue; } this->add_tag(other->tags); - delete other; + this->n_merges = other->get_n_merges() + 1; } } @@ -122,22 +131,43 @@ class Component { add_tag(tag); } } -}; -Component::n_created = 0; + uint64_t get_n_tags() { + return tags.size(); + } + + uint64_t get_n_merges() { + return n_merges; + } +}; +static uint64_t Component::n_created = 0; +typedef std::shared_ptr ComponentPtr; +typedef GuardedKmerMap GuardedKmerCompMap; class StreamingPartitioner { private: - const Hashtable * graph; - GuardedKmerMap tag_component_map; + uint32_t _tag_density; - void consume_sequence(const std::string& seq); - void map_tags_to_component(std::set tags, Component * comp); - void find_connected_tags(std::set start_kmers, - SeenSet& found_tags) + // We're not graph's owner, simply an observer. + std::weak_ptr graph; + // We should exlusively own tag_component_map. + std::unique_ptr tag_component_map; + uint64_t n_components; + + public: + explicit StreamingPartitioner(std::weak_ptr& graph); + + void consume_sequence(const std::string& seq); + void map_tags_to_component(std::set tags, ComponentPtr& comp); + void find_connected_tags(KmerQueue& node_q, + std::set& found_tags, + std::set& seen); + uint64_t get_n_components() { + return n_components; + } }; diff --git a/setup.py b/setup.py index d52ead38e7..4aafa3f0e1 100755 --- a/setup.py +++ b/setup.py @@ -54,6 +54,8 @@ from distutils.dist import Distribution from distutils.errors import DistutilsPlatformError +from Cython.Build import cythonize + import versioneer ez_setup.use_setuptools(version="3.4.1") @@ -131,7 +133,8 @@ def check_for_openmp(): "hllcounter", "khmer_exception", "read_aligner", "subset", "read_parsers", "kmer_filters", "traversal", "assembler", "alphabets"]) -SOURCES = ["khmer/_khmer.cc"] +#CP_SOURCES = ["khmer/_khmer.cc"] +SOURCES = [] SOURCES.extend(path_join("lib", bn + ".cc") for bn in [ "read_parsers", "kmer_hash", "hashtable", "hashbits", "labelhash", "counting", "subset", "read_aligner", @@ -163,7 +166,18 @@ def check_for_openmp(): "define_macros": [("VERSION", versioneer.get_version()), ], } -EXTENSION_MOD = Extension("khmer._khmer", ** EXTENSION_MOD_DICT) +CP_EXT_MOD_DICT = dict(EXTENSION_MOD_DICT) +CP_EXT_MOD_DICT['sources'].insert(0, "khmer/_khmer.cc") +EXTENSION_MODS = [Extension("khmer._khmer", ** CP_EXT_MOD_DICT)] + +CY_EXT_MOD_DICT = dict(EXTENSION_MOD_DICT) +CY_EXT_MOD_DICT['sources'].insert(0, "khmer/_oxli.pyx") +CY_EXT_MOD_DICT['sources'].append(path_join("lib", "partitioning.cc")) +CY_EXT_MOD_DICT['depends'].append(path_join("lib", "partitioning.hh")) +CY_EXTENSION_MOD = Extension("khmer._oxli", ** CY_EXT_MOD_DICT) +EXTENSION_MODS.extend(cythonize([CY_EXTENSION_MOD])) + + SCRIPTS = [] SCRIPTS.extend([path_join("scripts", script) for script in os_listdir("scripts") @@ -234,7 +248,7 @@ def check_for_openmp(): # "oxli = oxli:main" # ] # }, - "ext_modules": [EXTENSION_MOD, ], + "ext_modules": EXTENSION_MODS, # "platforms": '', # empty as is conveyed by the classifiers below # "license": '', # empty as is conveyed by the classifier below "include_package_data": True, From 9f6c196c6c7a0a83fec36fce8c8ddf3b974fa38e Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 4 Nov 2016 21:57:04 -0700 Subject: [PATCH 004/185] Move static member initialization to cc file --- lib/partitioning.cc | 2 ++ lib/partitioning.hh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 17e6dea5b2..b5d23200d6 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -11,6 +11,8 @@ using namespace khmer; +uint64_t Component::n_created = 0; + StreamingPartitioner::StreamingPartitioner(std::weak_ptr& graph) : graph(graph), _tag_density(DEFAULT_TAG_DENSITY), n_components(0) { diff --git a/lib/partitioning.hh b/lib/partitioning.hh index 9e002d4b7d..030c37f558 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -140,7 +140,7 @@ class Component { return n_merges; } }; -static uint64_t Component::n_created = 0; + typedef std::shared_ptr ComponentPtr; typedef GuardedKmerMap GuardedKmerCompMap; From b3c6a3d3b8901b9d15901e30e79f922f4725384f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 5 Nov 2016 15:40:17 -0700 Subject: [PATCH 005/185] Working Cython bindings and initial tests for streaming partitioner --- khmer/__init__.py | 2 + khmer/_khmer.cc | 93 +- khmer/_khmer.hh | 126 + khmer/_oxli.cpp | 5004 +++++++++++++++++++++++++++++++++++++ khmer/_oxli.pxd | 7 +- khmer/_oxli.pyx | 27 + lib/partitioning.cc | 26 +- lib/partitioning.hh | 7 +- setup.py | 45 +- tests/graph_features.py | 518 ++++ tests/test_assembly.py | 466 +--- tests/test_cython_oxli.py | 43 + 12 files changed, 5784 insertions(+), 580 deletions(-) create mode 100644 khmer/_khmer.hh create mode 100644 khmer/_oxli.cpp create mode 100644 tests/graph_features.py create mode 100644 tests/test_cython_oxli.py diff --git a/khmer/__init__.py b/khmer/__init__.py index 6525bcc2c5..d94f986fe8 100644 --- a/khmer/__init__.py +++ b/khmer/__init__.py @@ -69,6 +69,8 @@ # tests/test_read_parsers.py,scripts/{filter-abund-single,load-graph}.py # scripts/{abundance-dist-single,load-into-counting}.py +from khmer import _oxli + import sys from struct import pack, unpack diff --git a/khmer/_khmer.cc b/khmer/_khmer.cc index 3ac6a67eca..9f9f86ebf2 100644 --- a/khmer/_khmer.cc +++ b/khmer/_khmer.cc @@ -42,7 +42,7 @@ Contact: khmer-project@idyll.org // Must be first. #include - +#include "_khmer.hh" #include #include "khmer.hh" @@ -270,12 +270,6 @@ namespace khmer namespace python { -typedef struct { - PyObject_HEAD - //! Pointer to the low-level genomic read object. - read_parsers:: Read * read; -} khmer_Read_Object; - static void @@ -388,23 +382,6 @@ static PyTypeObject khmer_Read_Type = { // ReadPairIterator -- return pairs of Read objects // - -typedef struct { - PyObject_HEAD - //! Pointer to the low-level parser object. - read_parsers:: IParser * parser; -} khmer_ReadParser_Object; - - -typedef struct { - PyObject_HEAD - //! Pointer to Python parser object for reference counting purposes. - PyObject * parent; - //! Persistent value of pair mode across invocations. - int pair_mode; -} khmer_ReadPairIterator_Object; - - static void _ReadParser_dealloc(khmer_ReadParser_Object * obj) @@ -785,11 +762,6 @@ _PyObject_to_khmer_ReadParser( PyObject * py_object ) return ((python:: khmer_ReadParser_Object *)py_object)->parser; } -typedef struct { - PyObject_HEAD - pre_partition_info * PrePartitionInfo; -} khmer_PrePartitionInfo_Object; - static void khmer_PrePartitionInfo_dealloc(khmer_PrePartitionInfo_Object * obj) @@ -827,11 +799,6 @@ static PyTypeObject khmer_PrePartitionInfo_Type = { /***********************************************************************/ /***********************************************************************/ -typedef struct { - PyObject_HEAD - SeenSet * hashes; - WordLength ksize; -} khmer_HashSet_Object; static khmer_HashSet_Object * create_HashSet_Object(SeenSet * h, WordLength k); @@ -886,12 +853,6 @@ static PyObject* khmer_HashSet_new(PyTypeObject * type, PyObject * args, /***********************************************************************/ -typedef struct { - PyObject_HEAD - khmer_HashSet_Object * parent; - SeenSet::iterator * it; -} _HashSet_iterobj; - static void _HashSet_iter_dealloc(_HashSet_iterobj * obj) @@ -1190,16 +1151,6 @@ static khmer_HashSet_Object * create_HashSet_Object(SeenSet * h, WordLength k) /***********************************************************************/ -typedef struct { - PyObject_HEAD - Hashtable * hashtable; -} khmer_KHashtable_Object; - -typedef struct { - PyObject_HEAD - SubsetPartition * subset; -} khmer_KSubsetPartition_Object; - static void khmer_subset_dealloc(khmer_KSubsetPartition_Object * obj); static PyTypeObject khmer_KSubsetPartition_Type = { @@ -1226,10 +1177,7 @@ static PyTypeObject khmer_KSubsetPartition_Type = { "subset object", /* tp_doc */ }; -typedef struct { - khmer_KHashtable_Object khashtable; - Hashbits * hashbits; -} khmer_KHashbits_Object; + static void khmer_hashbits_dealloc(khmer_KHashbits_Object * obj); static PyObject* khmer_hashbits_new(PyTypeObject * type, PyObject * args, @@ -3219,16 +3167,6 @@ CPYCHECKER_TYPE_OBJECT_FOR_TYPEDEF("khmer_KHashtable_Object") // KCountingHash object // -typedef struct { - khmer_KHashtable_Object khashtable; - CountingHash * counting; -} khmer_KCountingHash_Object; - -typedef struct { - PyObject_HEAD - ReadAligner * aligner; -} khmer_ReadAligner_Object; - static void khmer_counting_dealloc(khmer_KCountingHash_Object * obj); static @@ -4039,11 +3977,6 @@ static PyMethodDef khmer_subset_methods[] = { {NULL, NULL, 0, NULL} /* sentinel */ }; -typedef struct { - PyObject_HEAD - LabelHash * labelhash; -} khmer_KGraphLabels_Object; - static PyObject * khmer_graphlabels_new(PyTypeObject * type, PyObject *args, PyObject *kwds); @@ -4852,11 +4785,6 @@ static void khmer_subset_dealloc(khmer_KSubsetPartition_Object * obj) // KHLLCounter object // -typedef struct { - PyObject_HEAD - HLLCounter * hllcounter; -} khmer_KHLLCounter_Object; - static PyObject* khmer_hllcounter_new(PyTypeObject * type, PyObject * args, PyObject * kwds) { @@ -5220,11 +5148,6 @@ static PyObject * hllcounter_merge(khmer_KHLLCounter_Object * me, ********************************/ -typedef struct { - PyObject_HEAD - LinearAssembler * assembler; -} khmer_KLinearAssembler_Object; - #define is_linearassembler_obj(v) (Py_TYPE(v) == &khmer_KLinearAssembler_Type) static void khmer_linearassembler_dealloc(khmer_KLinearAssembler_Object * obj) @@ -5382,13 +5305,6 @@ static PyTypeObject khmer_KLinearAssembler_Type = { }; - -typedef struct { - PyObject_HEAD - SimpleLabeledAssembler * assembler; -} khmer_KSimpleLabeledAssembler_Object; - - static void khmer_simplelabeledassembler_dealloc(khmer_KLinearAssembler_Object * obj) { delete obj->assembler; @@ -5533,11 +5449,6 @@ static PyTypeObject khmer_KSimpleLabeledAssembler_Type = { ********************************/ -typedef struct { - PyObject_HEAD - JunctionCountAssembler * assembler; -} khmer_KJunctionCountAssembler_Object; - #define is_junctioncountassembler_obj(v) (Py_TYPE(v) == &khmer_KJunctionCountAssembler_Type) static void khmer_junctioncountassembler_dealloc(khmer_KJunctionCountAssembler_Object * obj) diff --git a/khmer/_khmer.hh b/khmer/_khmer.hh new file mode 100644 index 0000000000..17fcb0d1c0 --- /dev/null +++ b/khmer/_khmer.hh @@ -0,0 +1,126 @@ +#include + +#include + +#include "khmer.hh" +#include "kmer_hash.hh" +#include "hashtable.hh" +#include "hashbits.hh" +#include "counting.hh" +#include "assembler.hh" +#include "read_aligner.hh" +#include "labelhash.hh" +#include "khmer_exception.hh" +#include "hllcounter.hh" + + +namespace khmer { +namespace python { + + +typedef struct { + PyObject_HEAD + //! Pointer to the low-level genomic read object. + read_parsers:: Read * read; +} khmer_Read_Object; + + +typedef struct { + PyObject_HEAD + //! Pointer to the low-level parser object. + read_parsers:: IParser * parser; +} khmer_ReadParser_Object; + + +typedef struct { + PyObject_HEAD + //! Pointer to Python parser object for reference counting purposes. + PyObject * parent; + //! Persistent value of pair mode across invocations. + int pair_mode; +} khmer_ReadPairIterator_Object; + +}; //python + +typedef struct { + PyObject_HEAD + pre_partition_info * PrePartitionInfo; +} khmer_PrePartitionInfo_Object; + + + +typedef struct { + PyObject_HEAD + SeenSet * hashes; + WordLength ksize; +} khmer_HashSet_Object; + + +typedef struct { + PyObject_HEAD + khmer_HashSet_Object * parent; + SeenSet::iterator * it; +} _HashSet_iterobj; + + +typedef struct { + PyObject_HEAD + Hashtable * hashtable; +} khmer_KHashtable_Object; + + +typedef struct { + PyObject_HEAD + SubsetPartition * subset; +} khmer_KSubsetPartition_Object; + + +typedef struct { + khmer_KHashtable_Object khashtable; + Hashbits * hashbits; +} khmer_KHashbits_Object; + + +typedef struct { + khmer_KHashtable_Object khashtable; + CountingHash * counting; +} khmer_KCountingHash_Object; + +typedef struct { + PyObject_HEAD + ReadAligner * aligner; +} khmer_ReadAligner_Object; + + +typedef struct { + PyObject_HEAD + LabelHash * labelhash; +} khmer_KGraphLabels_Object; + + +typedef struct { + PyObject_HEAD + HLLCounter * hllcounter; +} khmer_KHLLCounter_Object; + + +typedef struct { + PyObject_HEAD + LinearAssembler * assembler; +} khmer_KLinearAssembler_Object; + + +typedef struct { + PyObject_HEAD + SimpleLabeledAssembler * assembler; +} khmer_KSimpleLabeledAssembler_Object; + + +typedef struct { + PyObject_HEAD + JunctionCountAssembler * assembler; +} khmer_KJunctionCountAssembler_Object; + + + +}; diff --git a/khmer/_oxli.cpp b/khmer/_oxli.cpp new file mode 100644 index 0000000000..5f4a4e281b --- /dev/null +++ b/khmer/_oxli.cpp @@ -0,0 +1,5004 @@ +/* Generated by Cython 0.25.1 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "define_macros": [ + [ + "VERSION", + "2.0+520.g9f6c196.dirty" + ] + ], + "depends": [ + "khmer/_khmer.hh" + ], + "extra_compile_args": [ + "-O3", + "-std=c++11", + "-pedantic", + "-arch", + "x86_64", + "-mmacosx-version-min=10.7", + "-stdlib=libc++" + ], + "extra_objects": [ + "build/temp.macosx-10.6-x86_64-3.5/khmer/_khmer.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/read_parsers.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/kmer_hash.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/hashtable.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/hashbits.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/labelhash.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/counting.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/subset.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/read_aligner.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/hllcounter.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/traversal.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/kmer_filters.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/assembler.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/alphabets.o", + "build/temp.macosx-10.6-x86_64-3.5/lib/partitioning.o", + "build/temp.macosx-10.6-x86_64-3.5/third-party/smhasher/MurmurHash3.o" + ], + "language": "c++" + }, + "module_name": "khmer._oxli" +} +END: Cython Metadata */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) + #error Cython requires Python 2.6+ or Python 3.2+. +#else +#define CYTHON_ABI "0_25_1" +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#ifndef HAVE_LONG_LONG + #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) + #define HAVE_LONG_LONG + #endif +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#ifdef PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#elif defined(PYSTON_VERSION) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #if PY_MAJOR_VERSION < 3 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #ifndef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if PY_VERSION_HEX < 0x030300F0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif +#endif +#if !defined(CYTHON_FAST_PYCCALL) +#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #include "longintrepr.h" + #undef SHIFT + #undef BASE + #undef MASK +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) + #define Py_OptimizeFlag 0 +#endif +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyClass_Type +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #define __Pyx_PyCFunctionFast _PyCFunctionFast +#endif +#if CYTHON_FAST_PYCCALL +#define __Pyx_PyFastCFunction_Check(func)\ + ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) +#else +#define __Pyx_PyFastCFunction_Check(func) 0 +#endif +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) +#else + #define CYTHON_PEP393_ENABLED 0 + #define PyUnicode_1BYTE_KIND 1 + #define PyUnicode_2BYTE_KIND 2 + #define PyUnicode_4BYTE_KIND 4 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) + #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_PYSTON + #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) +#else + #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) +#endif +#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) + #define PyObject_ASCII(o) PyObject_Repr(o) +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyStringObject PyUnicodeObject + #define PyString_Type PyUnicode_Type + #define PyString_Check PyUnicode_Check + #define PyString_CheckExact PyUnicode_CheckExact +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) + #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) +#else + #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) + #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#endif +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#if PY_MAJOR_VERSION >= 3 + #define PyIntObject PyLongObject + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define PyNumber_Int PyNumber_Long +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBoolObject PyLongObject +#endif +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY + #ifndef PyUnicode_InternFromString + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) + #endif +#endif +#if PY_VERSION_HEX < 0x030200A4 + typedef long Py_hash_t; + #define __Pyx_PyInt_FromHash_t PyInt_FromLong + #define __Pyx_PyInt_AsHash_t PyInt_AsLong +#else + #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t + #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) +#else + #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) +#endif +#if CYTHON_USE_ASYNC_SLOTS + #if PY_VERSION_HEX >= 0x030500B1 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods + #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) + #else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) + +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #define CYTHON_INLINE inline +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(U other) { return *ptr == other; }; + template bool operator !=(U other) { return *ptr != other; }; + private: + T *ptr; +}; + +#if defined(WIN32) || defined(MS_WINDOWS) + #define _USE_MATH_DEFINES +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + + +#define __PYX_ERR(f_index, lineno, Ln_error) \ +{ \ + __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ +} + +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#define __PYX_HAVE__khmer___oxli +#define __PYX_HAVE_API__khmer___oxli +#include +#include +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include +#include +#include +#include +#include +#include +#include "khmer.hh" +#include "hashtable.hh" +#include "_khmer.hh" +#include "kmer_hash.hh" +#include "partitioning.hh" +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#ifdef PYREX_WITHOUT_ASSERTIONS +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; + const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) && defined (_M_X64) + #define __Pyx_sst_abs(value) _abs64(value) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#else + #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize +#endif +#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#if PY_MAJOR_VERSION < 3 +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ + const Py_UNICODE *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen +#endif +#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) +#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) +#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) +#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +#if CYTHON_ASSUME_SAFE_MACROS +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#else +#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#endif +#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#else +#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) +#endif +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII +static int __Pyx_sys_getdefaultencoding_not_ascii; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + PyObject* ascii_chars_u = NULL; + PyObject* ascii_chars_b = NULL; + const char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + if (strcmp(default_encoding_c, "ascii") == 0) { + __Pyx_sys_getdefaultencoding_not_ascii = 0; + } else { + char ascii_chars[128]; + int c; + for (c = 0; c < 128; c++) { + ascii_chars[c] = c; + } + __Pyx_sys_getdefaultencoding_not_ascii = 1; + ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); + if (!ascii_chars_u) goto bad; + ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); + if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { + PyErr_Format( + PyExc_ValueError, + "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", + default_encoding_c); + goto bad; + } + Py_DECREF(ascii_chars_u); + Py_DECREF(ascii_chars_b); + } + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return -1; +} +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#else +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +static char* __PYX_DEFAULT_STRING_ENCODING; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; + strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + return -1; +} +#endif +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_d; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static PyObject *__pyx_empty_bytes; +static PyObject *__pyx_empty_unicode; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; + + +static const char *__pyx_f[] = { + "khmer/_oxli.pxd", + "khmer/_oxli.pyx", + "stringsource", +}; + +/*--- Type declarations ---*/ +struct __pyx_obj_5khmer_5_oxli_Component; +struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner; +struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__; + +/* "khmer/_oxli.pyx":9 + * from khmer._khmer import Nodegraph + * + * cdef class Component: # <<<<<<<<<<<<<< + * + * cdef ComponentPtr _this + */ +struct __pyx_obj_5khmer_5_oxli_Component { + PyObject_HEAD + khmer::ComponentPtr _this; +}; + + +/* "khmer/_oxli.pyx":37 + * + * + * cdef class StreamingPartitioner: # <<<<<<<<<<<<<< + * + * cdef unique_ptr[CyStreamingPartitioner] _this + */ +struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner { + PyObject_HEAD + std::unique_ptr _this; + khmer::Hashtable *_graph_ptr; +}; + + +/* "khmer/_oxli.pyx":30 + * return deref(self._this).get_n_tags() + * + * def __iter__(self): # <<<<<<<<<<<<<< + * it = deref(self._this).tags.begin() + * while it != deref(self._this).tags.end(): + */ +struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ { + PyObject_HEAD + std::set ::iterator __pyx_v_it; + struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self; +}; + + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*SetupContext)(const char*, int, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; +#ifdef WITH_THREAD + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + } +#else + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) +#endif + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ + const char* function_name); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTest.proto */ +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact); + +/* GetModuleGlobalName.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* IncludeStringH.proto */ +#include + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* CodeObjectCache.proto */ +typedef struct { + PyCodeObject* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; +}; +static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static PyCodeObject *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* None.proto */ +#include + +/* CppExceptionConversion.proto */ +#ifndef __Pyx_CppExn2PyErr +#include +#include +#include +#include +static void __Pyx_CppExn2PyErr() { + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one + else + throw; + } catch (const std::bad_alloc& exn) { + PyErr_SetString(PyExc_MemoryError, exn.what()); + } catch (const std::bad_cast& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::bad_typeid& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::domain_error& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::invalid_argument& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::ios_base::failure& exn) { + PyErr_SetString(PyExc_IOError, exn.what()); + } catch (const std::out_of_range& exn) { + PyErr_SetString(PyExc_IndexError, exn.what()); + } catch (const std::overflow_error& exn) { + PyErr_SetString(PyExc_OverflowError, exn.what()); + } catch (const std::range_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::underflow_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } + catch (...) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectCallMethod1.proto */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* CoroutineBase.proto */ +typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); +typedef struct { + PyObject_HEAD + __pyx_coroutine_body_t body; + PyObject *closure; + PyObject *exc_type; + PyObject *exc_value; + PyObject *exc_traceback; + PyObject *gi_weakreflist; + PyObject *classobj; + PyObject *yieldfrom; + PyObject *gi_name; + PyObject *gi_qualname; + PyObject *gi_modulename; + int resume_label; + char is_running; +} __pyx_CoroutineObject; +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); +static int __Pyx_Coroutine_clear(PyObject *self); +#if 1 || PY_VERSION_HEX < 0x030300B0 +static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); +#else +#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) +#endif + +/* PatchModuleWithCoroutine.proto */ +static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code); + +/* PatchGeneratorABC.proto */ +static int __Pyx_patch_abc(void); + +/* Generator.proto */ +#define __Pyx_Generator_USED +static PyTypeObject *__pyx_GeneratorType = 0; +#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) +#define __Pyx_Generator_New(body, closure, name, qualname, module_name)\ + __Pyx__Coroutine_New(__pyx_GeneratorType, body, closure, name, qualname, module_name) +static PyObject *__Pyx_Generator_Next(PyObject *self); +static int __pyx_Generator_init(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(void); + +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); + + +/* Module declarations from 'libcpp' */ + +/* Module declarations from 'libc.string' */ + +/* Module declarations from 'libcpp.string' */ + +/* Module declarations from 'libcpp.vector' */ + +/* Module declarations from 'libcpp.utility' */ + +/* Module declarations from 'libcpp.map' */ + +/* Module declarations from 'libcpp.set' */ + +/* Module declarations from 'libcpp.queue' */ + +/* Module declarations from 'libcpp.memory' */ + +/* Module declarations from 'libc.stdint' */ + +/* Module declarations from 'cython' */ + +/* Module declarations from 'libc.limits' */ + +/* Module declarations from 'khmer._oxli' */ +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_Component = 0; +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_StreamingPartitioner = 0; +static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = 0; +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ +#define __Pyx_MODULE_NAME "khmer._oxli" +int __pyx_module_is_main_khmer___oxli = 0; + +/* Implementation of 'khmer._oxli' */ +static PyObject *__pyx_builtin_MemoryError; +static PyObject *__pyx_builtin_ValueError; +static const char __pyx_k_args[] = "args"; +static const char __pyx_k_iter[] = "__iter__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_send[] = "send"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_close[] = "close"; +static const char __pyx_k_graph[] = "graph"; +static const char __pyx_k_other[] = "other"; +static const char __pyx_k_throw[] = "throw"; +static const char __pyx_k_utf_8[] = "utf-8"; +static const char __pyx_k_encode[] = "encode"; +static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_Nodegraph[] = "Nodegraph"; +static const char __pyx_k_Countgraph[] = "Countgraph"; +static const char __pyx_k_ValueError[] = "ValueError"; +static const char __pyx_k_MemoryError[] = "MemoryError"; +static const char __pyx_k_khmer__oxli[] = "khmer._oxli"; +static const char __pyx_k_khmer__khmer[] = "khmer._khmer"; +static const char __pyx_k_Component___iter[] = "Component.__iter__"; +static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; +static PyObject *__pyx_n_s_Component___iter; +static PyObject *__pyx_n_s_Countgraph; +static PyObject *__pyx_n_s_MemoryError; +static PyObject *__pyx_kp_s_Must_take_an_object_with_Hashtab; +static PyObject *__pyx_n_s_Nodegraph; +static PyObject *__pyx_n_s_ValueError; +static PyObject *__pyx_n_s_args; +static PyObject *__pyx_n_s_close; +static PyObject *__pyx_n_s_encode; +static PyObject *__pyx_n_s_graph; +static PyObject *__pyx_n_s_import; +static PyObject *__pyx_n_s_iter; +static PyObject *__pyx_n_s_khmer__khmer; +static PyObject *__pyx_n_s_khmer__oxli; +static PyObject *__pyx_n_s_main; +static PyObject *__pyx_n_s_other; +static PyObject *__pyx_n_s_send; +static PyObject *__pyx_n_s_test; +static PyObject *__pyx_n_s_throw; +static PyObject *__pyx_kp_s_utf_8; +static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ +static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tuple_; +static PyObject *__pyx_tuple__2; + +/* "khmer/_oxli.pyx":13 + * cdef ComponentPtr _this + * + * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< + * if other is not None: + * self._this.reset(other._this.get()) + */ + +/* Python wrapper */ +static int __pyx_pw_5khmer_5_oxli_9Component_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5khmer_5_oxli_9Component_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_other,0}; + PyObject* values[1] = {0}; + values[0] = (PyObject *)((struct __pyx_obj_5khmer_5_oxli_Component *)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_other); + if (value) { values[0] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 13, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_other = ((struct __pyx_obj_5khmer_5_oxli_Component *)values[0]); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 13, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.Component.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5khmer_5_oxli_Component, 1, "other", 0))) __PYX_ERR(1, 13, __pyx_L1_error) + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component___cinit__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self), __pyx_v_other); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + khmer::Component *__pyx_t_3; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "khmer/_oxli.pyx":14 + * + * def __cinit__(self, Component other=None): + * if other is not None: # <<<<<<<<<<<<<< + * self._this.reset(other._this.get()) + * else: + */ + __pyx_t_1 = (((PyObject *)__pyx_v_other) != Py_None); + __pyx_t_2 = (__pyx_t_1 != 0); + if (__pyx_t_2) { + + /* "khmer/_oxli.pyx":15 + * def __cinit__(self, Component other=None): + * if other is not None: + * self._this.reset(other._this.get()) # <<<<<<<<<<<<<< + * else: + * self._this.reset(new CyComponent()) + */ + __pyx_v_self->_this.reset(__pyx_v_other->_this.get()); + + /* "khmer/_oxli.pyx":14 + * + * def __cinit__(self, Component other=None): + * if other is not None: # <<<<<<<<<<<<<< + * self._this.reset(other._this.get()) + * else: + */ + goto __pyx_L3; + } + + /* "khmer/_oxli.pyx":17 + * self._this.reset(other._this.get()) + * else: + * self._this.reset(new CyComponent()) # <<<<<<<<<<<<<< + * + * property component_id: + */ + /*else*/ { + try { + __pyx_t_3 = new khmer::Component(); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 17, __pyx_L1_error) + } + __pyx_v_self->_this.reset(__pyx_t_3); + } + __pyx_L3:; + + /* "khmer/_oxli.pyx":13 + * cdef ComponentPtr _this + * + * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< + * if other is not None: + * self._this.reset(other._this.get()) + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("khmer._oxli.Component.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":20 + * + * property component_id: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).component_id + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12component_id_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12component_id_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli.pyx":21 + * property component_id: + * def __get__(self): + * return deref(self._this).component_id # <<<<<<<<<<<<<< + * + * property n_merges: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":20 + * + * property component_id: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).component_id + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.Component.component_id.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":24 + * + * property n_merges: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_merges() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_8n_merges_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_8n_merges_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli.pyx":25 + * property n_merges: + * def __get__(self): + * return deref(self._this).get_n_merges() # <<<<<<<<<<<<<< + * + * def __len__(self): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_merges()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":24 + * + * property n_merges: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_merges() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.Component.n_merges.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":27 + * return deref(self._this).get_n_merges() + * + * def __len__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_tags() + * + */ + +/* Python wrapper */ +static Py_ssize_t __pyx_pw_5khmer_5_oxli_9Component_3__len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_pw_5khmer_5_oxli_9Component_3__len__(PyObject *__pyx_v_self) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_2__len__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__", 0); + + /* "khmer/_oxli.pyx":28 + * + * def __len__(self): + * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< + * + * def __iter__(self): + */ + __pyx_r = (*__pyx_v_self->_this).get_n_tags(); + goto __pyx_L0; + + /* "khmer/_oxli.pyx":27 + * return deref(self._this).get_n_merges() + * + * def __len__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_tags() + * + */ + + /* function exit code */ + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ + +/* "khmer/_oxli.pyx":30 + * return deref(self._this).get_n_tags() + * + * def __iter__(self): # <<<<<<<<<<<<<< + * it = deref(self._this).tags.begin() + * while it != deref(self._this).tags.end(): + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_5__iter__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_5__iter__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_4__iter__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__iter__", 0); + __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(__pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(1, 30, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 30, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("khmer._oxli.Component.__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("None", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L6_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 30, __pyx_L1_error) + + /* "khmer/_oxli.pyx":31 + * + * def __iter__(self): + * it = deref(self._this).tags.begin() # <<<<<<<<<<<<<< + * while it != deref(self._this).tags.end(): + * yield deref(it) + */ + __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_self->_this).tags.begin(); + + /* "khmer/_oxli.pyx":32 + * def __iter__(self): + * it = deref(self._this).tags.begin() + * while it != deref(self._this).tags.end(): # <<<<<<<<<<<<<< + * yield deref(it) + * inc(it) + */ + while (1) { + __pyx_t_1 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_self->_this).tags.end()) != 0); + if (!__pyx_t_1) break; + + /* "khmer/_oxli.pyx":33 + * it = deref(self._this).tags.begin() + * while it != deref(self._this).tags.end(): + * yield deref(it) # <<<<<<<<<<<<<< + * inc(it) + * + */ + __pyx_t_2 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L6_resume_from_yield:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 33, __pyx_L1_error) + + /* "khmer/_oxli.pyx":34 + * while it != deref(self._this).tags.end(): + * yield deref(it) + * inc(it) # <<<<<<<<<<<<<< + * + * + */ + (++__pyx_cur_scope->__pyx_v_it); + } + if (1); else __pyx_cur_scope = __pyx_cur_scope; + + /* "khmer/_oxli.pyx":30 + * return deref(self._this).get_n_tags() + * + * def __iter__(self): # <<<<<<<<<<<<<< + * it = deref(self._this).tags.begin() + * while it != deref(self._this).tags.end(): + */ + + /* function exit code */ + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); __pyx_r = 0; + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":42 + * cdef CyHashtable * _graph_ptr + * + * def __cinit__(self, graph): # <<<<<<<<<<<<<< + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') + */ + +/* Python wrapper */ +static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_graph = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_graph,0}; + PyObject* values[1] = {0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 42, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + } + __pyx_v_graph = values[0]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 42, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), __pyx_v_graph); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph) { + khmer::khmer_KHashtable_Object *__pyx_v_ptr; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + khmer::Hashtable *__pyx_t_5; + khmer::StreamingPartitioner *__pyx_t_6; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "khmer/_oxli.pyx":43 + * + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< + * raise ValueError('Must take an object with Hashtable *') + * + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = (__pyx_t_3 != 0); + if (!__pyx_t_4) { + } else { + __pyx_t_1 = __pyx_t_4; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_3 = (__pyx_t_4 != 0); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + __pyx_t_3 = ((!__pyx_t_1) != 0); + if (__pyx_t_3) { + + /* "khmer/_oxli.pyx":44 + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< + * + * + */ + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(1, 44, __pyx_L1_error) + + /* "khmer/_oxli.pyx":43 + * + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< + * raise ValueError('Must take an object with Hashtable *') + * + */ + } + + /* "khmer/_oxli.pyx":47 + * + * + * cdef CyCpHashtable_Object* ptr = graph # <<<<<<<<<<<<<< + * self._graph_ptr = deref(ptr).hashtable + * + */ + __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); + + /* "khmer/_oxli.pyx":48 + * + * cdef CyCpHashtable_Object* ptr = graph + * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< + * + * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) + */ + __pyx_t_5 = (*__pyx_v_ptr).hashtable; + __pyx_v_self->_graph_ptr = __pyx_t_5; + + /* "khmer/_oxli.pyx":50 + * self._graph_ptr = deref(ptr).hashtable + * + * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) # <<<<<<<<<<<<<< + * + * def consume_sequence(self, sequence): + */ + try { + __pyx_t_6 = new khmer::StreamingPartitioner(__pyx_v_self->_graph_ptr); + } catch(...) { + try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } + __PYX_ERR(1, 50, __pyx_L1_error) + } + __pyx_v_self->_this.reset(__pyx_t_6); + + /* "khmer/_oxli.pyx":42 + * cdef CyHashtable * _graph_ptr + * + * def __cinit__(self, graph): # <<<<<<<<<<<<<< + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":52 + * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) + * + * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< + * deref(self._this).consume_sequence(sequence.encode('utf-8')) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence(PyObject *__pyx_v_self, PyObject *__pyx_v_sequence); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence(PyObject *__pyx_v_self, PyObject *__pyx_v_sequence) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("consume_sequence (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_sequence)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + std::string __pyx_t_3; + __Pyx_RefNannySetupContext("consume_sequence", 0); + + /* "khmer/_oxli.pyx":53 + * + * def consume_sequence(self, sequence): + * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< + * + * property n_components: + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 53, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + try { + (*__pyx_v_self->_this).consume_sequence(__pyx_t_3); + } catch(...) { + try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } + __PYX_ERR(1, 53, __pyx_L1_error) + } + + /* "khmer/_oxli.pyx":52 + * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) + * + * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< + * deref(self._this).consume_sequence(sequence.encode('utf-8')) + * + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.consume_sequence", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":56 + * + * property n_components: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_components() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli.pyx":57 + * property n_components: + * def __get__(self): + * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 57, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":56 + * + * property n_components: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_components() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.n_components.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { + Py_ssize_t __pyx_v_length; + char *__pyx_v_data; + std::string __pyx_r; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); + + /* "string.from_py":15 + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< + * return string(data, length) + * + */ + __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(2, 15, __pyx_L1_error) + __pyx_v_data = __pyx_t_1; + + /* "string.from_py":16 + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * return string(data, length) # <<<<<<<<<<<<<< + * + * + */ + __pyx_r = std::string(__pyx_v_data, __pyx_v_length); + goto __pyx_L0; + + /* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_Component *p; + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5khmer_5_oxli_Component *)o); + new((void*)&(p->_this)) khmer::ComponentPtr(); + if (unlikely(__pyx_pw_5khmer_5_oxli_9Component_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_Component(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_Component *p = (struct __pyx_obj_5khmer_5_oxli_Component *)o; + #if PY_VERSION_HEX >= 0x030400a1 + if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + __Pyx_call_destructor(p->_this); + (*Py_TYPE(o)->tp_free)(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_9Component_component_id(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_9Component_12component_id_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_9Component_n_merges(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_9Component_8n_merges_1__get__(o); +} + +static PyMethodDef __pyx_methods_5khmer_5_oxli_Component[] = { + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_Component[] = { + {(char *)"component_id", __pyx_getprop_5khmer_5_oxli_9Component_component_id, 0, (char *)0, 0}, + {(char *)"n_merges", __pyx_getprop_5khmer_5_oxli_9Component_n_merges, 0, (char *)0, 0}, + {0, 0, 0, 0, 0} +}; + +static PySequenceMethods __pyx_tp_as_sequence_Component = { + __pyx_pw_5khmer_5_oxli_9Component_3__len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Component = { + __pyx_pw_5khmer_5_oxli_9Component_3__len__, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyTypeObject __pyx_type_5khmer_5_oxli_Component = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.Component", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_Component), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_Component, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_Component, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Component, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + __pyx_pw_5khmer_5_oxli_9Component_5__iter__, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5khmer_5_oxli_Component, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5khmer_5_oxli_Component, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_Component, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyObject *__pyx_tp_new_5khmer_5_oxli_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *p; + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)o); + new((void*)&(p->_this)) std::unique_ptr (); + if (unlikely(__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_StreamingPartitioner(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *p = (struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)o; + #if PY_VERSION_HEX >= 0x030400a1 + if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + __Pyx_call_destructor(p->_this); + (*Py_TYPE(o)->tp_free)(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_components(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(o); +} + +static PyMethodDef __pyx_methods_5khmer_5_oxli_StreamingPartitioner[] = { + {"consume_sequence", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence, METH_O, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_StreamingPartitioner[] = { + {(char *)"n_components", __pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_components, 0, (char *)0, 0}, + {0, 0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5khmer_5_oxli_StreamingPartitioner = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.StreamingPartitioner", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_StreamingPartitioner, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5khmer_5_oxli_StreamingPartitioner, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5khmer_5_oxli_StreamingPartitioner, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_StreamingPartitioner, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *__pyx_freelist_5khmer_5_oxli___pyx_scope_struct____iter__[8]; +static int __pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__ = 0; + +static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p; + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__ > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__)))) { + o = (PyObject*)__pyx_freelist_5khmer_5_oxli___pyx_scope_struct____iter__[--__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__]; + memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__)); + (void) PyObject_INIT(o, t); + PyObject_GC_Track(o); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o); + new((void*)&(p->__pyx_v_it)) std::set ::iterator(); + return o; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct____iter__(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o; + PyObject_GC_UnTrack(o); + __Pyx_call_destructor(p->__pyx_v_it); + Py_CLEAR(p->__pyx_v_self); + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__ < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__)))) { + __pyx_freelist_5khmer_5_oxli___pyx_scope_struct____iter__[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static int __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct____iter__(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o; + if (p->__pyx_v_self) { + e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct____iter__(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o; + tmp = ((PyObject*)p->__pyx_v_self); + p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_Component *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyTypeObject __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__ = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.__pyx_scope_struct____iter__", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct____iter__, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct____iter__, /*tp_traverse*/ + __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct____iter__, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + #if PY_VERSION_HEX < 0x03020000 + { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, + #else + PyModuleDef_HEAD_INIT, + #endif + "_oxli", + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_n_s_Component___iter, __pyx_k_Component___iter, sizeof(__pyx_k_Component___iter), 0, 0, 1, 1}, + {&__pyx_n_s_Countgraph, __pyx_k_Countgraph, sizeof(__pyx_k_Countgraph), 0, 0, 1, 1}, + {&__pyx_n_s_MemoryError, __pyx_k_MemoryError, sizeof(__pyx_k_MemoryError), 0, 0, 1, 1}, + {&__pyx_kp_s_Must_take_an_object_with_Hashtab, __pyx_k_Must_take_an_object_with_Hashtab, sizeof(__pyx_k_Must_take_an_object_with_Hashtab), 0, 0, 1, 0}, + {&__pyx_n_s_Nodegraph, __pyx_k_Nodegraph, sizeof(__pyx_k_Nodegraph), 0, 0, 1, 1}, + {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, + {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, + {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, + {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, + {&__pyx_n_s_graph, __pyx_k_graph, sizeof(__pyx_k_graph), 0, 0, 1, 1}, + {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, + {&__pyx_n_s_iter, __pyx_k_iter, sizeof(__pyx_k_iter), 0, 0, 1, 1}, + {&__pyx_n_s_khmer__khmer, __pyx_k_khmer__khmer, sizeof(__pyx_k_khmer__khmer), 0, 0, 1, 1}, + {&__pyx_n_s_khmer__oxli, __pyx_k_khmer__oxli, sizeof(__pyx_k_khmer__oxli), 0, 0, 1, 1}, + {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, + {&__pyx_n_s_other, __pyx_k_other, sizeof(__pyx_k_other), 0, 0, 1, 1}, + {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, + {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, + {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, + {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, + {0, 0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 57, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 44, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitCachedConstants(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "khmer/_oxli.pyx":44 + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< + * + * + */ + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple_); + __Pyx_GIVEREF(__pyx_tuple_); + + /* "khmer/_oxli.pyx":53 + * + * def consume_sequence(self, sequence): + * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< + * + * property n_components: + */ + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__2); + __Pyx_GIVEREF(__pyx_tuple__2); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(1, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC init_oxli(void); /*proto*/ +PyMODINIT_FUNC init_oxli(void) +#else +PyMODINIT_FUNC PyInit__oxli(void); /*proto*/ +PyMODINIT_FUNC PyInit__oxli(void) +#endif +{ + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannyDeclarations + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit__oxli(void)", 0); + if (__Pyx_check_binary_version() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(1, 1, __pyx_L1_error) + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_StopAsyncIteration_USED + if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4("_oxli", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (unlikely(!__pyx_m)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(1, 1, __pyx_L1_error) + Py_INCREF(__pyx_d); + __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(1, 1, __pyx_L1_error) + #if CYTHON_COMPILING_IN_PYPY + Py_INCREF(__pyx_b); + #endif + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(1, 1, __pyx_L1_error); + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitGlobals() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + if (__pyx_module_is_main_khmer___oxli) { + if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(1, 1, __pyx_L1_error) + } + #if PY_MAJOR_VERSION >= 3 + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(1, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "khmer._oxli")) { + if (unlikely(PyDict_SetItemString(modules, "khmer._oxli", __pyx_m) < 0)) __PYX_ERR(1, 1, __pyx_L1_error) + } + } + #endif + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + /*--- Global init code ---*/ + /*--- Variable export code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 9, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_Component.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 9, __pyx_L1_error) + __pyx_ptype_5khmer_5_oxli_Component = &__pyx_type_5khmer_5_oxli_Component; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 37, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_StreamingPartitioner.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 37, __pyx_L1_error) + __pyx_ptype_5khmer_5_oxli_StreamingPartitioner = &__pyx_type_5khmer_5_oxli_StreamingPartitioner; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 30, __pyx_L1_error) + __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__.tp_print = 0; + __pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = &__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__; + /*--- Type import code ---*/ + /*--- Variable import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + if (__Pyx_patch_abc() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + + /* "khmer/_oxli.pyx":6 + * from libcpp.memory cimport unique_ptr, weak_ptr + * + * from khmer._khmer import Countgraph # <<<<<<<<<<<<<< + * from khmer._khmer import Nodegraph + * + */ + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_n_s_Countgraph); + __Pyx_GIVEREF(__pyx_n_s_Countgraph); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_Countgraph); + __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "khmer/_oxli.pyx":7 + * + * from khmer._khmer import Countgraph + * from khmer._khmer import Nodegraph # <<<<<<<<<<<<<< + * + * cdef class Component: + */ + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_n_s_Nodegraph); + __Pyx_GIVEREF(__pyx_n_s_Nodegraph); + PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Nodegraph); + __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "khmer/_oxli.pyx":1 + * import cython # <<<<<<<<<<<<<< + * from cython.operator cimport dereference as deref, preincrement as inc + * from libc.limits cimport UINT_MAX + */ + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + if (__pyx_m) { + if (__pyx_d) { + __Pyx_AddTraceback("init khmer._oxli", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + Py_DECREF(__pyx_m); __pyx_m = 0; + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init khmer._oxli"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); + if (unlikely(!result)) { + PyErr_Format(PyExc_NameError, +#if PY_MAJOR_VERSION >= 3 + "name '%U' is not defined", name); +#else + "name '%.200s' is not defined", PyString_AS_STRING(name)); +#endif + } + return result; +} + +/* RaiseDoubleKeywords */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AsString(kw_name)); + #endif +} + +/* ParseKeywords */ +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + continue; + } + name = first_kw_arg; + #if PY_MAJOR_VERSION < 3 + if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + while (*name) { + if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) + && _PyString_Eq(**name, key)) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + if ((**argname == key) || ( + (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) + && _PyString_Eq(**argname, key))) { + goto arg_passed_twice; + } + argname++; + } + } + } else + #endif + if (likely(PyUnicode_Check(key))) { + while (*name) { + int cmp = (**name == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**name, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + int cmp = (**argname == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**argname, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + argname++; + } + } + } else + goto invalid_keyword_type; + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTest */ +static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); +} +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact) +{ + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (none_allowed && obj == Py_None) return 1; + else if (exact) { + if (likely(Py_TYPE(obj) == type)) return 1; + #if PY_MAJOR_VERSION == 2 + else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + #endif + } + else { + if (likely(PyObject_TypeCheck(obj, type))) return 1; + } + __Pyx_RaiseArgumentTypeInvalid(name, obj, type); + return 0; +} + +/* GetModuleGlobalName */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + +/* PyObjectCall */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyErrFetchRestore */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} +#endif + +/* RaiseException */ + #if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + __Pyx_PyThreadState_declare + Py_XINCREF(type); + if (!value || value == Py_None) + value = NULL; + else + Py_INCREF(value); + if (!tb || tb == Py_None) + tb = NULL; + else { + Py_INCREF(tb); + if (!PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + } + if (PyType_Check(type)) { +#if CYTHON_COMPILING_IN_PYPY + if (!value) { + Py_INCREF(Py_None); + value = Py_None; + } +#endif + PyErr_NormalizeException(&type, &value, &tb); + } else { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + value = type; + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + } + __Pyx_PyThreadState_assign + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } +#if PY_VERSION_HEX >= 0x03030000 + if (cause) { +#else + if (cause && cause != Py_None) { +#endif + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = PyThreadState_GET(); + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} +#endif + +/* Import */ + static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_import; + py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); + if (!py_import) + goto bad; + #endif + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + { + #if PY_MAJOR_VERSION >= 3 + if (level == -1) { + if (strchr(__Pyx_MODULE_NAME, '.')) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(1); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, 1); + #endif + if (!module) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + #endif + if (!module) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(level); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, level); + #endif + } + } +bad: + #if PY_VERSION_HEX < 0x03030000 + Py_XDECREF(py_import); + #endif + Py_XDECREF(empty_list); + Py_XDECREF(empty_dict); + return module; +} + +/* ImportFrom */ + static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Format(PyExc_ImportError, + #if PY_MAJOR_VERSION < 3 + "cannot import name %.230s", PyString_AS_STRING(name)); + #else + "cannot import name %S", name); + #endif + } + return value; +} + +/* CodeObjectCache */ + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); +} + +/* AddTraceback */ + #include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, + 0, + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + py_code = __pyx_find_code_object(c_line ? c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + } + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value) { + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(uint64_t) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(uint64_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(uint64_t) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(uint64_t), + little, !is_unsigned); + } +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value) { + const khmer::HashIntoType neg_one = (khmer::HashIntoType) -1, const_zero = (khmer::HashIntoType) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(khmer::HashIntoType) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(khmer::HashIntoType) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::HashIntoType) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(khmer::HashIntoType), + little, !is_unsigned); + } +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case -2: + if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (long) -1; + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (long) -1; + val = __Pyx_PyInt_As_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case -2: + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int) -1; + } + } else { + int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FetchCommonType */ + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* fake_module; + PyTypeObject* cached_type = NULL; + fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); + if (!fake_module) return NULL; + Py_INCREF(fake_module); + cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + if (cached_type) { + if (!PyType_Check((PyObject*)cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", + type->tp_name); + goto bad; + } + if (cached_type->tp_basicsize != type->tp_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + type->tp_name); + goto bad; + } + } else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + } +done: + Py_DECREF(fake_module); + return cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* SwapException */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* PyCFunctionFastCall */ + #if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + PyObject *result; + int flags; + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); +} +#endif // CYTHON_FAST_PYCCALL + +/* PyFunctionFastCall */ + #if CYTHON_FAST_PYCALL +#include "frameobject.h" +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = PyThreadState_GET(); + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif // CPython < 3.6 +#endif // CYTHON_FAST_PYCALL + +/* PyObjectCallMethO */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { +#else + if (likely(PyCFunction_Check(func))) { +#endif + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif + +/* PyObjectCallMethod1 */ + static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { + PyObject *method, *result = NULL; + method = __Pyx_PyObject_GetAttrStr(obj, method_name); + if (unlikely(!method)) goto done; +#if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(method))) { + PyObject *self = PyMethod_GET_SELF(method); + if (likely(self)) { + PyObject *args; + PyObject *function = PyMethod_GET_FUNCTION(method); + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(function)) { + PyObject *args[2] = {self, arg}; + result = __Pyx_PyFunction_FastCall(function, args, 2); + goto done; + } + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(function)) { + PyObject *args[2] = {self, arg}; + result = __Pyx_PyCFunction_FastCall(function, args, 2); + goto done; + } + #endif + args = PyTuple_New(2); + if (unlikely(!args)) goto done; + Py_INCREF(self); + PyTuple_SET_ITEM(args, 0, self); + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 1, arg); + Py_INCREF(function); + Py_DECREF(method); method = NULL; + result = __Pyx_PyObject_Call(function, args, NULL); + Py_DECREF(args); + Py_DECREF(function); + return result; + } + } +#endif + result = __Pyx_PyObject_CallOneArg(method, arg); +done: + Py_XDECREF(method); + return result; +} + +/* CoroutineBase */ + #include +#include +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); +static PyObject *__Pyx_Coroutine_Close(PyObject *self); +static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); +#define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) +#if 1 || PY_VERSION_HEX < 0x030300B0 +static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { + PyObject *et, *ev, *tb; + PyObject *value = NULL; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&et, &ev, &tb); + if (!et) { + Py_XDECREF(tb); + Py_XDECREF(ev); + Py_INCREF(Py_None); + *pvalue = Py_None; + return 0; + } + if (likely(et == PyExc_StopIteration)) { + if (!ev) { + Py_INCREF(Py_None); + value = Py_None; + } +#if PY_VERSION_HEX >= 0x030300A0 + else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + Py_DECREF(ev); + } +#endif + else if (unlikely(PyTuple_Check(ev))) { + if (PyTuple_GET_SIZE(ev) >= 1) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + value = PyTuple_GET_ITEM(ev, 0); + Py_INCREF(value); +#else + value = PySequence_ITEM(ev, 0); +#endif + } else { + Py_INCREF(Py_None); + value = Py_None; + } + Py_DECREF(ev); + } + else if (!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { + value = ev; + } + if (likely(value)) { + Py_XDECREF(tb); + Py_DECREF(et); + *pvalue = value; + return 0; + } + } else if (!PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + PyErr_NormalizeException(&et, &ev, &tb); + if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + Py_XDECREF(tb); + Py_DECREF(et); +#if PY_VERSION_HEX >= 0x030300A0 + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + Py_DECREF(ev); +#else + { + PyObject* args = __Pyx_PyObject_GetAttrStr(ev, __pyx_n_s_args); + Py_DECREF(ev); + if (likely(args)) { + value = PySequence_GetItem(args, 0); + Py_DECREF(args); + } + if (unlikely(!value)) { + __Pyx_ErrRestore(NULL, NULL, NULL); + Py_INCREF(Py_None); + value = Py_None; + } + } +#endif + *pvalue = value; + return 0; +} +#endif +static CYTHON_INLINE +void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { + PyObject *exc_type = self->exc_type; + PyObject *exc_value = self->exc_value; + PyObject *exc_traceback = self->exc_traceback; + self->exc_type = NULL; + self->exc_value = NULL; + self->exc_traceback = NULL; + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_traceback); +} +static CYTHON_INLINE +int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) { + if (unlikely(gen->is_running)) { + PyErr_SetString(PyExc_ValueError, + "generator already executing"); + return 1; + } + return 0; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) { + PyObject *retval; + __Pyx_PyThreadState_declare + assert(!self->is_running); + if (unlikely(self->resume_label == 0)) { + if (unlikely(value && value != Py_None)) { + PyErr_SetString(PyExc_TypeError, + "can't send non-None value to a " + "just-started generator"); + return NULL; + } + } + if (unlikely(self->resume_label == -1)) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + __Pyx_PyThreadState_assign + if (value) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#else + if (self->exc_traceback) { + PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyFrameObject *f = tb->tb_frame; + Py_XINCREF(__pyx_tstate->frame); + assert(f->f_back == NULL); + f->f_back = __pyx_tstate->frame; + } +#endif + __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, + &self->exc_traceback); + } else { + __Pyx_Coroutine_ExceptionClear(self); + } + self->is_running = 1; + retval = self->body((PyObject *) self, value); + self->is_running = 0; + if (retval) { + __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, + &self->exc_traceback); +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#else + if (self->exc_traceback) { + PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyFrameObject *f = tb->tb_frame; + Py_CLEAR(f->f_back); + } +#endif + } else { + __Pyx_Coroutine_ExceptionClear(self); + } + return retval; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) { + if (unlikely(!retval && !PyErr_Occurred())) { + PyErr_SetNone(PyExc_StopIteration); + } + return retval; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { + PyObject *ret; + PyObject *val = NULL; + __Pyx_Coroutine_Undelegate(gen); + __Pyx_PyGen_FetchStopIterationValue(&val); + ret = __Pyx_Coroutine_SendEx(gen, val); + Py_XDECREF(val); + return ret; +} +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { + PyObject *retval; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + PyObject *yf = gen->yieldfrom; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + { + if (value == Py_None) + ret = Py_TYPE(yf)->tp_iternext(yf); + else + ret = __Pyx_PyObject_CallMethod1(yf, __pyx_n_s_send, value); + } + gen->is_running = 0; + if (likely(ret)) { + return ret; + } + retval = __Pyx_Coroutine_FinishDelegation(gen); + } else { + retval = __Pyx_Coroutine_SendEx(gen, value); + } + return __Pyx_Coroutine_MethodReturn(retval); +} +static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { + PyObject *retval = NULL; + int err = 0; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + retval = __Pyx_Coroutine_Close(yf); + if (!retval) + return -1; + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + retval = __Pyx_Coroutine_Close(yf); + if (!retval) + return -1; + } else + #endif + { + PyObject *meth; + gen->is_running = 1; + meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_close); + if (unlikely(!meth)) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_WriteUnraisable(yf); + } + PyErr_Clear(); + } else { + retval = PyObject_CallFunction(meth, NULL); + Py_DECREF(meth); + if (!retval) + err = -1; + } + gen->is_running = 0; + } + Py_XDECREF(retval); + return err; +} +static PyObject *__Pyx_Generator_Next(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + PyObject *yf = gen->yieldfrom; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Generator_Next(yf); + } else + #endif + ret = Py_TYPE(yf)->tp_iternext(yf); + gen->is_running = 0; + if (likely(ret)) { + return ret; + } + return __Pyx_Coroutine_FinishDelegation(gen); + } + return __Pyx_Coroutine_SendEx(gen, Py_None); +} +static PyObject *__Pyx_Coroutine_Close(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject *retval, *raised_exception; + PyObject *yf = gen->yieldfrom; + int err = 0; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + Py_INCREF(yf); + err = __Pyx_Coroutine_CloseIter(gen, yf); + __Pyx_Coroutine_Undelegate(gen); + Py_DECREF(yf); + } + if (err == 0) + PyErr_SetNone(PyExc_GeneratorExit); + retval = __Pyx_Coroutine_SendEx(gen, NULL); + if (retval) { + Py_DECREF(retval); + PyErr_SetString(PyExc_RuntimeError, + "generator ignored GeneratorExit"); + return NULL; + } + raised_exception = PyErr_Occurred(); + if (!raised_exception + || raised_exception == PyExc_StopIteration + || raised_exception == PyExc_GeneratorExit + || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) + || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) + { + if (raised_exception) PyErr_Clear(); + Py_INCREF(Py_None); + return Py_None; + } + return NULL; +} +static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject *typ; + PyObject *tb = NULL; + PyObject *val = NULL; + PyObject *yf = gen->yieldfrom; + if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) + return NULL; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + Py_INCREF(yf); + if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + int err = __Pyx_Coroutine_CloseIter(gen, yf); + Py_DECREF(yf); + __Pyx_Coroutine_Undelegate(gen); + if (err < 0) + return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); + goto throw_here; + } + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Coroutine_Throw(yf, args); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + ret = __Pyx_Coroutine_Throw(yf, args); + } else + #endif + { + PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_throw); + if (unlikely(!meth)) { + Py_DECREF(yf); + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + gen->is_running = 0; + return NULL; + } + PyErr_Clear(); + __Pyx_Coroutine_Undelegate(gen); + gen->is_running = 0; + goto throw_here; + } + ret = PyObject_CallObject(meth, args); + Py_DECREF(meth); + } + gen->is_running = 0; + Py_DECREF(yf); + if (!ret) { + ret = __Pyx_Coroutine_FinishDelegation(gen); + } + return __Pyx_Coroutine_MethodReturn(ret); + } +throw_here: + __Pyx_Raise(typ, val, tb, NULL); + return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); +} +static int __Pyx_Coroutine_traverse(PyObject *self, visitproc visit, void *arg) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + Py_VISIT(gen->closure); + Py_VISIT(gen->classobj); + Py_VISIT(gen->yieldfrom); + Py_VISIT(gen->exc_type); + Py_VISIT(gen->exc_value); + Py_VISIT(gen->exc_traceback); + return 0; +} +static int __Pyx_Coroutine_clear(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + Py_CLEAR(gen->closure); + Py_CLEAR(gen->classobj); + Py_CLEAR(gen->yieldfrom); + Py_CLEAR(gen->exc_type); + Py_CLEAR(gen->exc_value); + Py_CLEAR(gen->exc_traceback); + Py_CLEAR(gen->gi_name); + Py_CLEAR(gen->gi_qualname); + return 0; +} +static void __Pyx_Coroutine_dealloc(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject_GC_UnTrack(gen); + if (gen->gi_weakreflist != NULL) + PyObject_ClearWeakRefs(self); + if (gen->resume_label > 0) { + PyObject_GC_Track(self); +#if PY_VERSION_HEX >= 0x030400a1 + if (PyObject_CallFinalizerFromDealloc(self)) +#else + Py_TYPE(gen)->tp_del(self); + if (self->ob_refcnt > 0) +#endif + { + return; + } + PyObject_GC_UnTrack(self); + } + __Pyx_Coroutine_clear(self); + PyObject_GC_Del(gen); +} +static void __Pyx_Coroutine_del(PyObject *self) { + PyObject *res; + PyObject *error_type, *error_value, *error_traceback; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + __Pyx_PyThreadState_declare + if (gen->resume_label <= 0) + return ; +#if PY_VERSION_HEX < 0x030400a1 + assert(self->ob_refcnt == 0); + self->ob_refcnt = 1; +#endif + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); + res = __Pyx_Coroutine_Close(self); + if (res == NULL) + PyErr_WriteUnraisable(self); + else + Py_DECREF(res); + __Pyx_ErrRestore(error_type, error_value, error_traceback); +#if PY_VERSION_HEX < 0x030400a1 + assert(self->ob_refcnt > 0); + if (--self->ob_refcnt == 0) { + return; + } + { + Py_ssize_t refcnt = self->ob_refcnt; + _Py_NewReference(self); + self->ob_refcnt = refcnt; + } +#if CYTHON_COMPILING_IN_CPYTHON + assert(PyType_IS_GC(self->ob_type) && + _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); + _Py_DEC_REFTOTAL; +#endif +#ifdef COUNT_ALLOCS + --Py_TYPE(self)->tp_frees; + --Py_TYPE(self)->tp_allocs; +#endif +#endif +} +static PyObject * +__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self) +{ + PyObject *name = self->gi_name; + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + tmp = self->gi_name; + Py_INCREF(value); + self->gi_name = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self) +{ + PyObject *name = self->gi_qualname; + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + tmp = self->gi_qualname; + Py_INCREF(value); + self->gi_qualname = value; + Py_XDECREF(tmp); + return 0; +} +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { + __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); + if (gen == NULL) + return NULL; + gen->body = body; + gen->closure = closure; + Py_XINCREF(closure); + gen->is_running = 0; + gen->resume_label = 0; + gen->classobj = NULL; + gen->yieldfrom = NULL; + gen->exc_type = NULL; + gen->exc_value = NULL; + gen->exc_traceback = NULL; + gen->gi_weakreflist = NULL; + Py_XINCREF(qualname); + gen->gi_qualname = qualname; + Py_XINCREF(name); + gen->gi_name = name; + Py_XINCREF(module_name); + gen->gi_modulename = module_name; + PyObject_GC_Track(gen); + return gen; +} + +/* PatchModuleWithCoroutine */ + static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code) { +#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + int result; + PyObject *globals, *result_obj; + globals = PyDict_New(); if (unlikely(!globals)) goto ignore; + result = PyDict_SetItemString(globals, "_cython_coroutine_type", + #ifdef __Pyx_Coroutine_USED + (PyObject*)__pyx_CoroutineType); + #else + Py_None); + #endif + if (unlikely(result < 0)) goto ignore; + result = PyDict_SetItemString(globals, "_cython_generator_type", + #ifdef __Pyx_Generator_USED + (PyObject*)__pyx_GeneratorType); + #else + Py_None); + #endif + if (unlikely(result < 0)) goto ignore; + if (unlikely(PyDict_SetItemString(globals, "_module", module) < 0)) goto ignore; + if (unlikely(PyDict_SetItemString(globals, "__builtins__", __pyx_b) < 0)) goto ignore; + result_obj = PyRun_String(py_code, Py_file_input, globals, globals); + if (unlikely(!result_obj)) goto ignore; + Py_DECREF(result_obj); + Py_DECREF(globals); + return module; +ignore: + Py_XDECREF(globals); + PyErr_WriteUnraisable(module); + if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, "Cython module failed to patch module with custom type", 1) < 0)) { + Py_DECREF(module); + module = NULL; + } +#else + py_code++; +#endif + return module; +} + +/* PatchGeneratorABC */ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) +static PyObject* __Pyx_patch_abc_module(PyObject *module); +static PyObject* __Pyx_patch_abc_module(PyObject *module) { + module = __Pyx_Coroutine_patch_module( + module, "" +"if _cython_generator_type is not None:\n" +" try: Generator = _module.Generator\n" +" except AttributeError: pass\n" +" else: Generator.register(_cython_generator_type)\n" +"if _cython_coroutine_type is not None:\n" +" try: Coroutine = _module.Coroutine\n" +" except AttributeError: pass\n" +" else: Coroutine.register(_cython_coroutine_type)\n" + ); + return module; +} +#endif +static int __Pyx_patch_abc(void) { +#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + static int abc_patched = 0; + if (!abc_patched) { + PyObject *module; + module = PyImport_ImportModule((PY_VERSION_HEX >= 0x03030000) ? "collections.abc" : "collections"); + if (!module) { + PyErr_WriteUnraisable(NULL); + if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, + ((PY_VERSION_HEX >= 0x03030000) ? + "Cython module failed to register with collections.abc module" : + "Cython module failed to register with collections module"), 1) < 0)) { + return -1; + } + } else { + module = __Pyx_patch_abc_module(module); + abc_patched = 1; + if (unlikely(!module)) + return -1; + Py_DECREF(module); + } + module = PyImport_ImportModule("backports_abc"); + if (module) { + module = __Pyx_patch_abc_module(module); + Py_XDECREF(module); + } + if (!module) { + PyErr_Clear(); + } + } +#else + if (0) __Pyx_Coroutine_patch_module(NULL, NULL); +#endif + return 0; +} + +/* Generator */ + static PyMethodDef __pyx_Generator_methods[] = { + {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, + (char*) PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, + {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, + (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, + {"close", (PyCFunction) __Pyx_Coroutine_Close, METH_NOARGS, + (char*) PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, + {0, 0, 0, 0} +}; +static PyMemberDef __pyx_Generator_memberlist[] = { + {(char *) "gi_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, + {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, + (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, + {0, 0, 0, 0, 0} +}; +static PyGetSetDef __pyx_Generator_getsets[] = { + {(char *) "__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, + (char*) PyDoc_STR("name of the generator"), 0}, + {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, + (char*) PyDoc_STR("qualified name of the generator"), 0}, + {0, 0, 0, 0, 0} +}; +static PyTypeObject __pyx_GeneratorType_type = { + PyVarObject_HEAD_INIT(0, 0) + "generator", + sizeof(__pyx_CoroutineObject), + 0, + (destructor) __Pyx_Coroutine_dealloc, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, + 0, + (traverseproc) __Pyx_Coroutine_traverse, + 0, + 0, + offsetof(__pyx_CoroutineObject, gi_weakreflist), + 0, + (iternextfunc) __Pyx_Generator_Next, + __pyx_Generator_methods, + __pyx_Generator_memberlist, + __pyx_Generator_getsets, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, +#if PY_VERSION_HEX >= 0x030400a1 + 0, +#else + __Pyx_Coroutine_del, +#endif + 0, +#if PY_VERSION_HEX >= 0x030400a1 + __Pyx_Coroutine_del, +#endif +}; +static int __pyx_Generator_init(void) { + __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; + __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; + __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); + if (unlikely(!__pyx_GeneratorType)) { + return -1; + } + return 0; +} + +/* CheckBinaryVersion */ + static int __Pyx_check_binary_version(void) { + char ctversion[4], rtversion[4]; + PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); + PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); + if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compiletime version %s of module '%.100s' " + "does not match runtime version %s", + ctversion, __Pyx_MODULE_NAME, rtversion); + return PyErr_WarnEx(NULL, message, 1); + } + return 0; +} + +/* InitStrings */ + static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else + if (t->is_unicode | t->is_str) { + if (t->intern) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); + } else { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { +#if PY_VERSION_HEX < 0x03030000 + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; + } + } + } +#endif + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +#else + if (__Pyx_PyUnicode_READY(o) == -1) return NULL; +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (PyUnicode_IS_ASCII(o)) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif + } else +#endif +#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) + if (PyByteArray_Check(o)) { + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); + } else +#endif + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + const char *name = NULL; + PyObject *res = NULL; +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + #if PY_MAJOR_VERSION < 3 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } + #else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } + #endif +#else + res = PyNumber_Int(x); +#endif + if (res) { +#if PY_MAJOR_VERSION < 3 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(b))) { + if (sizeof(Py_ssize_t) >= sizeof(long)) + return PyInt_AS_LONG(b); + else + return PyInt_AsSsize_t(x); + } +#endif + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)b)->ob_digit; + const Py_ssize_t size = Py_SIZE(b); + if (likely(__Pyx_sst_abs(size) <= 1)) { + ival = likely(size) ? digits[0] : 0; + if (size == -1) ival = -ival; + return ival; + } else { + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { + return PyInt_FromSize_t(ival); +} + + +#endif /* Py_PYTHON_H */ diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index 5c2e97bfa0..a8f5cd3edc 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -20,6 +20,11 @@ cdef extern from "hashtable.hh" namespace "khmer": CyHashtable(WordLength) +cdef extern from "_khmer.hh": + ctypedef struct CyCpHashtable_Object "khmer::khmer_KHashtable_Object": + CyHashtable* hashtable + + cdef extern from "kmer_hash.hh" namespace "khmer": cdef cppclass Kmer: HashIntoType kmer_f @@ -49,7 +54,7 @@ cdef extern from "partitioning.hh" namespace "khmer": cdef extern from "partitioning.hh" namespace "khmer": cdef cppclass CyStreamingPartitioner "khmer::StreamingPartitioner": - StreamingPartitioner(weak_ptr[CyHashtable] graph) except +MemoryError + CyStreamingPartitioner(CyHashtable * ) except +MemoryError void consume_sequence(string&) except +MemoryError void map_tags_to_component(set[HashIntoType], ComponentPtr) diff --git a/khmer/_oxli.pyx b/khmer/_oxli.pyx index df4835dde8..cc45a5e97c 100644 --- a/khmer/_oxli.pyx +++ b/khmer/_oxli.pyx @@ -1,6 +1,10 @@ import cython from cython.operator cimport dereference as deref, preincrement as inc from libc.limits cimport UINT_MAX +from libcpp.memory cimport unique_ptr, weak_ptr + +from khmer._khmer import Countgraph +from khmer._khmer import Nodegraph cdef class Component: @@ -29,3 +33,26 @@ cdef class Component: yield deref(it) inc(it) + +cdef class StreamingPartitioner: + + cdef unique_ptr[CyStreamingPartitioner] _this + cdef CyHashtable * _graph_ptr + + def __cinit__(self, graph): + if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + raise ValueError('Must take an object with Hashtable *') + + + cdef CyCpHashtable_Object* ptr = graph + self._graph_ptr = deref(ptr).hashtable + + self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) + + def consume_sequence(self, sequence): + deref(self._this).consume_sequence(sequence.encode('utf-8')) + + property n_components: + def __get__(self): + return deref(self._this).get_n_components() + diff --git a/lib/partitioning.cc b/lib/partitioning.cc index b5d23200d6..1254b28427 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -13,13 +13,13 @@ using namespace khmer; uint64_t Component::n_created = 0; -StreamingPartitioner::StreamingPartitioner(std::weak_ptr& graph) : +StreamingPartitioner::StreamingPartitioner(Hashtable * graph) : graph(graph), _tag_density(DEFAULT_TAG_DENSITY), n_components(0) { - this->graph = graph; - if (auto graphptr = graph.lock()) { - std::vector graph_table_sizes = graphptr->get_tablesizes(); + //if (auto graphptr = graph.lock()) { + if (graph != NULL) { + std::vector graph_table_sizes = graph->get_tablesizes(); uint64_t graph_max_table_size = *std::max_element(graph_table_sizes.begin(), graph_table_sizes.end()); @@ -29,8 +29,8 @@ StreamingPartitioner::StreamingPartitioner(std::weak_ptr& graph) : // sizes proportional by the number of tags. Here, we use _tag_density-2 // because we always tag the first and last k-mers in a read. tag_component_map = std::unique_ptr( - new GuardedKmerCompMap(graphptr->ksize(), - graphptr->n_tables(), + new GuardedKmerCompMap(graph->ksize(), + graph->n_tables(), graph_max_table_size / (_tag_density-2))); } else { throw khmer_ptr_exception("Hashtable has been deleted."); @@ -54,8 +54,9 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) * &seq, with / as the difference, + as the union, and & * as the intersect operator. */ - if (auto graphptr = graph.lock()) { - KmerIterator kmers(seq.c_str(), graphptr->ksize()); + //if (auto graphptr = graph.lock()) { + if(graph != NULL) { + KmerIterator kmers(seq.c_str(), graph->ksize()); unsigned int since = 1; std::set tags; @@ -69,7 +70,7 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) // First check if we overlap any tags Kmer kmer = kmers.next(); tags.insert(kmer); //always tag the first k-mer - bool is_new_kmer = graphptr->test_and_set_bits(kmer); + bool is_new_kmer = graph->test_and_set_bits(kmer); while(!kmers.done()) { bool kmer_tagged = false; @@ -109,7 +110,7 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) since = 1; } - is_new_kmer = graphptr->test_and_set_bits(kmer); + is_new_kmer = graph->test_and_set_bits(kmer); kmer = kmers.next(); } tags.insert(kmer); // always tag the last k-mer @@ -159,7 +160,8 @@ void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, std::set& seen) { - if (auto graphptr = graph.lock()) { + //if (auto graphptr = graph.lock()) { + if (graph != NULL) { // put a 0 on the breadth queue for each element in the starting node queue std::queue breadth_q(std::deque(node_q.size(), 0)); @@ -173,7 +175,7 @@ void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, KmerFilter filter = [&] (const Kmer& n) -> bool { return set_contains(seen, n); }; - Traverser traverser(graphptr.get(), filter); + Traverser traverser(graph, filter); while(!node_q.empty()) { diff --git a/lib/partitioning.hh b/lib/partitioning.hh index 030c37f558..806de773e1 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -151,14 +151,17 @@ class StreamingPartitioner { uint32_t _tag_density; // We're not graph's owner, simply an observer. - std::weak_ptr graph; + // Unforunately our ownership policies elsewhere are a mess + Hashtable * graph; + //std::weak_ptr graph; // We should exlusively own tag_component_map. std::unique_ptr tag_component_map; + std::unique_ptr components; uint64_t n_components; public: - explicit StreamingPartitioner(std::weak_ptr& graph); + explicit StreamingPartitioner(Hashtable * graph); void consume_sequence(const std::string& seq); void map_tags_to_component(std::set tags, ComponentPtr& comp); diff --git a/setup.py b/setup.py index 4aafa3f0e1..7545086b3c 100755 --- a/setup.py +++ b/setup.py @@ -42,8 +42,11 @@ import sys from os import listdir as os_listdir from os.path import join as path_join +from os.path import splitext import shutil import subprocess +import sys +import sysconfig import tempfile from setuptools import setup @@ -120,6 +123,16 @@ def check_for_openmp(): return exit_code == 0 +def distutils_dir_name(dname): + """Returns the name of a distutils build directory""" + f = "{dirname}.{platform}-{version[0]}.{version[1]}" + return f.format(dirname=dname, + platform=sysconfig.get_platform(), + version=sys.version_info) + +def build_dir(): + return path_join("build", distutils_dir_name("temp")) + # We bundle tested versions of zlib & bzip2. To use the system zlib and bzip2 # change setup.cfg or use the `--libraries z,bz2` parameter which will make our # custom build_ext command strip out the bundled versions. @@ -127,18 +140,18 @@ def check_for_openmp(): ZLIBDIR = 'third-party/zlib' BZIP2DIR = 'third-party/bzip2' -BUILD_DEPENDS = [] +BUILD_DEPENDS = ["khmer/_khmer.hh"] BUILD_DEPENDS.extend(path_join("lib", bn + ".hh") for bn in [ "khmer", "kmer_hash", "hashtable", "counting", "hashbits", "labelhash", "hllcounter", "khmer_exception", "read_aligner", "subset", "read_parsers", - "kmer_filters", "traversal", "assembler", "alphabets"]) + "kmer_filters", "traversal", "assembler", "alphabets", "partitioning"]) -#CP_SOURCES = ["khmer/_khmer.cc"] -SOURCES = [] +SOURCES = ["khmer/_khmer.cc"] SOURCES.extend(path_join("lib", bn + ".cc") for bn in [ "read_parsers", "kmer_hash", "hashtable", "hashbits", "labelhash", "counting", "subset", "read_aligner", - "hllcounter", "traversal", "kmer_filters", "assembler", "alphabets"]) + "hllcounter", "traversal", "kmer_filters", "assembler", "alphabets", + "partitioning"]) SOURCES.extend(path_join("third-party", "smhasher", bn + ".cc") for bn in [ "MurmurHash3"]) @@ -156,7 +169,7 @@ def check_for_openmp(): EXTRA_COMPILE_ARGS.extend(['-fopenmp']) EXTRA_LINK_ARGS.extend(['-fopenmp']) -EXTENSION_MOD_DICT = \ +CP_EXTENSION_MOD_DICT = \ { "sources": SOURCES, "extra_compile_args": EXTRA_COMPILE_ARGS, @@ -166,15 +179,27 @@ def check_for_openmp(): "define_macros": [("VERSION", versioneer.get_version()), ], } -CP_EXT_MOD_DICT = dict(EXTENSION_MOD_DICT) -CP_EXT_MOD_DICT['sources'].insert(0, "khmer/_khmer.cc") -EXTENSION_MODS = [Extension("khmer._khmer", ** CP_EXT_MOD_DICT)] +EXTENSION_MODS = [Extension("khmer._khmer", ** CP_EXTENSION_MOD_DICT)] + +CY_EXTENSION_MOD_DICT = \ + { + "sources": ["khmer/_oxli.pyx"], + "extra_compile_args": EXTRA_COMPILE_ARGS, + "extra_link_args": EXTRA_LINK_ARGS, + "extra_objects": [path_join(build_dir(), splitext(p)[0]+'.o') for p in SOURCES], + "depends": [], + "language": "c++", + "define_macros": [("VERSION", versioneer.get_version()), ], + } +''' CY_EXT_MOD_DICT = dict(EXTENSION_MOD_DICT) CY_EXT_MOD_DICT['sources'].insert(0, "khmer/_oxli.pyx") +CY_EXT_MOD_DICT['sources'].insert(1, "khmer/_khmer.cc") CY_EXT_MOD_DICT['sources'].append(path_join("lib", "partitioning.cc")) CY_EXT_MOD_DICT['depends'].append(path_join("lib", "partitioning.hh")) -CY_EXTENSION_MOD = Extension("khmer._oxli", ** CY_EXT_MOD_DICT) +''' +CY_EXTENSION_MOD = Extension("khmer._oxli", ** CY_EXTENSION_MOD_DICT) EXTENSION_MODS.extend(cythonize([CY_EXTENSION_MOD])) diff --git a/tests/graph_features.py b/tests/graph_features.py new file mode 100644 index 0000000000..72c610d134 --- /dev/null +++ b/tests/graph_features.py @@ -0,0 +1,518 @@ +# -*- coding: UTF-8 -*- +# +# This file is part of khmer, https://github.com/dib-lab/khmer/, and is +# Copyright (C) 2010-2015, Michigan State University. +# Copyright (C) 2015-2016, The Regents of the University of California. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of the Michigan State University nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Contact: khmer-project@idyll.org +# pylint: disable=missing-docstring,protected-access,no-member,invalid-name + +from __future__ import print_function +from __future__ import absolute_import + +import itertools +import random + +import khmer +from khmer.khmer_args import estimate_optimal_with_K_and_f as optimal_fp +from khmer import reverse_complement as revcomp +from . import khmer_tst_utils as utils + +import pytest +import screed + + +# We just define this globally rather than in a module-level fixture, +# as we need it during parameterization and whatnot. +K = 21 + + +class Kmer(str): + + def __init__(self, value, pos=0): + self.pos = pos + + def __new__(cls, value, pos=0): + if not len(value) == K: + raise ValueError('bad k-mer length') + return str.__new__(cls, value) + + +def mutate_base(base): + if base in 'AT': + return random.choice('GC') + elif base in 'GC': + return random.choice('AT') + else: + assert False, 'bad base' + + +def mutate_sequence(sequence, N=1): + sequence = list(sequence) + positions = random.sample(range(len(sequence)), N) + + for i in positions: + sequence[i] = mutate_base(sequence[i]) + + return ''.join(sequence) + + +def mutate_position(sequence, pos): + sequence = list(sequence) + sequence[pos] = mutate_base(sequence[pos]) + return ''.join(sequence) + + +def get_random_sequence(length, exclude=None): + '''Generate a random (non-looping) nucleotide sequence. + + To be non-overlapping, the sequence should not include any repeated + length K-1 k-mers. + + Args: + exclude (str): If not None, add the k-mers from this sequence to the + seen set. + + Returns: + str: A random non-looping sequence. + ''' + + seen = set() + + def add_seen(kmer): + seen.add(kmer) + seen.add(revcomp(kmer)) + + if exclude is not None: + for pos in range(0, len(exclude) - K): + add_seen(exclude[pos:pos + K - 1]) + + seq = [random.choice('ACGT') for _ in range(K - 1)] # do first K-1 bases + add_seen(''.join(seq)) + + while(len(seq) < length): + next_base = random.choice('ACGT') + next_kmer = ''.join(seq[-K + 2:] + [next_base]) + assert len(next_kmer) == K - 1 + if (next_kmer) not in seen: + seq.append(next_base) + add_seen(next_kmer) + else: + continue + return ''.join(seq) + + +def reads(sequence, L=100, N=100): + positions = list(range(len(sequence) - L)) + for i in range(N): + start = random.choice(positions) + yield sequence[start:start + L] + + +def kmers(sequence): + for i in range(len(sequence) - K + 1): + yield sequence[i:i + K] + + +def test_mutate_sequence(): + for _ in range(100): + assert 'A' not in mutate_sequence('A' * 10, 10) + assert 'T' not in mutate_sequence('T' * 10, 10) + assert 'C' not in mutate_sequence('C' * 10, 10) + assert 'G' not in mutate_sequence('G' * 10, 10) + + +def test_mutate_position(): + assert mutate_position('AAAA', 2) in ['AACA', 'AAGA'] + assert mutate_position('TTTT', 2) in ['TTCT', 'TTGT'] + assert mutate_position('CCCC', 2) in ['CCAC', 'CCTC'] + assert mutate_position('GGGG', 2) in ['GGAG', 'GGTG'] + + +def test_reads(): + contigfile = utils.get_test_data('simple-genome.fa') + contig = list(screed.open(contigfile))[0].sequence + + for read in reads(contig): + assert read in contig + + for read in reads(contig): + assert mutate_sequence(read) not in contig + + +''' +# GRAPH STRUCTURE FIXTURES + +These fixtures emit various graph structures with their corresponding +sequences and important nodes. They take a random sequence fixture and +a graph fixture, then consume sequence and generate k-mers accordingly. + +We're using a bespoke but simple language to describe graph structures in the +docstrings of these tests. It is as follows: + + o: Node + [x:y]: Node at position in sequence + [x:y]+S: Node at position in sequence with extra base (where S in ACGT) + (Name), ([x:y] Name): Named node, named node at position + → : Edge + ~~: Tandem →o→ repeats +''' + + +@pytest.fixture(params=['simple-genome.fa']) +def known_sequence(request): + fn = utils.get_test_data(request.param) + return list(screed.open(fn))[0].sequence + + +@pytest.fixture(params=list(range(500, 1600, 500)), + ids=lambda val: '(L={0})'.format(val)) +def random_sequence(request): + + def get(exclude=None): + return get_random_sequence(request.param, exclude=exclude) + + return get + + +@pytest.fixture(params=[khmer.Nodegraph, khmer.Countgraph], + ids=['(Type=Nodegraph)', '(Type=Countgraph)']) +def graph(request): + + num_kmers = 50000 + des_fp = 0.00001 + args = optimal_fp(num_kmers, des_fp) + print('Graph Params:', args) + + return request.param(K, args.htable_size, args.num_htables) + + +def hdn_counts(sequence, graph): + '''Get the degree distribution of nodes with degree more than 2. + ''' + + hdns = {} + for kmer in kmers(sequence): + d = graph.kmer_degree(kmer) + if d > 2: + hdns[d] = hdns.get(d, 0) + 1 + + return hdns + + +@pytest.fixture +def linear_structure(request, graph, random_sequence): + '''Sets up a simple linear path graph structure. + + sequence + [0]→o→o~~o→o→[-1] + ''' + sequence = random_sequence() + graph.consume(sequence) + + # Check for false positive neighbors in our graph + # Mark as an expected failure if any are found + if hdn_counts(sequence, graph): + request.applymarker(pytest.mark.xfail) + + return graph, sequence + + +@pytest.fixture(params=[K * 2, -K * 2], + ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) +def right_tip_structure(request, graph, random_sequence): + ''' + Sets up a graph structure like so: + ([S+1:S+K]+B tip) + sequence ↗ + [0]→o→o~~o→(L)→([S:S+K] HDN)→(R)→o→o→o~~o→[-1] + + Where S is the start position of the high degreen node (HDN). + That is, it has a single branch at the Sth K-mer. + ''' + sequence = random_sequence() + S = request.param + if S < 0: + S = len(sequence) + S + # the HDN + HDN = Kmer(sequence[S:S + K], pos=S) + # left of the HDN + L = Kmer(sequence[S - 1:S - 1 + K], pos=S - 1) + # right of the HDN + R = Kmer(sequence[S + 1:S + 1 + K], pos=S + 1) + # the branch kmer + tip = Kmer(mutate_position(R, -1), + pos=R.pos) + + graph.consume(sequence) + graph.count(tip) + + # Check for false positive neighbors and mark as expected failure if found + if hdn_counts(sequence, graph) != {3: 1}: + request.applymarker(pytest.mark.xfail) + + return graph, sequence, L, HDN, R, tip + + +@pytest.fixture(params=[K * 2, -K * 2], + ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) +def right_double_fork_structure(request, linear_structure, random_sequence): + ''' + Sets up a graph structure like so: + branch + ([S+1:S+K]+B)→o~~o→o + core_sequence ↗ + [0]→o→o~~o→(L)→([S:S+K] HDN)→(R)→o→o→o~~o→[-1] + + Where S is the start position of the high degreen node (HDN) + and B is the mutated base starting the branch. + ''' + + graph, core_sequence = linear_structure + print('\nCore Len:', len(core_sequence)) + branch_sequence = random_sequence(exclude=core_sequence) + print('Branch len:', len(branch_sequence)) + + # start position of the HDN + S = request.param + if S < 0: + S = len(core_sequence) + S + # the HDN + HDN = Kmer(core_sequence[S:S + K], pos=S) + # left of the HDN + L = Kmer(core_sequence[S - 1:S - 1 + K], pos=S - 1) + # right of the HDN + R = Kmer(core_sequence[S + 1:S + 1 + K], pos=S + 1) + # the branch sequence, mutated at position S+1 + branch_start = core_sequence[:R.pos] + mutate_position(R, -1) + branch_sequence = branch_start + branch_sequence + + graph.consume(core_sequence) + graph.consume(branch_sequence) + + # Check for false positive neighbors and mark as expected failure if found + core_hdns = hdn_counts(core_sequence, graph) + branch_hdns = hdn_counts(branch_sequence, graph) + + # the core and branch sequences should each have exactly + # ONE node of degree 3 (HDN) + if core_hdns != {3: 1} or branch_hdns != {3: 1}: + print(core_hdns, branch_hdns) + request.applymarker(pytest.mark.xfail) + + return graph, core_sequence, L, HDN, R, branch_sequence + + +@pytest.fixture +def right_triple_fork_structure(request, right_double_fork_structure, + random_sequence): + ''' + Sets up a graph structure like so: + + top_branch + ([:S+1]+B)→o~~o→o + core_sequence ↗ + [0]→o→o~~o→(L)→([S:S+K] HDN)→(R)→o→o→o~~o→[-1] + ↘ + ([:S+1]+B)→o~~o→o + bottom_branch + + Where S is the start position of the high degreen node (HDN). + ''' + + graph, core_sequence, L, HDN, R, top_sequence = right_double_fork_structure + bottom_branch = random_sequence(exclude=core_sequence + top_sequence) + print(len(core_sequence), len(top_sequence), len(bottom_branch)) + + # the branch sequence, mutated at position S+1 + # choose a base not already represented at that position + bases = {'A', 'C', 'G', 'T'} + mutated = random.choice(list(bases - {R[-1], top_sequence[R.pos + K - 1]})) + + bottom_sequence = core_sequence[:HDN.pos + K] + mutated + bottom_branch + + graph.consume(bottom_sequence) + + # Check for false positive neighbors and mark as expected failure if found + core_hdns = hdn_counts(core_sequence, graph) + top_hdns = hdn_counts(top_sequence, graph) + bottom_hdns = hdn_counts(bottom_sequence, graph) + + # the core, top, and bottom sequences should each have exactly + # ONE node of degree 4 (HDN) + if not (core_hdns == top_hdns == bottom_hdns == {4: 1}): + print(core_hdns, top_hdns, bottom_hdns) + request.applymarker(pytest.mark.xfail) + + return graph, core_sequence, L, HDN, R, top_sequence, bottom_sequence + + +@pytest.fixture(params=[K * 2, -K * 2], + ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) +def left_tip_structure(request, graph, random_sequence): + ''' + Sets up a graph structure like so: + + branch + (B+[S:S+K-1] tip) + ↘ sequence + [0]→o~~o→(L)→([S:S+K] HDN)→(R)→o→o~~o→[-1] + + Where S is the start position of the HDN. + ''' + sequence = random_sequence() + S = request.param + if S < 0: + S = len(sequence) + S + tip = Kmer(mutate_position(sequence[S - 1:S - 1 + K], 0), + pos=S - 1 + K) + HDN = Kmer(sequence[S:S + K], pos=S) + L = Kmer(sequence[S - 1:S - 1 + K], pos=S - 1) + R = Kmer(sequence[S + 1:S + 1 + K], pos=S + 1) + + graph.consume(sequence) + graph.count(tip) + + # Check for false positive neighbors and mark as expected failure if found + if hdn_counts(sequence, graph) != {3: 1}: + request.applymarker(pytest.mark.xfail) + + return graph, sequence, L, HDN, R, tip + + +@pytest.fixture(params=[K * 2, -K * 2], + ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) +def left_double_fork_structure(request, linear_structure, random_sequence): + ''' + Sets up a graph structure like so: + + o→o~~o→(B+[S:S+K-1]) + ↘ core_sequence + [0]→o→o~~o→(L)→([S:S+K] HDN)→(R)→o→o→o~~o→[-1] + + Where S is the start position of the high degreen node (HDN). + ''' + + graph, core_sequence = linear_structure + branch_sequence = random_sequence(exclude=core_sequence) + + # start position of the HDN + S = request.param + if S < 0: + S = len(core_sequence) + S + # the HDN + HDN = Kmer(core_sequence[S:S + K], pos=S) + # left of the HDN + L = Kmer(core_sequence[S - 1:S - 1 + K], pos=S - 1) + # right of the HDN + R = Kmer(core_sequence[S + 1:S + 1 + K], pos=S + 1) + # the branch sequence, mutated at position 0 in L, + # whih is equivalent to the K-1 prefix of HDN prepended with a new base + branch_start = mutate_position(L, 0) + branch_sequence = branch_sequence + \ + branch_start + core_sequence[L.pos + K:] + + graph.consume(core_sequence) + graph.consume(branch_sequence) + + # Check for false positive neighbors and mark as expected failure if found + core_hdns = hdn_counts(core_sequence, graph) + branch_hdns = hdn_counts(branch_sequence, graph) + + # the core and branch sequences should each have exactly + # ONE node of degree 3 (HDN) + if not (core_hdns == branch_hdns == {3: 1}): + request.applymarker(pytest.mark.xfail) + + return graph, core_sequence, L, HDN, R, branch_sequence + + +@pytest.fixture(params=[K * 2, (-K * 2) - 2], + ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) +def snp_bubble_structure(request, linear_structure): + ''' + Sets up a graph structure resulting from a SNP (Single Nucleotide + Polymorphism). + + (HDN_L[1:]+SNP)→o~~o→(SNP+) + ↗ ↘ + o~~([S:S+K] HDN_L) ([S+K+1:S+2K+1] HDN_R)~~o + ↘ ↗ + (HDN_L[1:]+W)→o~~o~~o→(W+) + + Where S is the start position of HDN directly left of the SNP (HDN_L), + SNP is the mutated base, and W is the wildtype (original) base. + Of course, W and SNP could be interchanged here, we don't actually + know which is which ;) + + Note our parameterization: we need a bit more room from the ends, + so we bring the rightmost SNP a tad left. + ''' + + graph, wildtype_sequence = linear_structure + S = request.param + if S < 0: + S = len(wildtype_sequence) + S + snp_sequence = mutate_position(wildtype_sequence, S + K) + HDN_L = Kmer(wildtype_sequence[S:S + K], pos=S) + HDN_R = Kmer(wildtype_sequence[S + K + 1:S + 2 * K + 1], pos=S + K + 1) + + graph.consume(wildtype_sequence) + graph.consume(snp_sequence) + + # Check for false positive neighbors and mark as expected failure if found + w_hdns = hdn_counts(wildtype_sequence, graph) + snp_hdns = hdn_counts(snp_sequence, graph) + if not (w_hdns == snp_hdns == {3: 2}): + print(w_hdns, snp_hdns) + print(HDN_L, HDN_R) + print(wildtype_sequence[HDN_L.pos + K + 1]) + print(snp_sequence[HDN_L.pos + K + 1]) + request.applymarker(pytest.mark.xfail) + + return graph, wildtype_sequence, snp_sequence, HDN_L, HDN_R + + +@pytest.fixture(params=[2, 3, 4, 5, 6, 7, 8]) +def tandem_repeat_structure(request, linear_structure): + + graph, sequence = linear_structure + + tandem_repeats = sequence * request.param + graph.consume(tandem_repeats) + + if hdn_counts(tandem_repeats, graph): + request.applymarker(pytest.mark.xfail) + + return graph, sequence, tandem_repeats + + diff --git a/tests/test_assembly.py b/tests/test_assembly.py index 3c26612397..621b56ef01 100644 --- a/tests/test_assembly.py +++ b/tests/test_assembly.py @@ -51,474 +51,12 @@ import pytest import screed +from .graph_features import * +from .graph_features import K def teardown(): utils.cleanup() -# We just define this globally rather than in a module-level fixture, -# as we need it during parameterization and whatnot. -K = 21 - - -class Kmer(str): - - def __init__(self, value, pos=0): - self.pos = pos - - def __new__(cls, value, pos=0): - if not len(value) == K: - raise ValueError('bad k-mer length') - return str.__new__(cls, value) - - -def mutate_base(base): - if base in 'AT': - return random.choice('GC') - elif base in 'GC': - return random.choice('AT') - else: - assert False, 'bad base' - - -def mutate_sequence(sequence, N=1): - sequence = list(sequence) - positions = random.sample(range(len(sequence)), N) - - for i in positions: - sequence[i] = mutate_base(sequence[i]) - - return ''.join(sequence) - - -def mutate_position(sequence, pos): - sequence = list(sequence) - sequence[pos] = mutate_base(sequence[pos]) - return ''.join(sequence) - - -def get_random_sequence(length, exclude=None): - '''Generate a random (non-looping) nucleotide sequence. - - To be non-overlapping, the sequence should not include any repeated - length K-1 k-mers. - - Args: - exclude (str): If not None, add the k-mers from this sequence to the - seen set. - - Returns: - str: A random non-looping sequence. - ''' - - seen = set() - - def add_seen(kmer): - seen.add(kmer) - seen.add(revcomp(kmer)) - - if exclude is not None: - for pos in range(0, len(exclude) - K): - add_seen(exclude[pos:pos + K - 1]) - - seq = [random.choice('ACGT') for _ in range(K - 1)] # do first K-1 bases - add_seen(''.join(seq)) - - while(len(seq) < length): - next_base = random.choice('ACGT') - next_kmer = ''.join(seq[-K + 2:] + [next_base]) - assert len(next_kmer) == K - 1 - if (next_kmer) not in seen: - seq.append(next_base) - add_seen(next_kmer) - else: - continue - return ''.join(seq) - - -def reads(sequence, L=100, N=100): - positions = list(range(len(sequence) - L)) - for i in range(N): - start = random.choice(positions) - yield sequence[start:start + L] - - -def kmers(sequence): - for i in range(len(sequence) - K + 1): - yield sequence[i:i + K] - - -def test_mutate_sequence(): - for _ in range(100): - assert 'A' not in mutate_sequence('A' * 10, 10) - assert 'T' not in mutate_sequence('T' * 10, 10) - assert 'C' not in mutate_sequence('C' * 10, 10) - assert 'G' not in mutate_sequence('G' * 10, 10) - - -def test_mutate_position(): - assert mutate_position('AAAA', 2) in ['AACA', 'AAGA'] - assert mutate_position('TTTT', 2) in ['TTCT', 'TTGT'] - assert mutate_position('CCCC', 2) in ['CCAC', 'CCTC'] - assert mutate_position('GGGG', 2) in ['GGAG', 'GGTG'] - - -def test_reads(): - contigfile = utils.get_test_data('simple-genome.fa') - contig = list(screed.open(contigfile))[0].sequence - - for read in reads(contig): - assert read in contig - - for read in reads(contig): - assert mutate_sequence(read) not in contig - - -''' -# GRAPH STRUCTURE FIXTURES - -These fixtures emit various graph structures with their corresponding -sequences and important nodes. They take a random sequence fixture and -a graph fixture, then consume sequence and generate k-mers accordingly. - -We're using a bespoke but simple language to describe graph structures in the -docstrings of these tests. It is as follows: - - o: Node - [x:y]: Node at position in sequence - [x:y]+S: Node at position in sequence with extra base (where S in ACGT) - (Name), ([x:y] Name): Named node, named node at position - → : Edge - ~~: Tandem →o→ repeats -''' - - -@pytest.fixture(params=['simple-genome.fa']) -def known_sequence(request): - fn = utils.get_test_data(request.param) - return list(screed.open(fn))[0].sequence - - -@pytest.fixture(params=list(range(500, 1600, 500)), - ids=lambda val: '(L={0})'.format(val)) -def random_sequence(request): - - def get(exclude=None): - return get_random_sequence(request.param, exclude=exclude) - - return get - - -@pytest.fixture(params=[khmer.Nodegraph, khmer.Countgraph], - ids=['(Type=Nodegraph)', '(Type=Countgraph)']) -def graph(request): - - num_kmers = 50000 - des_fp = 0.00001 - args = optimal_fp(num_kmers, des_fp) - print('Graph Params:', args) - - return request.param(K, args.htable_size, args.num_htables) - - -def hdn_counts(sequence, graph): - '''Get the degree distribution of nodes with degree more than 2. - ''' - - hdns = {} - for kmer in kmers(sequence): - d = graph.kmer_degree(kmer) - if d > 2: - hdns[d] = hdns.get(d, 0) + 1 - - return hdns - - -@pytest.fixture -def linear_structure(request, graph, random_sequence): - '''Sets up a simple linear path graph structure. - - sequence - [0]→o→o~~o→o→[-1] - ''' - sequence = random_sequence() - graph.consume(sequence) - - # Check for false positive neighbors in our graph - # Mark as an expected failure if any are found - if hdn_counts(sequence, graph): - request.applymarker(pytest.mark.xfail) - - return graph, sequence - - -@pytest.fixture(params=[K * 2, -K * 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def right_tip_structure(request, graph, random_sequence): - ''' - Sets up a graph structure like so: - ([S+1:S+K]+B tip) - sequence ↗ - [0]→o→o~~o→(L)→([S:S+K] HDN)→(R)→o→o→o~~o→[-1] - - Where S is the start position of the high degreen node (HDN). - That is, it has a single branch at the Sth K-mer. - ''' - sequence = random_sequence() - S = request.param - if S < 0: - S = len(sequence) + S - # the HDN - HDN = Kmer(sequence[S:S + K], pos=S) - # left of the HDN - L = Kmer(sequence[S - 1:S - 1 + K], pos=S - 1) - # right of the HDN - R = Kmer(sequence[S + 1:S + 1 + K], pos=S + 1) - # the branch kmer - tip = Kmer(mutate_position(R, -1), - pos=R.pos) - - graph.consume(sequence) - graph.count(tip) - - # Check for false positive neighbors and mark as expected failure if found - if hdn_counts(sequence, graph) != {3: 1}: - request.applymarker(pytest.mark.xfail) - - return graph, sequence, L, HDN, R, tip - - -@pytest.fixture(params=[K * 2, -K * 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def right_double_fork_structure(request, linear_structure, random_sequence): - ''' - Sets up a graph structure like so: - branch - ([S+1:S+K]+B)→o~~o→o - core_sequence ↗ - [0]→o→o~~o→(L)→([S:S+K] HDN)→(R)→o→o→o~~o→[-1] - - Where S is the start position of the high degreen node (HDN) - and B is the mutated base starting the branch. - ''' - - graph, core_sequence = linear_structure - print('\nCore Len:', len(core_sequence)) - branch_sequence = random_sequence(exclude=core_sequence) - print('Branch len:', len(branch_sequence)) - - # start position of the HDN - S = request.param - if S < 0: - S = len(core_sequence) + S - # the HDN - HDN = Kmer(core_sequence[S:S + K], pos=S) - # left of the HDN - L = Kmer(core_sequence[S - 1:S - 1 + K], pos=S - 1) - # right of the HDN - R = Kmer(core_sequence[S + 1:S + 1 + K], pos=S + 1) - # the branch sequence, mutated at position S+1 - branch_start = core_sequence[:R.pos] + mutate_position(R, -1) - branch_sequence = branch_start + branch_sequence - - graph.consume(core_sequence) - graph.consume(branch_sequence) - - # Check for false positive neighbors and mark as expected failure if found - core_hdns = hdn_counts(core_sequence, graph) - branch_hdns = hdn_counts(branch_sequence, graph) - - # the core and branch sequences should each have exactly - # ONE node of degree 3 (HDN) - if core_hdns != {3: 1} or branch_hdns != {3: 1}: - print(core_hdns, branch_hdns) - request.applymarker(pytest.mark.xfail) - - return graph, core_sequence, L, HDN, R, branch_sequence - - -@pytest.fixture -def right_triple_fork_structure(request, right_double_fork_structure, - random_sequence): - ''' - Sets up a graph structure like so: - - top_branch - ([:S+1]+B)→o~~o→o - core_sequence ↗ - [0]→o→o~~o→(L)→([S:S+K] HDN)→(R)→o→o→o~~o→[-1] - ↘ - ([:S+1]+B)→o~~o→o - bottom_branch - - Where S is the start position of the high degreen node (HDN). - ''' - - graph, core_sequence, L, HDN, R, top_sequence = right_double_fork_structure - bottom_branch = random_sequence(exclude=core_sequence + top_sequence) - print(len(core_sequence), len(top_sequence), len(bottom_branch)) - - # the branch sequence, mutated at position S+1 - # choose a base not already represented at that position - bases = {'A', 'C', 'G', 'T'} - mutated = random.choice(list(bases - {R[-1], top_sequence[R.pos + K - 1]})) - - bottom_sequence = core_sequence[:HDN.pos + K] + mutated + bottom_branch - - graph.consume(bottom_sequence) - - # Check for false positive neighbors and mark as expected failure if found - core_hdns = hdn_counts(core_sequence, graph) - top_hdns = hdn_counts(top_sequence, graph) - bottom_hdns = hdn_counts(bottom_sequence, graph) - - # the core, top, and bottom sequences should each have exactly - # ONE node of degree 4 (HDN) - if not (core_hdns == top_hdns == bottom_hdns == {4: 1}): - print(core_hdns, top_hdns, bottom_hdns) - request.applymarker(pytest.mark.xfail) - - return graph, core_sequence, L, HDN, R, top_sequence, bottom_sequence - - -@pytest.fixture(params=[K * 2, -K * 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def left_tip_structure(request, graph, random_sequence): - ''' - Sets up a graph structure like so: - - branch - (B+[S:S+K-1] tip) - ↘ sequence - [0]→o~~o→(L)→([S:S+K] HDN)→(R)→o→o~~o→[-1] - - Where S is the start position of the HDN. - ''' - sequence = random_sequence() - S = request.param - if S < 0: - S = len(sequence) + S - tip = Kmer(mutate_position(sequence[S - 1:S - 1 + K], 0), - pos=S - 1 + K) - HDN = Kmer(sequence[S:S + K], pos=S) - L = Kmer(sequence[S - 1:S - 1 + K], pos=S - 1) - R = Kmer(sequence[S + 1:S + 1 + K], pos=S + 1) - - graph.consume(sequence) - graph.count(tip) - - # Check for false positive neighbors and mark as expected failure if found - if hdn_counts(sequence, graph) != {3: 1}: - request.applymarker(pytest.mark.xfail) - - return graph, sequence, L, HDN, R, tip - - -@pytest.fixture(params=[K * 2, -K * 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def left_double_fork_structure(request, linear_structure, random_sequence): - ''' - Sets up a graph structure like so: - - o→o~~o→(B+[S:S+K-1]) - ↘ core_sequence - [0]→o→o~~o→(L)→([S:S+K] HDN)→(R)→o→o→o~~o→[-1] - - Where S is the start position of the high degreen node (HDN). - ''' - - graph, core_sequence = linear_structure - branch_sequence = random_sequence(exclude=core_sequence) - - # start position of the HDN - S = request.param - if S < 0: - S = len(core_sequence) + S - # the HDN - HDN = Kmer(core_sequence[S:S + K], pos=S) - # left of the HDN - L = Kmer(core_sequence[S - 1:S - 1 + K], pos=S - 1) - # right of the HDN - R = Kmer(core_sequence[S + 1:S + 1 + K], pos=S + 1) - # the branch sequence, mutated at position 0 in L, - # whih is equivalent to the K-1 prefix of HDN prepended with a new base - branch_start = mutate_position(L, 0) - branch_sequence = branch_sequence + \ - branch_start + core_sequence[L.pos + K:] - - graph.consume(core_sequence) - graph.consume(branch_sequence) - - # Check for false positive neighbors and mark as expected failure if found - core_hdns = hdn_counts(core_sequence, graph) - branch_hdns = hdn_counts(branch_sequence, graph) - - # the core and branch sequences should each have exactly - # ONE node of degree 3 (HDN) - if not (core_hdns == branch_hdns == {3: 1}): - request.applymarker(pytest.mark.xfail) - - return graph, core_sequence, L, HDN, R, branch_sequence - - -@pytest.fixture(params=[K * 2, (-K * 2) - 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def snp_bubble_structure(request, linear_structure): - ''' - Sets up a graph structure resulting from a SNP (Single Nucleotide - Polymorphism). - - (HDN_L[1:]+SNP)→o~~o→(SNP+) - ↗ ↘ - o~~([S:S+K] HDN_L) ([S+K+1:S+2K+1] HDN_R)~~o - ↘ ↗ - (HDN_L[1:]+W)→o~~o~~o→(W+) - - Where S is the start position of HDN directly left of the SNP (HDN_L), - SNP is the mutated base, and W is the wildtype (original) base. - Of course, W and SNP could be interchanged here, we don't actually - know which is which ;) - - Note our parameterization: we need a bit more room from the ends, - so we bring the rightmost SNP a tad left. - ''' - - graph, wildtype_sequence = linear_structure - S = request.param - if S < 0: - S = len(wildtype_sequence) + S - snp_sequence = mutate_position(wildtype_sequence, S + K) - HDN_L = Kmer(wildtype_sequence[S:S + K], pos=S) - HDN_R = Kmer(wildtype_sequence[S + K + 1:S + 2 * K + 1], pos=S + K + 1) - - graph.consume(wildtype_sequence) - graph.consume(snp_sequence) - - # Check for false positive neighbors and mark as expected failure if found - w_hdns = hdn_counts(wildtype_sequence, graph) - snp_hdns = hdn_counts(snp_sequence, graph) - if not (w_hdns == snp_hdns == {3: 2}): - print(w_hdns, snp_hdns) - print(HDN_L, HDN_R) - print(wildtype_sequence[HDN_L.pos + K + 1]) - print(snp_sequence[HDN_L.pos + K + 1]) - request.applymarker(pytest.mark.xfail) - - return graph, wildtype_sequence, snp_sequence, HDN_L, HDN_R - - -@pytest.fixture(params=[2, 3, 4, 5, 6, 7, 8]) -def tandem_repeat_structure(request, linear_structure): - - graph, sequence = linear_structure - - tandem_repeats = sequence * request.param - graph.consume(tandem_repeats) - - if hdn_counts(tandem_repeats, graph): - request.applymarker(pytest.mark.xfail) - - return graph, sequence, tandem_repeats - class TestNonBranching: diff --git a/tests/test_cython_oxli.py b/tests/test_cython_oxli.py new file mode 100644 index 0000000000..125ee3767f --- /dev/null +++ b/tests/test_cython_oxli.py @@ -0,0 +1,43 @@ +from __future__ import print_function +from __future__ import absolute_import + +import itertools +import random + +import khmer +from khmer import _oxli +from khmer.khmer_args import estimate_optimal_with_K_and_f as optimal_fp +from khmer import reverse_complement as revcomp +from . import khmer_tst_utils as utils +from .graph_features import * + +import pytest +import screed + + +def teardown(): + utils.cleanup() + +class TestStreamingPartitionerBasic: + + def test_one_component(self, known_sequence): + inpath = utils.get_test_data('random-20-a.fa') + + cg = khmer.Countgraph(K, 1e5, 4) + sp = _oxli.StreamingPartitioner(cg) + sp.consume_sequence(known_sequence) + + assert sp.n_components == 1 + + def test_two_components(self, random_sequence): + comp1 = random_sequence() + comp2 = random_sequence(exclude=comp1) + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = _oxli.StreamingPartitioner(cg) + + sp.consume_sequence(comp1) + assert sp.n_components == 1 + + sp.consume_sequence(comp2) + assert sp.n_components == 2 From 1967b746bdc963e270122f6e60fd98f548230ac0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 6 Nov 2016 02:32:55 -0800 Subject: [PATCH 006/185] Expose underlying StreamingPartitioner data --- khmer/_oxli.cpp | 455 ++++++++++++++++++++++++++++++++------ khmer/_oxli.pxd | 15 +- khmer/_oxli.pyx | 27 ++- lib/partitioning.cc | 64 +++++- lib/partitioning.hh | 28 ++- tests/test_cython_oxli.py | 13 ++ 6 files changed, 528 insertions(+), 74 deletions(-) diff --git a/khmer/_oxli.cpp b/khmer/_oxli.cpp index 5f4a4e281b..5c6ecf8f60 100644 --- a/khmer/_oxli.cpp +++ b/khmer/_oxli.cpp @@ -6,7 +6,7 @@ "define_macros": [ [ "VERSION", - "2.0+520.g9f6c196.dirty" + "2.0+521.gb3c6a3d.dirty" ] ], "depends": [ @@ -713,7 +713,7 @@ struct __pyx_obj_5khmer_5_oxli_Component { }; -/* "khmer/_oxli.pyx":37 +/* "khmer/_oxli.pyx":43 * * * cdef class StreamingPartitioner: # <<<<<<<<<<<<<< @@ -723,6 +723,8 @@ struct __pyx_obj_5khmer_5_oxli_Component { struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner { PyObject_HEAD std::unique_ptr _this; + std::weak_ptr _components; + std::weak_ptr _tag_component_map; khmer::Hashtable *_graph_ptr; }; @@ -840,9 +842,6 @@ static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, const char *name, int exact); -/* GetModuleGlobalName.proto */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); - /* PyObjectCall.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); @@ -850,6 +849,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg #define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) #endif +/* GetModuleGlobalName.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); + /* PyThreadStateGet.proto */ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; @@ -1083,6 +1085,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); static PyTypeObject *__pyx_ptype_5khmer_5_oxli_Component = 0; static PyTypeObject *__pyx_ptype_5khmer_5_oxli_StreamingPartitioner = 0; static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = 0; +static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_build_component(khmer::ComponentPtr); /*proto*/ static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ #define __Pyx_MODULE_NAME "khmer._oxli" int __pyx_module_is_main_khmer___oxli = 0; @@ -1137,12 +1140,16 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5khmer_5_oxli_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tuple_; static PyObject *__pyx_tuple__2; +static PyObject *__pyx_tuple__3; +static PyObject *__pyx_tuple__4; /* "khmer/_oxli.pyx":13 * cdef ComponentPtr _this @@ -1601,7 +1608,75 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj return __pyx_r; } -/* "khmer/_oxli.pyx":42 +/* "khmer/_oxli.pyx":37 + * + * + * cdef Component build_component(ComponentPtr ptr): # <<<<<<<<<<<<<< + * cdef Component comp = Component() + * comp._this.reset(ptr.get()) + */ + +static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_build_component(khmer::ComponentPtr __pyx_v_ptr) { + struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_comp = 0; + struct __pyx_obj_5khmer_5_oxli_Component *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("build_component", 0); + + /* "khmer/_oxli.pyx":38 + * + * cdef Component build_component(ComponentPtr ptr): + * cdef Component comp = Component() # <<<<<<<<<<<<<< + * comp._this.reset(ptr.get()) + * return comp + */ + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_Component), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_comp = ((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "khmer/_oxli.pyx":39 + * cdef Component build_component(ComponentPtr ptr): + * cdef Component comp = Component() + * comp._this.reset(ptr.get()) # <<<<<<<<<<<<<< + * return comp + * + */ + __pyx_v_comp->_this.reset(__pyx_v_ptr.get()); + + /* "khmer/_oxli.pyx":40 + * cdef Component comp = Component() + * comp._this.reset(ptr.get()) + * return comp # <<<<<<<<<<<<<< + * + * + */ + __Pyx_XDECREF(((PyObject *)__pyx_r)); + __Pyx_INCREF(((PyObject *)__pyx_v_comp)); + __pyx_r = __pyx_v_comp; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":37 + * + * + * cdef Component build_component(ComponentPtr ptr): # <<<<<<<<<<<<<< + * cdef Component comp = Component() + * comp._this.reset(ptr.get()) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.build_component", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_comp); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":51 * cdef CyHashtable * _graph_ptr * * def __cinit__(self, graph): # <<<<<<<<<<<<<< @@ -1634,7 +1709,7 @@ static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__ else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 42, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 51, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { goto __pyx_L5_argtuple_error; @@ -1645,7 +1720,7 @@ static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__ } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 42, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 51, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -1670,16 +1745,16 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ khmer::StreamingPartitioner *__pyx_t_6; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "khmer/_oxli.pyx":43 + /* "khmer/_oxli.pyx":52 * * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< * raise ValueError('Must take an object with Hashtable *') * */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 43, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 43, __pyx_L1_error) + __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 52, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_4 = (__pyx_t_3 != 0); if (!__pyx_t_4) { @@ -1687,9 +1762,9 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_1 = __pyx_t_4; goto __pyx_L4_bool_binop_done; } - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 43, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 43, __pyx_L1_error) + __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 52, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_3 = (__pyx_t_4 != 0); __pyx_t_1 = __pyx_t_3; @@ -1697,20 +1772,20 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_3 = ((!__pyx_t_1) != 0); if (__pyx_t_3) { - /* "khmer/_oxli.pyx":44 + /* "khmer/_oxli.pyx":53 * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< * * */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 44, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 53, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 44, __pyx_L1_error) + __PYX_ERR(1, 53, __pyx_L1_error) - /* "khmer/_oxli.pyx":43 + /* "khmer/_oxli.pyx":52 * * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< @@ -1719,7 +1794,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ } - /* "khmer/_oxli.pyx":47 + /* "khmer/_oxli.pyx":56 * * * cdef CyCpHashtable_Object* ptr = graph # <<<<<<<<<<<<<< @@ -1728,7 +1803,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - /* "khmer/_oxli.pyx":48 + /* "khmer/_oxli.pyx":57 * * cdef CyCpHashtable_Object* ptr = graph * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< @@ -1738,7 +1813,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_5 = (*__pyx_v_ptr).hashtable; __pyx_v_self->_graph_ptr = __pyx_t_5; - /* "khmer/_oxli.pyx":50 + /* "khmer/_oxli.pyx":59 * self._graph_ptr = deref(ptr).hashtable * * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) # <<<<<<<<<<<<<< @@ -1749,11 +1824,11 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_6 = new khmer::StreamingPartitioner(__pyx_v_self->_graph_ptr); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 50, __pyx_L1_error) + __PYX_ERR(1, 59, __pyx_L1_error) } __pyx_v_self->_this.reset(__pyx_t_6); - /* "khmer/_oxli.pyx":42 + /* "khmer/_oxli.pyx":51 * cdef CyHashtable * _graph_ptr * * def __cinit__(self, graph): # <<<<<<<<<<<<<< @@ -1773,7 +1848,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ return __pyx_r; } -/* "khmer/_oxli.pyx":52 +/* "khmer/_oxli.pyx":61 * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) * * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< @@ -1802,28 +1877,28 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence std::string __pyx_t_3; __Pyx_RefNannySetupContext("consume_sequence", 0); - /* "khmer/_oxli.pyx":53 + /* "khmer/_oxli.pyx":62 * * def consume_sequence(self, sequence): * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< * - * property n_components: + * def get_tag_component(self, kmer): */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 53, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 62, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 53, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 62, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 53, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 62, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; try { (*__pyx_v_self->_this).consume_sequence(__pyx_t_3); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 53, __pyx_L1_error) + __PYX_ERR(1, 62, __pyx_L1_error) } - /* "khmer/_oxli.pyx":52 + /* "khmer/_oxli.pyx":61 * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) * * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< @@ -1845,7 +1920,235 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence return __pyx_r; } -/* "khmer/_oxli.pyx":56 +/* "khmer/_oxli.pyx":64 + * deref(self._this).consume_sequence(sequence.encode('utf-8')) + * + * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< + * cdef ComponentPtr comp + * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("get_tag_component (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_component(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { + khmer::ComponentPtr __pyx_v_comp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + std::string __pyx_t_3; + int __pyx_t_4; + __Pyx_RefNannySetupContext("get_tag_component", 0); + + /* "khmer/_oxli.pyx":66 + * def get_tag_component(self, kmer): + * cdef ComponentPtr comp + * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if comp == NULL: + * return None + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_comp = (*__pyx_v_self->_this).get_tag_component(__pyx_t_3); + + /* "khmer/_oxli.pyx":67 + * cdef ComponentPtr comp + * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if comp == NULL: # <<<<<<<<<<<<<< + * return None + * else: + */ + __pyx_t_4 = ((__pyx_v_comp == NULL) != 0); + if (__pyx_t_4) { + + /* "khmer/_oxli.pyx":68 + * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if comp == NULL: + * return None # <<<<<<<<<<<<<< + * else: + * return build_component(comp) + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_None); + __pyx_r = Py_None; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":67 + * cdef ComponentPtr comp + * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if comp == NULL: # <<<<<<<<<<<<<< + * return None + * else: + */ + } + + /* "khmer/_oxli.pyx":70 + * return None + * else: + * return build_component(comp) # <<<<<<<<<<<<<< + * + * def get_nearest_component(self, kmer): + */ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_build_component(__pyx_v_comp)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 70, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "khmer/_oxli.pyx":64 + * deref(self._this).consume_sequence(sequence.encode('utf-8')) + * + * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< + * cdef ComponentPtr comp + * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.get_tag_component", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":72 + * return build_component(comp) + * + * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< + * cdef ComponentPtr comp + * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("get_nearest_component (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_component(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { + khmer::ComponentPtr __pyx_v_comp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + std::string __pyx_t_3; + int __pyx_t_4; + __Pyx_RefNannySetupContext("get_nearest_component", 0); + + /* "khmer/_oxli.pyx":74 + * def get_nearest_component(self, kmer): + * cdef ComponentPtr comp + * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if comp == NULL: + * return None + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 74, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 74, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 74, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_comp = (*__pyx_v_self->_this).get_nearest_component(__pyx_t_3); + + /* "khmer/_oxli.pyx":75 + * cdef ComponentPtr comp + * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if comp == NULL: # <<<<<<<<<<<<<< + * return None + * else: + */ + __pyx_t_4 = ((__pyx_v_comp == NULL) != 0); + if (__pyx_t_4) { + + /* "khmer/_oxli.pyx":76 + * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if comp == NULL: + * return None # <<<<<<<<<<<<<< + * else: + * return build_component(comp) + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_None); + __pyx_r = Py_None; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":75 + * cdef ComponentPtr comp + * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if comp == NULL: # <<<<<<<<<<<<<< + * return None + * else: + */ + } + + /* "khmer/_oxli.pyx":78 + * return None + * else: + * return build_component(comp) # <<<<<<<<<<<<<< + * + * property n_components: + */ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_build_component(__pyx_v_comp)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "khmer/_oxli.pyx":72 + * return build_component(comp) + * + * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< + * cdef ComponentPtr comp + * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.get_nearest_component", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":81 * * property n_components: * def __get__(self): # <<<<<<<<<<<<<< @@ -1872,20 +2175,20 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - /* "khmer/_oxli.pyx":57 + /* "khmer/_oxli.pyx":82 * property n_components: * def __get__(self): * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 57, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 82, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":56 + /* "khmer/_oxli.pyx":81 * * property n_components: * def __get__(self): # <<<<<<<<<<<<<< @@ -2091,6 +2394,8 @@ static PyObject *__pyx_tp_new_5khmer_5_oxli_StreamingPartitioner(PyTypeObject *t if (unlikely(!o)) return 0; p = ((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)o); new((void*)&(p->_this)) std::unique_ptr (); + new((void*)&(p->_components)) std::weak_ptr (); + new((void*)&(p->_tag_component_map)) std::weak_ptr (); if (unlikely(__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(o, a, k) < 0)) goto bad; return o; bad: @@ -2106,6 +2411,8 @@ static void __pyx_tp_dealloc_5khmer_5_oxli_StreamingPartitioner(PyObject *o) { } #endif __Pyx_call_destructor(p->_this); + __Pyx_call_destructor(p->_components); + __Pyx_call_destructor(p->_tag_component_map); (*Py_TYPE(o)->tp_free)(o); } @@ -2115,6 +2422,8 @@ static PyObject *__pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_components static PyMethodDef __pyx_methods_5khmer_5_oxli_StreamingPartitioner[] = { {"consume_sequence", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence, METH_O, 0}, + {"get_tag_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5get_tag_component, METH_O, 0}, + {"get_nearest_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_nearest_component, METH_O, 0}, {0, 0, 0, 0} }; @@ -2335,8 +2644,8 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 57, __pyx_L1_error) - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 44, __pyx_L1_error) + __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 64, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 53, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -2346,27 +2655,49 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "khmer/_oxli.pyx":44 + /* "khmer/_oxli.pyx":53 * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< * * */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 44, __pyx_L1_error) + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 53, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); - /* "khmer/_oxli.pyx":53 + /* "khmer/_oxli.pyx":62 * * def consume_sequence(self, sequence): * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< * - * property n_components: + * def get_tag_component(self, kmer): */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 53, __pyx_L1_error) + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 62, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); + + /* "khmer/_oxli.pyx":66 + * def get_tag_component(self, kmer): + * cdef ComponentPtr comp + * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if comp == NULL: + * return None + */ + __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__3); + __Pyx_GIVEREF(__pyx_tuple__3); + + /* "khmer/_oxli.pyx":74 + * def get_nearest_component(self, kmer): + * cdef ComponentPtr comp + * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if comp == NULL: + * return None + */ + __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 74, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__4); + __Pyx_GIVEREF(__pyx_tuple__4); __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -2470,9 +2801,9 @@ PyMODINIT_FUNC PyInit__oxli(void) __pyx_type_5khmer_5_oxli_Component.tp_print = 0; if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 9, __pyx_L1_error) __pyx_ptype_5khmer_5_oxli_Component = &__pyx_type_5khmer_5_oxli_Component; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 37, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 43, __pyx_L1_error) __pyx_type_5khmer_5_oxli_StreamingPartitioner.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 37, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 43, __pyx_L1_error) __pyx_ptype_5khmer_5_oxli_StreamingPartitioner = &__pyx_type_5khmer_5_oxli_StreamingPartitioner; if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 30, __pyx_L1_error) __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__.tp_print = 0; @@ -2769,26 +3100,8 @@ static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, in return 0; } -/* GetModuleGlobalName */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS - result = PyDict_GetItem(__pyx_d, name); - if (likely(result)) { - Py_INCREF(result); - } else { -#else - result = PyObject_GetItem(__pyx_d, name); - if (!result) { - PyErr_Clear(); -#endif - result = __Pyx_GetBuiltinName(name); - } - return result; -} - /* PyObjectCall */ - #if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { PyObject *result; ternaryfunc call = func->ob_type->tp_call; @@ -2807,6 +3120,24 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg } #endif +/* GetModuleGlobalName */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + /* PyErrFetchRestore */ #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index a8f5cd3edc..a498981811 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -39,6 +39,7 @@ cdef extern from "kmer_hash.hh" namespace "khmer": cdef extern from "partitioning.hh" namespace "khmer": + cdef cppclass CyComponent "khmer::Component": const uint64_t component_id set[HashIntoType] tags @@ -50,9 +51,15 @@ cdef extern from "partitioning.hh" namespace "khmer": uint64_t get_n_merges() ctypedef shared_ptr[CyComponent] ComponentPtr + ctypedef set[ComponentPtr] ComponentPtrSet + cdef cppclass GuardedKmerCompMap "khmer::GuardedKmerCompMap": + map[HashIntoType, ComponentPtr] data + + ComponentPtr get(HashIntoType) + void set(HashIntoType, ComponentPtr) + bool contains(HashIntoType) -cdef extern from "partitioning.hh" namespace "khmer": cdef cppclass CyStreamingPartitioner "khmer::StreamingPartitioner": CyStreamingPartitioner(CyHashtable * ) except +MemoryError @@ -62,3 +69,9 @@ cdef extern from "partitioning.hh" namespace "khmer": set[HashIntoType]&, set[HashIntoType]&) except +MemoryError uint64_t get_n_components() + + ComponentPtr get_tag_component(string&) const + ComponentPtr get_nearest_component(string&) const + + weak_ptr[ComponentPtrSet] get_component_set() + weak_ptr[GuardedKmerCompMap] get_tag_component_map() diff --git a/khmer/_oxli.pyx b/khmer/_oxli.pyx index cc45a5e97c..eec15f9e57 100644 --- a/khmer/_oxli.pyx +++ b/khmer/_oxli.pyx @@ -15,7 +15,7 @@ cdef class Component: self._this.reset(other._this.get()) else: self._this.reset(new CyComponent()) - + property component_id: def __get__(self): return deref(self._this).component_id @@ -34,9 +34,18 @@ cdef class Component: inc(it) +cdef Component build_component(ComponentPtr ptr): + cdef Component comp = Component() + comp._this.reset(ptr.get()) + return comp + + cdef class StreamingPartitioner: cdef unique_ptr[CyStreamingPartitioner] _this + cdef weak_ptr[ComponentPtrSet] _components + cdef weak_ptr[GuardedKmerCompMap] _tag_component_map + cdef CyHashtable * _graph_ptr def __cinit__(self, graph): @@ -52,6 +61,22 @@ cdef class StreamingPartitioner: def consume_sequence(self, sequence): deref(self._this).consume_sequence(sequence.encode('utf-8')) + def get_tag_component(self, kmer): + cdef ComponentPtr comp + comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) + if comp == NULL: + return None + else: + return build_component(comp) + + def get_nearest_component(self, kmer): + cdef ComponentPtr comp + comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + if comp == NULL: + return None + else: + return build_component(comp) + property n_components: def __get__(self): return deref(self._this).get_n_components() diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 1254b28427..c4b6ad4b67 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -123,7 +123,7 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) find_connected_tags(search_from, tags, seen); // Now resolve components. First, get components from existing tags. - std::set comps; + ComponentPtrSet comps; ComponentPtr comp; for (auto tag: tags) { if ((comp = tag_component_map->get(tag)) != NULL) { @@ -133,6 +133,7 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) if (comps.size() == 0) { comp = std::make_shared(); + components->insert(comp); n_components++; } else { // get the first component @@ -142,9 +143,7 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) comp->add_tag(tags); map_tags_to_component(tags, comp); } else { - // merge the components - comp->merge(comps); - n_components -= comps.size() - 1; + merge_components(comp, comps); } } // (re)map all the tags to the component @@ -155,9 +154,61 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) } +void StreamingPartitioner::merge_components(ComponentPtr root, + std::set comps) +{ + root->merge(comps); + n_components = comps.size() - 1; + for (auto other : comps) { + if (other == root) { + continue; + } + components->erase(other); + } +} + + +ComponentPtr StreamingPartitioner::get_tag_component(HashIntoType tag) const +{ + return tag_component_map->get(tag); +} + + +ComponentPtr StreamingPartitioner::get_tag_component(std::string& kmer) const +{ + HashIntoType h = graph->hash_dna(kmer.c_str()); + return get_tag_component(h); +} + + +ComponentPtr StreamingPartitioner::get_nearest_component(std::string& kmer) const +{ + Kmer hashed = graph->build_kmer(kmer); + return get_nearest_component(hashed); +} + + +ComponentPtr StreamingPartitioner::get_nearest_component(Kmer kmer) const +{ + std::set tags; + std::set seen; + KmerQueue node_q; + node_q.push(kmer); + + find_connected_tags(node_q, tags, seen, true); + if (tags.size() > 0) { + HashIntoType tag = *(tags.begin()); + return tag_component_map->get(tag); + } else { + return NULL; + } +} + + void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, std::set& found_tags, - std::set& seen) + std::set& seen, + bool truncate) const { //if (auto graphptr = graph.lock()) { @@ -192,6 +243,9 @@ void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, // Found a tag! if (tag_component_map->contains(node)) { found_tags.insert(node); + if (truncate) { + return; + } continue; } diff --git a/lib/partitioning.hh b/lib/partitioning.hh index 806de773e1..de84170e30 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -142,6 +142,7 @@ class Component { }; typedef std::shared_ptr ComponentPtr; +typedef std::set ComponentPtrSet; typedef GuardedKmerMap GuardedKmerCompMap; class StreamingPartitioner { @@ -154,9 +155,9 @@ class StreamingPartitioner { // Unforunately our ownership policies elsewhere are a mess Hashtable * graph; //std::weak_ptr graph; - // We should exlusively own tag_component_map. - std::unique_ptr tag_component_map; - std::unique_ptr components; + // We should exclusively own tag_component_map. + std::shared_ptr tag_component_map; + std::shared_ptr components; uint64_t n_components; public: @@ -167,10 +168,27 @@ class StreamingPartitioner { void map_tags_to_component(std::set tags, ComponentPtr& comp); void find_connected_tags(KmerQueue& node_q, std::set& found_tags, - std::set& seen); - uint64_t get_n_components() { + std::set& seen, + bool truncate=false) const; + uint64_t get_n_components() const { return n_components; } + + void merge_components(ComponentPtr root, ComponentPtrSet comps); + + ComponentPtr get_tag_component(HashIntoType tag) const; + ComponentPtr get_tag_component(std::string& tag) const; + + ComponentPtr get_nearest_component(Kmer kmer) const; + ComponentPtr get_nearest_component(std::string& kmer) const; + + std::weak_ptr get_component_set() { + return std::weak_ptr(components); + } + + std::weak_ptr get_tag_component_map() { + return std::weak_ptr(tag_component_map); + } }; diff --git a/tests/test_cython_oxli.py b/tests/test_cython_oxli.py index 125ee3767f..b92b756b98 100644 --- a/tests/test_cython_oxli.py +++ b/tests/test_cython_oxli.py @@ -41,3 +41,16 @@ def test_two_components(self, random_sequence): sp.consume_sequence(comp2) assert sp.n_components == 2 + + def test_get_nearest_component(self, random_sequence): + comp1 = random_sequence() + comp2 = random_sequence(exclude=comp1) + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = _oxli.StreamingPartitioner(cg) + assert False + sp.consume_sequence(comp1) + sp.consume_sequence(comp2) + + c = sp.get_nearest_component(comp1[:K]) + assert c.component_id == 0 From c3c57c7f8dc527e744293bd1a489902b86bdc8e3 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 6 Nov 2016 17:24:14 -0800 Subject: [PATCH 007/185] Add in missing add_tag when creating component, debug statements --- khmer/_oxli.pxd | 4 ++-- lib/partitioning.cc | 39 ++++++++++++++++++++++++++++++++++++--- lib/partitioning.hh | 25 ++++++++++++++++--------- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index a498981811..195b9a2644 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -47,8 +47,8 @@ cdef extern from "partitioning.hh" namespace "khmer": void merge(set[shared_ptr[CyComponent]]) void add_tag(HashIntoType) void add_tag(set[HashIntoType]) - uint64_t get_n_tags() - uint64_t get_n_merges() + uint64_t get_n_tags() const + uint64_t get_n_merges() const ctypedef shared_ptr[CyComponent] ComponentPtr ctypedef set[ComponentPtr] ComponentPtrSet diff --git a/lib/partitioning.cc b/lib/partitioning.cc index c4b6ad4b67..6362e2e339 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -13,8 +13,14 @@ using namespace khmer; uint64_t Component::n_created = 0; +std::ostream& operator<< (std::ostream& stream, Component& comp) { + stream << ""; + return stream; +} + StreamingPartitioner::StreamingPartitioner(Hashtable * graph) : - graph(graph), _tag_density(DEFAULT_TAG_DENSITY), n_components(0) + graph(graph), _tag_density(DEFAULT_TAG_DENSITY) { //if (auto graphptr = graph.lock()) { @@ -32,6 +38,7 @@ StreamingPartitioner::StreamingPartitioner(Hashtable * graph) : new GuardedKmerCompMap(graph->ksize(), graph->n_tables(), graph_max_table_size / (_tag_density-2))); + components = std::make_shared(); } else { throw khmer_ptr_exception("Hashtable has been deleted."); } @@ -43,6 +50,7 @@ void StreamingPartitioner::map_tags_to_component(std::set tags, { for (auto tag: tags) { tag_component_map->set(tag, comp); + comp->add_tag(tag); } } @@ -119,25 +127,45 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) } intersection.clear(); +#if(DEBUG_SP) + std::cout << "Done iterating k-mers" << std::endl; +#endif // Now search for tagged nodes connected to U. find_connected_tags(search_from, tags, seen); // Now resolve components. First, get components from existing tags. ComponentPtrSet comps; ComponentPtr comp; +#if(DEBUG_SP) + std::cout << "Get found comps: " << std::endl; +#endif for (auto tag: tags) { +#if(DEBUG_SP) + std::cout << "Tag: " << tag; +#endif if ((comp = tag_component_map->get(tag)) != NULL) { +#if(DEBUG_SP) + std::cout << "->" << *comp; +#endif comps.insert(comp); } +#if(DEBUG_SP) + std::cout << std::endl; +#endif } if (comps.size() == 0) { comp = std::make_shared(); +#if(DEBUG_SP) + std::cout << "Build new comp: " << *comp << std::endl; +#endif components->insert(comp); - n_components++; } else { // get the first component comp = *(comps.begin()); +#if(DEBUG_SP) + std::cout << "Merge into: " << *comp << std::endl; +#endif if (comps.size() == 1) { // map the new tags to this component comp->add_tag(tags); @@ -157,12 +185,17 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) void StreamingPartitioner::merge_components(ComponentPtr root, std::set comps) { +#if(DEBUG_SP) + std::cout << "Merge components." << std::endl; +#endif root->merge(comps); - n_components = comps.size() - 1; for (auto other : comps) { if (other == root) { continue; } +#if(DEBUG_SP) + std::cout << "Erase " << *other << std::endl; +#endif components->erase(other); } } diff --git a/lib/partitioning.hh b/lib/partitioning.hh index de84170e30..ca85a98562 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -48,6 +48,9 @@ Contact: khmer-project@idyll.org #include "kmer_filters.hh" #include "traversal.hh" +#ifndef DEBUG_SP +#define DEBUG_SP 0 +#endif namespace khmer { @@ -69,7 +72,7 @@ class GuardedKmerMap { filter = std::unique_ptr(new Hashbits(ksize, table_sizes)); } - T get(HashIntoType kmer) { + T get(HashIntoType kmer) const { if (filter->get_count(kmer)) { auto search = data.find(kmer); if (search != data.end()) { @@ -85,7 +88,7 @@ class GuardedKmerMap { data[kmer] = item; } - bool contains(HashIntoType kmer) { + bool contains(HashIntoType kmer) const { return get(kmer) != NULL; } @@ -126,21 +129,25 @@ class Component { tags.insert(tag); } - void add_tag(std::set new_tags) { + void add_tag(std::set& new_tags) { for (auto tag: new_tags) { add_tag(tag); } } - uint64_t get_n_tags() { + uint64_t get_n_tags() const { return tags.size(); } - uint64_t get_n_merges() { + uint64_t get_n_merges() const { return n_merges; } + + friend std::ostream& operator<< (std::ostream& stream, const Component& comp) ; }; + + typedef std::shared_ptr ComponentPtr; typedef std::set ComponentPtrSet; typedef GuardedKmerMap GuardedKmerCompMap; @@ -158,7 +165,6 @@ class StreamingPartitioner { // We should exclusively own tag_component_map. std::shared_ptr tag_component_map; std::shared_ptr components; - uint64_t n_components; public: @@ -170,8 +176,9 @@ class StreamingPartitioner { std::set& found_tags, std::set& seen, bool truncate=false) const; + uint64_t get_n_components() const { - return n_components; + return components->size(); } void merge_components(ComponentPtr root, ComponentPtrSet comps); @@ -182,11 +189,11 @@ class StreamingPartitioner { ComponentPtr get_nearest_component(Kmer kmer) const; ComponentPtr get_nearest_component(std::string& kmer) const; - std::weak_ptr get_component_set() { + std::weak_ptr get_component_set() const { return std::weak_ptr(components); } - std::weak_ptr get_tag_component_map() { + std::weak_ptr get_tag_component_map() const { return std::weak_ptr(tag_component_map); } }; From 7f63817ddd4a92bdb38f87eddd6862d0be054f23 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 6 Nov 2016 17:25:03 -0800 Subject: [PATCH 008/185] Fix Component constructor, expose StreamingPartitioner containers --- khmer/_oxli.cpp | 1700 +++++++++++++++++++++++++++++++++++++---------- khmer/_oxli.pyx | 64 +- 2 files changed, 1411 insertions(+), 353 deletions(-) diff --git a/khmer/_oxli.cpp b/khmer/_oxli.cpp index 5c6ecf8f60..5bf2ebc039 100644 --- a/khmer/_oxli.cpp +++ b/khmer/_oxli.cpp @@ -6,7 +6,7 @@ "define_macros": [ [ "VERSION", - "2.0+521.gb3c6a3d.dirty" + "2.0+522.g1967b74.dirty" ] ], "depends": [ @@ -699,8 +699,10 @@ static const char *__pyx_f[] = { struct __pyx_obj_5khmer_5_oxli_Component; struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner; struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__; +struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components; +struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components; -/* "khmer/_oxli.pyx":9 +/* "khmer/_oxli.pyx":10 * from khmer._khmer import Nodegraph * * cdef class Component: # <<<<<<<<<<<<<< @@ -709,11 +711,12 @@ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__; */ struct __pyx_obj_5khmer_5_oxli_Component { PyObject_HEAD + struct __pyx_vtabstruct_5khmer_5_oxli_Component *__pyx_vtab; khmer::ComponentPtr _this; }; -/* "khmer/_oxli.pyx":43 +/* "khmer/_oxli.pyx":51 * * * cdef class StreamingPartitioner: # <<<<<<<<<<<<<< @@ -729,7 +732,7 @@ struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner { }; -/* "khmer/_oxli.pyx":30 +/* "khmer/_oxli.pyx":29 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< @@ -743,6 +746,52 @@ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ { }; +/* "khmer/_oxli.pyx":90 + * return Component.create(compptr) + * + * def components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[ComponentPtrSet] locked + * lockedptr = self._components.lock() + */ +struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components { + PyObject_HEAD + std::set ::iterator __pyx_v_it; + std::shared_ptr __pyx_v_locked; + std::shared_ptr __pyx_v_lockedptr; + struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self; +}; + + +/* "khmer/_oxli.pyx":101 + * raise MemoryError("Can't locked underlying Component set") + * + * def tag_components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[GuardedKmerCompMap] locked + * lockedptr = self._tag_component_map.lock() + */ +struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components { + PyObject_HEAD + std::map ::iterator __pyx_v_it; + std::shared_ptr __pyx_v_locked; + std::shared_ptr __pyx_v_lockedptr; + struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self; +}; + + + +/* "khmer/_oxli.pyx":10 + * from khmer._khmer import Nodegraph + * + * cdef class Component: # <<<<<<<<<<<<<< + * + * cdef ComponentPtr _this + */ + +struct __pyx_vtabstruct_5khmer_5_oxli_Component { + struct __pyx_obj_5khmer_5_oxli_Component *(*create)(khmer::ComponentPtr); +}; +static struct __pyx_vtabstruct_5khmer_5_oxli_Component *__pyx_vtabptr_5khmer_5_oxli_Component; + /* --- Runtime support code (head) --- */ /* Refnanny.proto */ #ifndef CYTHON_REFNANNY @@ -842,6 +891,14 @@ static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, const char *name, int exact); +/* PyIntBinop.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); +#else +#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ + PyObject_RichCompare(op1, op2, Py_EQ) + #endif + /* PyObjectCall.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); @@ -849,9 +906,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg #define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) #endif -/* GetModuleGlobalName.proto */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); - /* PyThreadStateGet.proto */ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; @@ -879,9 +933,15 @@ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject /* RaiseException.proto */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); +/* GetModuleGlobalName.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); + /* IncludeStringH.proto */ #include +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyObject *dict, void *vtable); + /* Import.proto */ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); @@ -910,47 +970,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, /* None.proto */ #include -/* CppExceptionConversion.proto */ -#ifndef __Pyx_CppExn2PyErr -#include -#include -#include -#include -static void __Pyx_CppExn2PyErr() { - try { - if (PyErr_Occurred()) - ; // let the latest Python exn pass through and ignore the current one - else - throw; - } catch (const std::bad_alloc& exn) { - PyErr_SetString(PyExc_MemoryError, exn.what()); - } catch (const std::bad_cast& exn) { - PyErr_SetString(PyExc_TypeError, exn.what()); - } catch (const std::bad_typeid& exn) { - PyErr_SetString(PyExc_TypeError, exn.what()); - } catch (const std::domain_error& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::invalid_argument& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::ios_base::failure& exn) { - PyErr_SetString(PyExc_IOError, exn.what()); - } catch (const std::out_of_range& exn) { - PyErr_SetString(PyExc_IndexError, exn.what()); - } catch (const std::overflow_error& exn) { - PyErr_SetString(PyExc_OverflowError, exn.what()); - } catch (const std::range_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::underflow_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::exception& exn) { - PyErr_SetString(PyExc_RuntimeError, exn.what()); - } - catch (...) - { - PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); - } -} -#endif +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value); @@ -1056,6 +1077,7 @@ static int __Pyx_check_binary_version(void); /* InitStrings.proto */ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); +static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Component_create(khmer::ComponentPtr __pyx_v_ptr); /* proto*/ /* Module declarations from 'libcpp' */ @@ -1085,13 +1107,15 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); static PyTypeObject *__pyx_ptype_5khmer_5_oxli_Component = 0; static PyTypeObject *__pyx_ptype_5khmer_5_oxli_StreamingPartitioner = 0; static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = 0; -static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_build_component(khmer::ComponentPtr); /*proto*/ +static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components = 0; +static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components = 0; static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ #define __Pyx_MODULE_NAME "khmer._oxli" int __pyx_module_is_main_khmer___oxli = 0; /* Implementation of 'khmer._oxli' */ static PyObject *__pyx_builtin_MemoryError; +static PyObject *__pyx_builtin_NotImplementedError; static PyObject *__pyx_builtin_ValueError; static const char __pyx_k_args[] = "args"; static const char __pyx_k_iter[] = "__iter__"; @@ -1108,19 +1132,35 @@ static const char __pyx_k_import[] = "__import__"; static const char __pyx_k_Nodegraph[] = "Nodegraph"; static const char __pyx_k_Countgraph[] = "Countgraph"; static const char __pyx_k_ValueError[] = "ValueError"; +static const char __pyx_k_components[] = "components"; +static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; static const char __pyx_k_MemoryError[] = "MemoryError"; static const char __pyx_k_khmer__oxli[] = "khmer._oxli"; +static const char __pyx_k_component_id[] = "component_id"; static const char __pyx_k_khmer__khmer[] = "khmer._khmer"; +static const char __pyx_k_tag_components[] = "tag_components"; static const char __pyx_k_Component___iter[] = "Component.__iter__"; +static const char __pyx_k_NotImplementedError[] = "NotImplementedError"; +static const char __pyx_k_Operator_not_available[] = "Operator not available."; +static const char __pyx_k_StreamingPartitioner_components[] = "StreamingPartitioner.components"; +static const char __pyx_k_Can_t_locked_underlying_Componen[] = "Can't locked underlying Component set"; static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; +static const char __pyx_k_StreamingPartitioner_tag_compone[] = "StreamingPartitioner.tag_components"; +static PyObject *__pyx_kp_s_Can_t_locked_underlying_Componen; static PyObject *__pyx_n_s_Component___iter; static PyObject *__pyx_n_s_Countgraph; static PyObject *__pyx_n_s_MemoryError; static PyObject *__pyx_kp_s_Must_take_an_object_with_Hashtab; static PyObject *__pyx_n_s_Nodegraph; +static PyObject *__pyx_n_s_NotImplementedError; +static PyObject *__pyx_kp_s_Operator_not_available; +static PyObject *__pyx_n_s_StreamingPartitioner_components; +static PyObject *__pyx_n_s_StreamingPartitioner_tag_compone; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_n_s_args; static PyObject *__pyx_n_s_close; +static PyObject *__pyx_n_s_component_id; +static PyObject *__pyx_n_s_components; static PyObject *__pyx_n_s_encode; static PyObject *__pyx_n_s_graph; static PyObject *__pyx_n_s_import; @@ -1129,7 +1169,9 @@ static PyObject *__pyx_n_s_khmer__khmer; static PyObject *__pyx_n_s_khmer__oxli; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_other; +static PyObject *__pyx_n_s_pyx_vtable; static PyObject *__pyx_n_s_send; +static PyObject *__pyx_n_s_tag_components; static PyObject *__pyx_n_s_test; static PyObject *__pyx_n_s_throw; static PyObject *__pyx_kp_s_utf_8; @@ -1138,20 +1180,30 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ +static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op); /* proto */ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5khmer_5_oxli_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_int_2; static PyObject *__pyx_tuple_; static PyObject *__pyx_tuple__2; static PyObject *__pyx_tuple__3; static PyObject *__pyx_tuple__4; +static PyObject *__pyx_tuple__5; +static PyObject *__pyx_tuple__6; +static PyObject *__pyx_tuple__7; -/* "khmer/_oxli.pyx":13 +/* "khmer/_oxli.pyx":14 * cdef ComponentPtr _this * * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< @@ -1187,7 +1239,7 @@ static int __pyx_pw_5khmer_5_oxli_9Component_1__cinit__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 13, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 14, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -1200,13 +1252,13 @@ static int __pyx_pw_5khmer_5_oxli_9Component_1__cinit__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 13, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 14, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("khmer._oxli.Component.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return -1; __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5khmer_5_oxli_Component, 1, "other", 0))) __PYX_ERR(1, 13, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5khmer_5_oxli_Component, 1, "other", 0))) __PYX_ERR(1, 14, __pyx_L1_error) __pyx_r = __pyx_pf_5khmer_5_oxli_9Component___cinit__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self), __pyx_v_other); /* function exit code */ @@ -1223,58 +1275,38 @@ static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5 __Pyx_RefNannyDeclarations int __pyx_t_1; int __pyx_t_2; - khmer::Component *__pyx_t_3; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "khmer/_oxli.pyx":14 + /* "khmer/_oxli.pyx":15 * * def __cinit__(self, Component other=None): * if other is not None: # <<<<<<<<<<<<<< * self._this.reset(other._this.get()) - * else: + * */ __pyx_t_1 = (((PyObject *)__pyx_v_other) != Py_None); __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "khmer/_oxli.pyx":15 + /* "khmer/_oxli.pyx":16 * def __cinit__(self, Component other=None): * if other is not None: * self._this.reset(other._this.get()) # <<<<<<<<<<<<<< - * else: - * self._this.reset(new CyComponent()) + * + * property component_id: */ __pyx_v_self->_this.reset(__pyx_v_other->_this.get()); - /* "khmer/_oxli.pyx":14 + /* "khmer/_oxli.pyx":15 * * def __cinit__(self, Component other=None): * if other is not None: # <<<<<<<<<<<<<< * self._this.reset(other._this.get()) - * else: - */ - goto __pyx_L3; - } - - /* "khmer/_oxli.pyx":17 - * self._this.reset(other._this.get()) - * else: - * self._this.reset(new CyComponent()) # <<<<<<<<<<<<<< * - * property component_id: */ - /*else*/ { - try { - __pyx_t_3 = new khmer::Component(); - } catch(...) { - __Pyx_CppExn2PyErr(); - __PYX_ERR(1, 17, __pyx_L1_error) - } - __pyx_v_self->_this.reset(__pyx_t_3); } - __pyx_L3:; - /* "khmer/_oxli.pyx":13 + /* "khmer/_oxli.pyx":14 * cdef ComponentPtr _this * * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< @@ -1284,16 +1316,11 @@ static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5 /* function exit code */ __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("khmer._oxli.Component.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "khmer/_oxli.pyx":20 +/* "khmer/_oxli.pyx":19 * * property component_id: * def __get__(self): # <<<<<<<<<<<<<< @@ -1320,7 +1347,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - /* "khmer/_oxli.pyx":21 + /* "khmer/_oxli.pyx":20 * property component_id: * def __get__(self): * return deref(self._this).component_id # <<<<<<<<<<<<<< @@ -1328,13 +1355,13 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct * property n_merges: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 21, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":20 + /* "khmer/_oxli.pyx":19 * * property component_id: * def __get__(self): # <<<<<<<<<<<<<< @@ -1353,7 +1380,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct return __pyx_r; } -/* "khmer/_oxli.pyx":24 +/* "khmer/_oxli.pyx":23 * * property n_merges: * def __get__(self): # <<<<<<<<<<<<<< @@ -1380,7 +1407,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __py PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - /* "khmer/_oxli.pyx":25 + /* "khmer/_oxli.pyx":24 * property n_merges: * def __get__(self): * return deref(self._this).get_n_merges() # <<<<<<<<<<<<<< @@ -1388,13 +1415,13 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __py * def __len__(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_merges()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 25, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_merges()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":24 + /* "khmer/_oxli.pyx":23 * * property n_merges: * def __get__(self): # <<<<<<<<<<<<<< @@ -1413,7 +1440,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __py return __pyx_r; } -/* "khmer/_oxli.pyx":27 +/* "khmer/_oxli.pyx":26 * return deref(self._this).get_n_merges() * * def __len__(self): # <<<<<<<<<<<<<< @@ -1439,7 +1466,7 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "khmer/_oxli.pyx":28 + /* "khmer/_oxli.pyx":27 * * def __len__(self): * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< @@ -1449,7 +1476,7 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k __pyx_r = (*__pyx_v_self->_this).get_n_tags(); goto __pyx_L0; - /* "khmer/_oxli.pyx":27 + /* "khmer/_oxli.pyx":26 * return deref(self._this).get_n_merges() * * def __len__(self): # <<<<<<<<<<<<<< @@ -1464,7 +1491,7 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k } static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "khmer/_oxli.pyx":30 +/* "khmer/_oxli.pyx":29 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< @@ -1494,7 +1521,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5k if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(1, 30, __pyx_L1_error) + __PYX_ERR(1, 29, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } @@ -1502,7 +1529,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5k __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 30, __pyx_L1_error) + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 29, __pyx_L1_error) __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -1534,9 +1561,9 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 30, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 29, __pyx_L1_error) - /* "khmer/_oxli.pyx":31 + /* "khmer/_oxli.pyx":30 * * def __iter__(self): * it = deref(self._this).tags.begin() # <<<<<<<<<<<<<< @@ -1545,7 +1572,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj */ __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_self->_this).tags.begin(); - /* "khmer/_oxli.pyx":32 + /* "khmer/_oxli.pyx":31 * def __iter__(self): * it = deref(self._this).tags.begin() * while it != deref(self._this).tags.end(): # <<<<<<<<<<<<<< @@ -1556,14 +1583,14 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj __pyx_t_1 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_self->_this).tags.end()) != 0); if (!__pyx_t_1) break; - /* "khmer/_oxli.pyx":33 + /* "khmer/_oxli.pyx":32 * it = deref(self._this).tags.begin() * while it != deref(self._this).tags.end(): * yield deref(it) # <<<<<<<<<<<<<< * inc(it) * */ - __pyx_t_2 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 33, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 32, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; @@ -1573,20 +1600,20 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L6_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 33, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 32, __pyx_L1_error) - /* "khmer/_oxli.pyx":34 + /* "khmer/_oxli.pyx":33 * while it != deref(self._this).tags.end(): * yield deref(it) * inc(it) # <<<<<<<<<<<<<< * - * + * def __hash__(self): */ (++__pyx_cur_scope->__pyx_v_it); } if (1); else __pyx_cur_scope = __pyx_cur_scope; - /* "khmer/_oxli.pyx":30 + /* "khmer/_oxli.pyx":29 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< @@ -1608,46 +1635,214 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj return __pyx_r; } -/* "khmer/_oxli.pyx":37 +/* "khmer/_oxli.pyx":35 + * inc(it) * + * def __hash__(self): # <<<<<<<<<<<<<< + * return self._this.get() * - * cdef Component build_component(ComponentPtr ptr): # <<<<<<<<<<<<<< - * cdef Component comp = Component() - * comp._this.reset(ptr.get()) */ -static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_build_component(khmer::ComponentPtr __pyx_v_ptr) { - struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_comp = 0; - struct __pyx_obj_5khmer_5_oxli_Component *__pyx_r = NULL; +/* Python wrapper */ +static Py_hash_t __pyx_pw_5khmer_5_oxli_9Component_8__hash__(PyObject *__pyx_v_self); /*proto*/ +static Py_hash_t __pyx_pw_5khmer_5_oxli_9Component_8__hash__(PyObject *__pyx_v_self) { + Py_hash_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_7__hash__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { + Py_hash_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__hash__", 0); + + /* "khmer/_oxli.pyx":36 + * + * def __hash__(self): + * return self._this.get() # <<<<<<<<<<<<<< + * + * def __richcmp__(x, y, op): + */ + __pyx_r = ((uintptr_t)__pyx_v_self->_this.get()); + goto __pyx_L0; + + /* "khmer/_oxli.pyx":35 + * inc(it) + * + * def __hash__(self): # <<<<<<<<<<<<<< + * return self._this.get() + * + */ + + /* function exit code */ + __pyx_L0:; + if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":38 + * return self._this.get() + * + * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< + * if op == 2: + * return x.component_id == y.component_id + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_arg_op); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_arg_op) { + PyObject *__pyx_v_op = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); + __pyx_v_op = __Pyx_PyInt_From_int(__pyx_arg_op); if (unlikely(!__pyx_v_op)) __PYX_ERR(1, 38, __pyx_L3_error) + __Pyx_GOTREF(__pyx_v_op); + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.Component.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), ((PyObject *)__pyx_v_op)); + + /* function exit code */ + __Pyx_XDECREF(__pyx_v_op); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op) { + PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("build_component", 0); + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannySetupContext("__richcmp__", 0); + + /* "khmer/_oxli.pyx":39 + * + * def __richcmp__(x, y, op): + * if op == 2: # <<<<<<<<<<<<<< + * return x.component_id == y.component_id + * else: + */ + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_v_op, __pyx_int_2, 2, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 39, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(1, 39, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "khmer/_oxli.pyx":40 + * def __richcmp__(x, y, op): + * if op == 2: + * return x.component_id == y.component_id # <<<<<<<<<<<<<< + * else: + * raise NotImplementedError('Operator not available.') + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_x, __pyx_n_s_component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_y, __pyx_n_s_component_id); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyObject_RichCompare(__pyx_t_1, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 40, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":39 + * + * def __richcmp__(x, y, op): + * if op == 2: # <<<<<<<<<<<<<< + * return x.component_id == y.component_id + * else: + */ + } + + /* "khmer/_oxli.pyx":42 + * return x.component_id == y.component_id + * else: + * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< + * + * @staticmethod + */ + /*else*/ { + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_NotImplementedError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(1, 42, __pyx_L1_error) + } /* "khmer/_oxli.pyx":38 + * return self._this.get() + * + * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< + * if op == 2: + * return x.component_id == y.component_id + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("khmer._oxli.Component.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":45 * - * cdef Component build_component(ComponentPtr ptr): - * cdef Component comp = Component() # <<<<<<<<<<<<<< - * comp._this.reset(ptr.get()) - * return comp + * @staticmethod + * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< + * cdef Component comp = Component() + * comp._this = ptr + */ + +static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Component_create(khmer::ComponentPtr __pyx_v_ptr) { + struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_comp = 0; + struct __pyx_obj_5khmer_5_oxli_Component *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("create", 0); + + /* "khmer/_oxli.pyx":46 + * @staticmethod + * cdef Component create(ComponentPtr ptr): + * cdef Component comp = Component() # <<<<<<<<<<<<<< + * comp._this = ptr + * return comp */ - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_Component), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 38, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_Component), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 46, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_comp = ((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_t_1); __pyx_t_1 = 0; - /* "khmer/_oxli.pyx":39 - * cdef Component build_component(ComponentPtr ptr): - * cdef Component comp = Component() - * comp._this.reset(ptr.get()) # <<<<<<<<<<<<<< - * return comp + /* "khmer/_oxli.pyx":47 + * cdef Component create(ComponentPtr ptr): + * cdef Component comp = Component() + * comp._this = ptr # <<<<<<<<<<<<<< + * return comp * */ - __pyx_v_comp->_this.reset(__pyx_v_ptr.get()); + __pyx_v_comp->_this = __pyx_v_ptr; - /* "khmer/_oxli.pyx":40 - * cdef Component comp = Component() - * comp._this.reset(ptr.get()) - * return comp # <<<<<<<<<<<<<< + /* "khmer/_oxli.pyx":48 + * cdef Component comp = Component() + * comp._this = ptr + * return comp # <<<<<<<<<<<<<< * * */ @@ -1656,18 +1851,18 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_build_com __pyx_r = __pyx_v_comp; goto __pyx_L0; - /* "khmer/_oxli.pyx":37 + /* "khmer/_oxli.pyx":45 * - * - * cdef Component build_component(ComponentPtr ptr): # <<<<<<<<<<<<<< - * cdef Component comp = Component() - * comp._this.reset(ptr.get()) + * @staticmethod + * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< + * cdef Component comp = Component() + * comp._this = ptr */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.build_component", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("khmer._oxli.Component.create", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; __Pyx_XDECREF((PyObject *)__pyx_v_comp); @@ -1676,7 +1871,7 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_build_com return __pyx_r; } -/* "khmer/_oxli.pyx":51 +/* "khmer/_oxli.pyx":58 * cdef CyHashtable * _graph_ptr * * def __cinit__(self, graph): # <<<<<<<<<<<<<< @@ -1709,7 +1904,7 @@ static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__ else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 51, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 58, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { goto __pyx_L5_argtuple_error; @@ -1720,7 +1915,7 @@ static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__ } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 51, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 58, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -1745,16 +1940,16 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ khmer::StreamingPartitioner *__pyx_t_6; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "khmer/_oxli.pyx":52 + /* "khmer/_oxli.pyx":59 * * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< * raise ValueError('Must take an object with Hashtable *') * */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 52, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 59, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 52, __pyx_L1_error) + __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 59, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_4 = (__pyx_t_3 != 0); if (!__pyx_t_4) { @@ -1762,9 +1957,9 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_1 = __pyx_t_4; goto __pyx_L4_bool_binop_done; } - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 52, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 59, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 52, __pyx_L1_error) + __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 59, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_3 = (__pyx_t_4 != 0); __pyx_t_1 = __pyx_t_3; @@ -1772,20 +1967,20 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_3 = ((!__pyx_t_1) != 0); if (__pyx_t_3) { - /* "khmer/_oxli.pyx":53 + /* "khmer/_oxli.pyx":60 * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< * * */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 53, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 60, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 53, __pyx_L1_error) + __PYX_ERR(1, 60, __pyx_L1_error) - /* "khmer/_oxli.pyx":52 + /* "khmer/_oxli.pyx":59 * * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< @@ -1794,7 +1989,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ } - /* "khmer/_oxli.pyx":56 + /* "khmer/_oxli.pyx":63 * * * cdef CyCpHashtable_Object* ptr = graph # <<<<<<<<<<<<<< @@ -1803,7 +1998,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - /* "khmer/_oxli.pyx":57 + /* "khmer/_oxli.pyx":64 * * cdef CyCpHashtable_Object* ptr = graph * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< @@ -1813,22 +2008,40 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_5 = (*__pyx_v_ptr).hashtable; __pyx_v_self->_graph_ptr = __pyx_t_5; - /* "khmer/_oxli.pyx":59 + /* "khmer/_oxli.pyx":66 * self._graph_ptr = deref(ptr).hashtable * * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) # <<<<<<<<<<<<<< * - * def consume_sequence(self, sequence): + * self._tag_component_map = deref(self._this).get_tag_component_map() */ try { __pyx_t_6 = new khmer::StreamingPartitioner(__pyx_v_self->_graph_ptr); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 59, __pyx_L1_error) + __PYX_ERR(1, 66, __pyx_L1_error) } __pyx_v_self->_this.reset(__pyx_t_6); - /* "khmer/_oxli.pyx":51 + /* "khmer/_oxli.pyx":68 + * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) + * + * self._tag_component_map = deref(self._this).get_tag_component_map() # <<<<<<<<<<<<<< + * self._components = deref(self._this).get_component_set() + * + */ + __pyx_v_self->_tag_component_map = (*__pyx_v_self->_this).get_tag_component_map(); + + /* "khmer/_oxli.pyx":69 + * + * self._tag_component_map = deref(self._this).get_tag_component_map() + * self._components = deref(self._this).get_component_set() # <<<<<<<<<<<<<< + * + * def consume_sequence(self, sequence): + */ + __pyx_v_self->_components = (*__pyx_v_self->_this).get_component_set(); + + /* "khmer/_oxli.pyx":58 * cdef CyHashtable * _graph_ptr * * def __cinit__(self, graph): # <<<<<<<<<<<<<< @@ -1848,8 +2061,8 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ return __pyx_r; } -/* "khmer/_oxli.pyx":61 - * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) +/* "khmer/_oxli.pyx":71 + * self._components = deref(self._this).get_component_set() * * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< * deref(self._this).consume_sequence(sequence.encode('utf-8')) @@ -1877,29 +2090,29 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence std::string __pyx_t_3; __Pyx_RefNannySetupContext("consume_sequence", 0); - /* "khmer/_oxli.pyx":62 + /* "khmer/_oxli.pyx":72 * * def consume_sequence(self, sequence): * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< * * def get_tag_component(self, kmer): */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 62, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 72, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 62, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 72, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 62, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 72, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; try { (*__pyx_v_self->_this).consume_sequence(__pyx_t_3); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 62, __pyx_L1_error) + __PYX_ERR(1, 72, __pyx_L1_error) } - /* "khmer/_oxli.pyx":61 - * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) + /* "khmer/_oxli.pyx":71 + * self._components = deref(self._this).get_component_set() * * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< * deref(self._this).consume_sequence(sequence.encode('utf-8')) @@ -1920,12 +2133,12 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence return __pyx_r; } -/* "khmer/_oxli.pyx":64 +/* "khmer/_oxli.pyx":74 * deref(self._this).consume_sequence(sequence.encode('utf-8')) * * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr comp - * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) */ /* Python wrapper */ @@ -1942,7 +2155,7 @@ static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5get_tag_componen } static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { - khmer::ComponentPtr __pyx_v_comp; + khmer::ComponentPtr __pyx_v_compptr; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; @@ -1951,75 +2164,75 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen int __pyx_t_4; __Pyx_RefNannySetupContext("get_tag_component", 0); - /* "khmer/_oxli.pyx":66 + /* "khmer/_oxli.pyx":76 * def get_tag_component(self, kmer): - * cdef ComponentPtr comp - * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if comp == NULL: + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if compptr == NULL: * return None */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 66, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 66, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_comp = (*__pyx_v_self->_this).get_tag_component(__pyx_t_3); + __pyx_v_compptr = (*__pyx_v_self->_this).get_tag_component(__pyx_t_3); - /* "khmer/_oxli.pyx":67 - * cdef ComponentPtr comp - * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if comp == NULL: # <<<<<<<<<<<<<< + /* "khmer/_oxli.pyx":77 + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if compptr == NULL: # <<<<<<<<<<<<<< * return None * else: */ - __pyx_t_4 = ((__pyx_v_comp == NULL) != 0); + __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); if (__pyx_t_4) { - /* "khmer/_oxli.pyx":68 - * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if comp == NULL: + /* "khmer/_oxli.pyx":78 + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if compptr == NULL: * return None # <<<<<<<<<<<<<< * else: - * return build_component(comp) + * return Component.create(compptr) */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0; - /* "khmer/_oxli.pyx":67 - * cdef ComponentPtr comp - * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if comp == NULL: # <<<<<<<<<<<<<< + /* "khmer/_oxli.pyx":77 + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if compptr == NULL: # <<<<<<<<<<<<<< * return None * else: */ } - /* "khmer/_oxli.pyx":70 + /* "khmer/_oxli.pyx":80 * return None * else: - * return build_component(comp) # <<<<<<<<<<<<<< + * return Component.create(compptr) # <<<<<<<<<<<<<< * * def get_nearest_component(self, kmer): */ /*else*/ { __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_build_component(__pyx_v_comp)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 70, __pyx_L1_error) + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; } - /* "khmer/_oxli.pyx":64 + /* "khmer/_oxli.pyx":74 * deref(self._this).consume_sequence(sequence.encode('utf-8')) * * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr comp - * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) */ /* function exit code */ @@ -2034,12 +2247,12 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen return __pyx_r; } -/* "khmer/_oxli.pyx":72 - * return build_component(comp) +/* "khmer/_oxli.pyx":82 + * return Component.create(compptr) * * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr comp - * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) */ /* Python wrapper */ @@ -2056,7 +2269,7 @@ static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_nearest_comp } static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { - khmer::ComponentPtr __pyx_v_comp; + khmer::ComponentPtr __pyx_v_compptr; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; @@ -2065,75 +2278,75 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp int __pyx_t_4; __Pyx_RefNannySetupContext("get_nearest_component", 0); - /* "khmer/_oxli.pyx":74 + /* "khmer/_oxli.pyx":84 * def get_nearest_component(self, kmer): - * cdef ComponentPtr comp - * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if comp == NULL: + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if compptr == NULL: * return None */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 74, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 74, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 74, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 84, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_comp = (*__pyx_v_self->_this).get_nearest_component(__pyx_t_3); + __pyx_v_compptr = (*__pyx_v_self->_this).get_nearest_component(__pyx_t_3); - /* "khmer/_oxli.pyx":75 - * cdef ComponentPtr comp - * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if comp == NULL: # <<<<<<<<<<<<<< + /* "khmer/_oxli.pyx":85 + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if compptr == NULL: # <<<<<<<<<<<<<< * return None * else: */ - __pyx_t_4 = ((__pyx_v_comp == NULL) != 0); + __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); if (__pyx_t_4) { - /* "khmer/_oxli.pyx":76 - * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if comp == NULL: + /* "khmer/_oxli.pyx":86 + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if compptr == NULL: * return None # <<<<<<<<<<<<<< * else: - * return build_component(comp) + * return Component.create(compptr) */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0; - /* "khmer/_oxli.pyx":75 - * cdef ComponentPtr comp - * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if comp == NULL: # <<<<<<<<<<<<<< + /* "khmer/_oxli.pyx":85 + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if compptr == NULL: # <<<<<<<<<<<<<< * return None * else: */ } - /* "khmer/_oxli.pyx":78 + /* "khmer/_oxli.pyx":88 * return None * else: - * return build_component(comp) # <<<<<<<<<<<<<< + * return Component.create(compptr) # <<<<<<<<<<<<<< * - * property n_components: + * def components(self): */ /*else*/ { __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_build_component(__pyx_v_comp)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 78, __pyx_L1_error) + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; } - /* "khmer/_oxli.pyx":72 - * return build_component(comp) + /* "khmer/_oxli.pyx":82 + * return Component.create(compptr) * * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr comp - * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) */ /* function exit code */ @@ -2147,78 +2360,474 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp __Pyx_RefNannyFinishContext(); return __pyx_r; } +static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "khmer/_oxli.pyx":81 - * - * property n_components: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_components() +/* "khmer/_oxli.pyx":90 + * return Component.create(compptr) * + * def components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[ComponentPtrSet] locked + * lockedptr = self._components.lock() */ /* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self) { +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); + __Pyx_RefNannySetupContext("components (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli.pyx":82 - * property n_components: - * def __get__(self): - * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 82, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":81 - * - * property n_components: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_components() - * - */ + __Pyx_RefNannySetupContext("components", 0); + __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components(__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(1, 90, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_components, __pyx_n_s_StreamingPartitioner_components, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 90, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } /* function exit code */ __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.n_components.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.components", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; - __pyx_L0:; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - -static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { - Py_ssize_t __pyx_v_length; - char *__pyx_v_data; - std::string __pyx_r; +static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + bool __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("None", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L7_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 90, __pyx_L1_error) + + /* "khmer/_oxli.pyx":92 + * def components(self): + * cdef shared_ptr[ComponentPtrSet] locked + * lockedptr = self._components.lock() # <<<<<<<<<<<<<< + * if lockedptr: + * it = deref(lockedptr).begin() + */ + __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_components.lock(); + + /* "khmer/_oxli.pyx":93 + * cdef shared_ptr[ComponentPtrSet] locked + * lockedptr = self._components.lock() + * if lockedptr: # <<<<<<<<<<<<<< + * it = deref(lockedptr).begin() + * while it != deref(lockedptr).end(): + */ + __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); + if (__pyx_t_1) { + + /* "khmer/_oxli.pyx":94 + * lockedptr = self._components.lock() + * if lockedptr: + * it = deref(lockedptr).begin() # <<<<<<<<<<<<<< + * while it != deref(lockedptr).end(): + * yield Component.create(deref(it)) + */ + __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_lockedptr).begin(); + + /* "khmer/_oxli.pyx":95 + * if lockedptr: + * it = deref(lockedptr).begin() + * while it != deref(lockedptr).end(): # <<<<<<<<<<<<<< + * yield Component.create(deref(it)) + * inc(it) + */ + while (1) { + __pyx_t_2 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_lockedptr).end()) != 0); + if (!__pyx_t_2) break; + + /* "khmer/_oxli.pyx":96 + * it = deref(lockedptr).begin() + * while it != deref(lockedptr).end(): + * yield Component.create(deref(it)) # <<<<<<<<<<<<<< + * inc(it) + * else: + */ + __pyx_t_3 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create((*__pyx_cur_scope->__pyx_v_it))); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 96, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L7_resume_from_yield:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 96, __pyx_L1_error) + + /* "khmer/_oxli.pyx":97 + * while it != deref(lockedptr).end(): + * yield Component.create(deref(it)) + * inc(it) # <<<<<<<<<<<<<< + * else: + * raise MemoryError("Can't locked underlying Component set") + */ + (++__pyx_cur_scope->__pyx_v_it); + } + + /* "khmer/_oxli.pyx":93 + * cdef shared_ptr[ComponentPtrSet] locked + * lockedptr = self._components.lock() + * if lockedptr: # <<<<<<<<<<<<<< + * it = deref(lockedptr).begin() + * while it != deref(lockedptr).end(): + */ + goto __pyx_L4; + } + + /* "khmer/_oxli.pyx":99 + * inc(it) + * else: + * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< + * + * def tag_components(self): + */ + /*else*/ { + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 99, __pyx_L1_error) + } + __pyx_L4:; + if (1); else __pyx_cur_scope = __pyx_cur_scope; + + /* "khmer/_oxli.pyx":90 + * return Component.create(compptr) + * + * def components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[ComponentPtrSet] locked + * lockedptr = self._components.lock() + */ + + /* function exit code */ + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); __pyx_r = 0; + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ + +/* "khmer/_oxli.pyx":101 + * raise MemoryError("Can't locked underlying Component set") + * + * def tag_components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[GuardedKmerCompMap] locked + * lockedptr = self._tag_component_map.lock() + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("tag_components (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("tag_components", 0); + __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components(__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(1, 101, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_tag_components, __pyx_n_s_StreamingPartitioner_tag_compone, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 101, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + bool __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("None", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L7_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 101, __pyx_L1_error) + + /* "khmer/_oxli.pyx":103 + * def tag_components(self): + * cdef shared_ptr[GuardedKmerCompMap] locked + * lockedptr = self._tag_component_map.lock() # <<<<<<<<<<<<<< + * if lockedptr: + * it = deref(lockedptr).data.begin() + */ + __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_tag_component_map.lock(); + + /* "khmer/_oxli.pyx":104 + * cdef shared_ptr[GuardedKmerCompMap] locked + * lockedptr = self._tag_component_map.lock() + * if lockedptr: # <<<<<<<<<<<<<< + * it = deref(lockedptr).data.begin() + * while it != deref(lockedptr).data.end(): + */ + __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); + if (__pyx_t_1) { + + /* "khmer/_oxli.pyx":105 + * lockedptr = self._tag_component_map.lock() + * if lockedptr: + * it = deref(lockedptr).data.begin() # <<<<<<<<<<<<<< + * while it != deref(lockedptr).data.end(): + * yield deref(it).first, Component.create(deref(it).second) + */ + __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_lockedptr).data.begin(); + + /* "khmer/_oxli.pyx":106 + * if lockedptr: + * it = deref(lockedptr).data.begin() + * while it != deref(lockedptr).data.end(): # <<<<<<<<<<<<<< + * yield deref(it).first, Component.create(deref(it).second) + * inc(it) + */ + while (1) { + __pyx_t_2 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_lockedptr).data.end()) != 0); + if (!__pyx_t_2) break; + + /* "khmer/_oxli.pyx":107 + * it = deref(lockedptr).data.begin() + * while it != deref(lockedptr).data.end(): + * yield deref(it).first, Component.create(deref(it).second) # <<<<<<<<<<<<<< + * inc(it) + * else: + */ + __pyx_t_3 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it).first); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create((*__pyx_cur_scope->__pyx_v_it).second)); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L7_resume_from_yield:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 107, __pyx_L1_error) + + /* "khmer/_oxli.pyx":108 + * while it != deref(lockedptr).data.end(): + * yield deref(it).first, Component.create(deref(it).second) + * inc(it) # <<<<<<<<<<<<<< + * else: + * raise MemoryError("Can't locked underlying Component set") + */ + (++__pyx_cur_scope->__pyx_v_it); + } + + /* "khmer/_oxli.pyx":104 + * cdef shared_ptr[GuardedKmerCompMap] locked + * lockedptr = self._tag_component_map.lock() + * if lockedptr: # <<<<<<<<<<<<<< + * it = deref(lockedptr).data.begin() + * while it != deref(lockedptr).data.end(): + */ + goto __pyx_L4; + } + + /* "khmer/_oxli.pyx":110 + * inc(it) + * else: + * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< + * + * + */ + /*else*/ { + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_Raise(__pyx_t_5, 0, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 110, __pyx_L1_error) + } + __pyx_L4:; + if (1); else __pyx_cur_scope = __pyx_cur_scope; + + /* "khmer/_oxli.pyx":101 + * raise MemoryError("Can't locked underlying Component set") + * + * def tag_components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[GuardedKmerCompMap] locked + * lockedptr = self._tag_component_map.lock() + */ + + /* function exit code */ + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); __pyx_r = 0; + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":115 + * + * property n_components: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_components() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli.pyx":116 + * property n_components: + * def __get__(self): + * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":115 + * + * property n_components: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_components() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.n_components.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { + Py_ssize_t __pyx_v_length; + char *__pyx_v_data; + std::string __pyx_r; __Pyx_RefNannyDeclarations char *__pyx_t_1; __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); @@ -2258,6 +2867,7 @@ static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v __Pyx_RefNannyFinishContext(); return __pyx_r; } +static struct __pyx_vtabstruct_5khmer_5_oxli_Component __pyx_vtable_5khmer_5_oxli_Component; static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5khmer_5_oxli_Component *p; @@ -2269,6 +2879,7 @@ static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject } if (unlikely(!o)) return 0; p = ((struct __pyx_obj_5khmer_5_oxli_Component *)o); + p->__pyx_vtab = __pyx_vtabptr_5khmer_5_oxli_Component; new((void*)&(p->_this)) khmer::ComponentPtr(); if (unlikely(__pyx_pw_5khmer_5_oxli_9Component_1__cinit__(o, a, k) < 0)) goto bad; return o; @@ -2344,7 +2955,7 @@ static PyTypeObject __pyx_type_5khmer_5_oxli_Component = { 0, /*tp_as_number*/ &__pyx_tp_as_sequence_Component, /*tp_as_sequence*/ &__pyx_tp_as_mapping_Component, /*tp_as_mapping*/ - 0, /*tp_hash*/ + __pyx_pw_5khmer_5_oxli_9Component_8__hash__, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ @@ -2354,7 +2965,7 @@ static PyTypeObject __pyx_type_5khmer_5_oxli_Component = { 0, /*tp_doc*/ 0, /*tp_traverse*/ 0, /*tp_clear*/ - 0, /*tp_richcompare*/ + __pyx_pw_5khmer_5_oxli_9Component_10__richcmp__, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ __pyx_pw_5khmer_5_oxli_9Component_5__iter__, /*tp_iter*/ 0, /*tp_iternext*/ @@ -2424,6 +3035,8 @@ static PyMethodDef __pyx_methods_5khmer_5_oxli_StreamingPartitioner[] = { {"consume_sequence", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence, METH_O, 0}, {"get_tag_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5get_tag_component, METH_O, 0}, {"get_nearest_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_nearest_component, METH_O, 0}, + {"components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9components, METH_NOARGS, 0}, + {"tag_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12tag_components, METH_NOARGS, 0}, {0, 0, 0, 0} }; @@ -2598,6 +3211,230 @@ static PyTypeObject __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__ = { #endif }; +static struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *__pyx_freelist_5khmer_5_oxli___pyx_scope_struct_1_components[8]; +static int __pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components = 0; + +static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p; + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components)))) { + o = (PyObject*)__pyx_freelist_5khmer_5_oxli___pyx_scope_struct_1_components[--__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components]; + memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components)); + (void) PyObject_INIT(o, t); + PyObject_GC_Track(o); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o); + new((void*)&(p->__pyx_v_it)) std::set ::iterator(); + new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); + new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); + return o; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_1_components(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o; + PyObject_GC_UnTrack(o); + __Pyx_call_destructor(p->__pyx_v_it); + __Pyx_call_destructor(p->__pyx_v_locked); + __Pyx_call_destructor(p->__pyx_v_lockedptr); + Py_CLEAR(p->__pyx_v_self); + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components)))) { + __pyx_freelist_5khmer_5_oxli___pyx_scope_struct_1_components[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static int __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct_1_components(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o; + if (p->__pyx_v_self) { + e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct_1_components(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o; + tmp = ((PyObject*)p->__pyx_v_self); + p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyTypeObject __pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.__pyx_scope_struct_1_components", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_1_components, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct_1_components, /*tp_traverse*/ + __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct_1_components, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *__pyx_freelist_5khmer_5_oxli___pyx_scope_struct_2_tag_components[8]; +static int __pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components = 0; + +static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p; + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components)))) { + o = (PyObject*)__pyx_freelist_5khmer_5_oxli___pyx_scope_struct_2_tag_components[--__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components]; + memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components)); + (void) PyObject_INIT(o, t); + PyObject_GC_Track(o); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o); + new((void*)&(p->__pyx_v_it)) std::map ::iterator(); + new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); + new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); + return o; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o; + PyObject_GC_UnTrack(o); + __Pyx_call_destructor(p->__pyx_v_it); + __Pyx_call_destructor(p->__pyx_v_locked); + __Pyx_call_destructor(p->__pyx_v_lockedptr); + Py_CLEAR(p->__pyx_v_self); + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components)))) { + __pyx_freelist_5khmer_5_oxli___pyx_scope_struct_2_tag_components[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static int __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o; + if (p->__pyx_v_self) { + e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o; + tmp = ((PyObject*)p->__pyx_v_self); + p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyTypeObject __pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.__pyx_scope_struct_2_tag_components", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_2_tag_components, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct_2_tag_components, /*tp_traverse*/ + __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct_2_tag_components, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + static PyMethodDef __pyx_methods[] = { {0, 0, 0, 0} }; @@ -2621,14 +3458,21 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp_s_Can_t_locked_underlying_Componen, __pyx_k_Can_t_locked_underlying_Componen, sizeof(__pyx_k_Can_t_locked_underlying_Componen), 0, 0, 1, 0}, {&__pyx_n_s_Component___iter, __pyx_k_Component___iter, sizeof(__pyx_k_Component___iter), 0, 0, 1, 1}, {&__pyx_n_s_Countgraph, __pyx_k_Countgraph, sizeof(__pyx_k_Countgraph), 0, 0, 1, 1}, {&__pyx_n_s_MemoryError, __pyx_k_MemoryError, sizeof(__pyx_k_MemoryError), 0, 0, 1, 1}, {&__pyx_kp_s_Must_take_an_object_with_Hashtab, __pyx_k_Must_take_an_object_with_Hashtab, sizeof(__pyx_k_Must_take_an_object_with_Hashtab), 0, 0, 1, 0}, {&__pyx_n_s_Nodegraph, __pyx_k_Nodegraph, sizeof(__pyx_k_Nodegraph), 0, 0, 1, 1}, + {&__pyx_n_s_NotImplementedError, __pyx_k_NotImplementedError, sizeof(__pyx_k_NotImplementedError), 0, 0, 1, 1}, + {&__pyx_kp_s_Operator_not_available, __pyx_k_Operator_not_available, sizeof(__pyx_k_Operator_not_available), 0, 0, 1, 0}, + {&__pyx_n_s_StreamingPartitioner_components, __pyx_k_StreamingPartitioner_components, sizeof(__pyx_k_StreamingPartitioner_components), 0, 0, 1, 1}, + {&__pyx_n_s_StreamingPartitioner_tag_compone, __pyx_k_StreamingPartitioner_tag_compone, sizeof(__pyx_k_StreamingPartitioner_tag_compone), 0, 0, 1, 1}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, + {&__pyx_n_s_component_id, __pyx_k_component_id, sizeof(__pyx_k_component_id), 0, 0, 1, 1}, + {&__pyx_n_s_components, __pyx_k_components, sizeof(__pyx_k_components), 0, 0, 1, 1}, {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, {&__pyx_n_s_graph, __pyx_k_graph, sizeof(__pyx_k_graph), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, @@ -2637,7 +3481,9 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_khmer__oxli, __pyx_k_khmer__oxli, sizeof(__pyx_k_khmer__oxli), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_other, __pyx_k_other, sizeof(__pyx_k_other), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, + {&__pyx_n_s_tag_components, __pyx_k_tag_components, sizeof(__pyx_k_tag_components), 0, 0, 1, 1}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, @@ -2645,7 +3491,8 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 64, __pyx_L1_error) - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 53, __pyx_L1_error) + __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(1, 42, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 60, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -2655,49 +3502,82 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "khmer/_oxli.pyx":53 + /* "khmer/_oxli.pyx":42 + * return x.component_id == y.component_id + * else: + * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Operator_not_available); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple_); + __Pyx_GIVEREF(__pyx_tuple_); + + /* "khmer/_oxli.pyx":60 * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< * * */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 53, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple_); - __Pyx_GIVEREF(__pyx_tuple_); + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 60, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__2); + __Pyx_GIVEREF(__pyx_tuple__2); - /* "khmer/_oxli.pyx":62 + /* "khmer/_oxli.pyx":72 * * def consume_sequence(self, sequence): * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< * * def get_tag_component(self, kmer): */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 62, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__2); - __Pyx_GIVEREF(__pyx_tuple__2); + __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 72, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__3); + __Pyx_GIVEREF(__pyx_tuple__3); - /* "khmer/_oxli.pyx":66 + /* "khmer/_oxli.pyx":76 * def get_tag_component(self, kmer): - * cdef ComponentPtr comp - * comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if comp == NULL: + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if compptr == NULL: * return None */ - __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 66, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__3); - __Pyx_GIVEREF(__pyx_tuple__3); + __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__4); + __Pyx_GIVEREF(__pyx_tuple__4); - /* "khmer/_oxli.pyx":74 + /* "khmer/_oxli.pyx":84 * def get_nearest_component(self, kmer): - * cdef ComponentPtr comp - * comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if comp == NULL: + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if compptr == NULL: * return None */ - __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 74, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__4); - __Pyx_GIVEREF(__pyx_tuple__4); + __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__5); + __Pyx_GIVEREF(__pyx_tuple__5); + + /* "khmer/_oxli.pyx":99 + * inc(it) + * else: + * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< + * + * def tag_components(self): + */ + __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__6); + __Pyx_GIVEREF(__pyx_tuple__6); + + /* "khmer/_oxli.pyx":110 + * inc(it) + * else: + * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< + * + * + */ + __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__7); + __Pyx_GIVEREF(__pyx_tuple__7); __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -2707,6 +3587,7 @@ static int __Pyx_InitCachedConstants(void) { static int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(1, 1, __pyx_L1_error); + __pyx_int_2 = PyInt_FromLong(2); if (unlikely(!__pyx_int_2)) __PYX_ERR(1, 1, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -2797,17 +3678,26 @@ PyMODINIT_FUNC PyInit__oxli(void) /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 9, __pyx_L1_error) + __pyx_vtabptr_5khmer_5_oxli_Component = &__pyx_vtable_5khmer_5_oxli_Component; + __pyx_vtable_5khmer_5_oxli_Component.create = (struct __pyx_obj_5khmer_5_oxli_Component *(*)(khmer::ComponentPtr))__pyx_f_5khmer_5_oxli_9Component_create; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 10, __pyx_L1_error) __pyx_type_5khmer_5_oxli_Component.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 9, __pyx_L1_error) + if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_Component.tp_dict, __pyx_vtabptr_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 10, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 10, __pyx_L1_error) __pyx_ptype_5khmer_5_oxli_Component = &__pyx_type_5khmer_5_oxli_Component; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 43, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 51, __pyx_L1_error) __pyx_type_5khmer_5_oxli_StreamingPartitioner.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 43, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 51, __pyx_L1_error) __pyx_ptype_5khmer_5_oxli_StreamingPartitioner = &__pyx_type_5khmer_5_oxli_StreamingPartitioner; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 30, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 29, __pyx_L1_error) __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__.tp_print = 0; __pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = &__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components) < 0) __PYX_ERR(1, 90, __pyx_L1_error) + __pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components.tp_print = 0; + __pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components = &__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components) < 0) __PYX_ERR(1, 101, __pyx_L1_error) + __pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components.tp_print = 0; + __pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components = &__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components; /*--- Type import code ---*/ /*--- Variable import code ---*/ /*--- Function import code ---*/ @@ -2816,45 +3706,45 @@ PyMODINIT_FUNC PyInit__oxli(void) if (__Pyx_patch_abc() < 0) __PYX_ERR(1, 1, __pyx_L1_error) #endif - /* "khmer/_oxli.pyx":6 - * from libcpp.memory cimport unique_ptr, weak_ptr + /* "khmer/_oxli.pyx":7 + * from libc.stdint cimport uintptr_t * * from khmer._khmer import Countgraph # <<<<<<<<<<<<<< * from khmer._khmer import Nodegraph * */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 7, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_Countgraph); __Pyx_GIVEREF(__pyx_n_s_Countgraph); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_Countgraph); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 6, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 7, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(1, 6, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(1, 7, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "khmer/_oxli.pyx":7 + /* "khmer/_oxli.pyx":8 * * from khmer._khmer import Countgraph * from khmer._khmer import Nodegraph # <<<<<<<<<<<<<< * * cdef class Component: */ - __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_Nodegraph); __Pyx_GIVEREF(__pyx_n_s_Nodegraph); PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Nodegraph); - __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(1, 7, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(1, 8, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -3100,6 +3990,91 @@ static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, in return 0; } +/* PyIntBinop */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { + if (op1 == op2) { + Py_RETURN_TRUE; + } + #if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(op1))) { + const long b = intval; + long a = PyInt_AS_LONG(op1); + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + const long b = intval; + long a; + const digit* digits = ((PyLongObject*)op1)->ob_digit; + const Py_ssize_t size = Py_SIZE(op1); + if (likely(__Pyx_sst_abs(size) <= 1)) { + a = likely(size) ? digits[0] : 0; + if (size == -1) a = -a; + } else { + switch (size) { + case -2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 + default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); + #else + default: Py_RETURN_FALSE; + #endif + } + } + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = PyFloat_AS_DOUBLE(op1); + if ((double)a == (double)b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + return PyObject_RichCompare(op1, op2, Py_EQ); +} +#endif + /* PyObjectCall */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { @@ -3120,26 +4095,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg } #endif -/* GetModuleGlobalName */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS - result = PyDict_GetItem(__pyx_d, name); - if (likely(result)) { - Py_INCREF(result); - } else { -#else - result = PyObject_GetItem(__pyx_d, name); - if (!result) { - PyErr_Clear(); -#endif - result = __Pyx_GetBuiltinName(name); - } - return result; -} - /* PyErrFetchRestore */ - #if CYTHON_FAST_THREAD_STATE +#if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; tmp_type = tstate->curexc_type; @@ -3163,7 +4120,7 @@ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject #endif /* RaiseException */ - #if PY_MAJOR_VERSION < 3 +#if PY_MAJOR_VERSION < 3 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, CYTHON_UNUSED PyObject *cause) { __Pyx_PyThreadState_declare @@ -3325,6 +4282,42 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject } #endif +/* GetModuleGlobalName */ + static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + +/* SetVTable */ + static int __Pyx_SetVtable(PyObject *dict, void *vtable) { +#if PY_VERSION_HEX >= 0x02070000 + PyObject *ob = PyCapsule_New(vtable, 0, 0); +#else + PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); +#endif + if (!ob) + goto bad; + if (PyDict_SetItem(dict, __pyx_n_s_pyx_vtable, ob) < 0) + goto bad; + Py_DECREF(ob); + return 0; +bad: + Py_XDECREF(ob); + return -1; +} + /* Import */ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { PyObject *empty_list = 0; @@ -3574,6 +4567,37 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, Py_XDECREF(py_frame); } +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + } +} + /* CIntToPy */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value) { const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; diff --git a/khmer/_oxli.pyx b/khmer/_oxli.pyx index eec15f9e57..f1e4c691f5 100644 --- a/khmer/_oxli.pyx +++ b/khmer/_oxli.pyx @@ -2,6 +2,7 @@ import cython from cython.operator cimport dereference as deref, preincrement as inc from libc.limits cimport UINT_MAX from libcpp.memory cimport unique_ptr, weak_ptr +from libc.stdint cimport uintptr_t from khmer._khmer import Countgraph from khmer._khmer import Nodegraph @@ -13,8 +14,6 @@ cdef class Component: def __cinit__(self, Component other=None): if other is not None: self._this.reset(other._this.get()) - else: - self._this.reset(new CyComponent()) property component_id: def __get__(self): @@ -33,11 +32,20 @@ cdef class Component: yield deref(it) inc(it) + def __hash__(self): + return self._this.get() + + def __richcmp__(x, y, op): + if op == 2: + return x.component_id == y.component_id + else: + raise NotImplementedError('Operator not available.') -cdef Component build_component(ComponentPtr ptr): - cdef Component comp = Component() - comp._this.reset(ptr.get()) - return comp + @staticmethod + cdef Component create(ComponentPtr ptr): + cdef Component comp = Component() + comp._this = ptr + return comp cdef class StreamingPartitioner: @@ -45,7 +53,6 @@ cdef class StreamingPartitioner: cdef unique_ptr[CyStreamingPartitioner] _this cdef weak_ptr[ComponentPtrSet] _components cdef weak_ptr[GuardedKmerCompMap] _tag_component_map - cdef CyHashtable * _graph_ptr def __cinit__(self, graph): @@ -58,24 +65,51 @@ cdef class StreamingPartitioner: self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) + self._tag_component_map = deref(self._this).get_tag_component_map() + self._components = deref(self._this).get_component_set() + def consume_sequence(self, sequence): deref(self._this).consume_sequence(sequence.encode('utf-8')) def get_tag_component(self, kmer): - cdef ComponentPtr comp - comp = deref(self._this).get_tag_component(kmer.encode('utf-8')) - if comp == NULL: + cdef ComponentPtr compptr + compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + if compptr == NULL: return None else: - return build_component(comp) + return Component.create(compptr) def get_nearest_component(self, kmer): - cdef ComponentPtr comp - comp = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - if comp == NULL: + cdef ComponentPtr compptr + compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + if compptr == NULL: return None else: - return build_component(comp) + return Component.create(compptr) + + def components(self): + cdef shared_ptr[ComponentPtrSet] locked + lockedptr = self._components.lock() + if lockedptr: + it = deref(lockedptr).begin() + while it != deref(lockedptr).end(): + yield Component.create(deref(it)) + inc(it) + else: + raise MemoryError("Can't locked underlying Component set") + + def tag_components(self): + cdef shared_ptr[GuardedKmerCompMap] locked + lockedptr = self._tag_component_map.lock() + if lockedptr: + it = deref(lockedptr).data.begin() + while it != deref(lockedptr).data.end(): + yield deref(it).first, Component.create(deref(it).second) + inc(it) + else: + raise MemoryError("Can't locked underlying Component set") + + property n_components: def __get__(self): From def411a80d9554a89f0965854c8f28d633ae34eb Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 6 Nov 2016 17:37:35 -0800 Subject: [PATCH 009/185] Fix Component n_merges tracking --- lib/partitioning.hh | 2 +- tests/test_cython_oxli.py | 72 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/lib/partitioning.hh b/lib/partitioning.hh index ca85a98562..e42f0b7a07 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -121,7 +121,7 @@ class Component { continue; } this->add_tag(other->tags); - this->n_merges = other->get_n_merges() + 1; + this->n_merges += other->get_n_merges() + 1; } } diff --git a/tests/test_cython_oxli.py b/tests/test_cython_oxli.py index b92b756b98..9759e22072 100644 --- a/tests/test_cython_oxli.py +++ b/tests/test_cython_oxli.py @@ -8,6 +8,7 @@ from khmer import _oxli from khmer.khmer_args import estimate_optimal_with_K_and_f as optimal_fp from khmer import reverse_complement as revcomp +from khmer import reverse_hash as revhash from . import khmer_tst_utils as utils from .graph_features import * @@ -42,15 +43,76 @@ def test_two_components(self, random_sequence): sp.consume_sequence(comp2) assert sp.n_components == 2 - def test_get_nearest_component(self, random_sequence): - comp1 = random_sequence() + def test_components_iter(self, random_sequence): + comp1 = random_sequence() + comp2 = random_sequence(exclude=comp1) + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = _oxli.StreamingPartitioner(cg) + + sp.consume_sequence(comp1) + sp.consume_sequence(comp2) + assert sp.n_components == 2 + + comps = list(sp.components()) + assert len(comps) == 2 + + def test_tag_components_iter(self, random_sequence): + comp1 = random_sequence() + comp2 = random_sequence(exclude=comp1) + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = _oxli.StreamingPartitioner(cg) + + sp.consume_sequence(comp1) + sp.consume_sequence(comp2) + assert sp.n_components == 2 + + tags = [] + comps = set() + for tag, comp in sp.tag_components(): + tags.append(tag) + comps.add(comp) + + assert sum([len([tag for tag in comp]) for comp in comps]) == len(tags) + assert len(comps) == 2 + assert len(tags) == sum([len(c) for c in comps]) + + def test_merge_components(self, random_sequence): + comp1 = random_sequence() comp2 = random_sequence(exclude=comp1) cg = khmer.Nodegraph(K, 1e5, 4) sp = _oxli.StreamingPartitioner(cg) - assert False + sp.consume_sequence(comp1) sp.consume_sequence(comp2) + assert sp.n_components == 2 + + sp.consume_sequence(comp1 + comp2) + assert sp.n_components == 1 + + comps = list(sp.components()) + assert len(comps) == 1 + + def test_get_nearest_component(self, random_sequence): + seq1 = random_sequence() + seq2 = random_sequence(exclude=seq1) + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = _oxli.StreamingPartitioner(cg) + + sp.consume_sequence(seq1) + sp.consume_sequence(seq2) + + c1 = sp.get_nearest_component(seq1[:K]) + c2 = sp.get_nearest_component(seq2[:K]) + assert c1.component_id != c2.component_id + + for tag in c1: + assert utils._contains_rc(seq1, revhash(tag, K)) + assert not utils._contains_rc(seq2, revhash(tag, K)) - c = sp.get_nearest_component(comp1[:K]) - assert c.component_id == 0 + for tag in c2: + assert utils._contains_rc(seq2, revhash(tag, K)) + assert not utils._contains_rc(seq1, revhash(tag, K)) From 9a3d8275c496dc163a6015fe5809abd309a34a5d Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 6 Nov 2016 17:38:05 -0800 Subject: [PATCH 010/185] Add tests for container access, n_merges, component merging, tag access --- tests/test_cython_oxli.py | 75 ++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/tests/test_cython_oxli.py b/tests/test_cython_oxli.py index 9759e22072..59dbe9037a 100644 --- a/tests/test_cython_oxli.py +++ b/tests/test_cython_oxli.py @@ -57,6 +57,17 @@ def test_components_iter(self, random_sequence): comps = list(sp.components()) assert len(comps) == 2 + def test_component_n_tags(self, random_sequence): + seq = random_sequence() + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = _oxli.StreamingPartitioner(cg) + sp.consume_sequence(seq) + + tags = [t for t,c in sp.tag_components()] + comp = sp.get_nearest_component(seq[:K]) + assert len(tags) == len(comp) + def test_tag_components_iter(self, random_sequence): comp1 = random_sequence() comp2 = random_sequence(exclude=comp1) @@ -78,41 +89,73 @@ def test_tag_components_iter(self, random_sequence): assert len(comps) == 2 assert len(tags) == sum([len(c) for c in comps]) + def test_get_nearest_component(self, random_sequence): + seq1 = random_sequence() + seq2 = random_sequence(exclude=seq1) + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = _oxli.StreamingPartitioner(cg) + + sp.consume_sequence(seq1) + sp.consume_sequence(seq2) + + c1 = sp.get_nearest_component(seq1[:K]) + c2 = sp.get_nearest_component(seq2[:K]) + assert c1.component_id != c2.component_id + + for tag in c1: + assert utils._contains_rc(seq1, revhash(tag, K)) + assert not utils._contains_rc(seq2, revhash(tag, K)) + + for tag in c2: + assert utils._contains_rc(seq2, revhash(tag, K)) + assert not utils._contains_rc(seq1, revhash(tag, K)) + def test_merge_components(self, random_sequence): - comp1 = random_sequence() - comp2 = random_sequence(exclude=comp1) + seq1 = random_sequence() + seq2 = random_sequence(exclude=seq1) cg = khmer.Nodegraph(K, 1e5, 4) sp = _oxli.StreamingPartitioner(cg) - sp.consume_sequence(comp1) - sp.consume_sequence(comp2) + sp.consume_sequence(seq1) + sp.consume_sequence(seq2) assert sp.n_components == 2 - sp.consume_sequence(comp1 + comp2) + sp.consume_sequence(seq1 + seq2) assert sp.n_components == 1 comps = list(sp.components()) assert len(comps) == 1 - def test_get_nearest_component(self, random_sequence): - seq1 = random_sequence() + assert comps[0].n_merges == 1 + + def test_multi_merge_components(self, random_sequence): + seq1 = random_sequence() seq2 = random_sequence(exclude=seq1) + seq3 = random_sequence(exclude=seq1+seq2) cg = khmer.Nodegraph(K, 1e5, 4) sp = _oxli.StreamingPartitioner(cg) sp.consume_sequence(seq1) sp.consume_sequence(seq2) + assert sp.n_components == 2 + + sp.consume_sequence(seq1 + seq2) + assert sp.n_components == 1 + + sp.consume_sequence(seq3) + assert sp.n_components == 2 + + sp.consume_sequence(seq2 + seq3) + assert sp.n_components == 1 + + comps = list(sp.components()) + assert len(comps) == 1 + + assert comps[0].n_merges == 2 + - c1 = sp.get_nearest_component(seq1[:K]) - c2 = sp.get_nearest_component(seq2[:K]) - assert c1.component_id != c2.component_id - for tag in c1: - assert utils._contains_rc(seq1, revhash(tag, K)) - assert not utils._contains_rc(seq2, revhash(tag, K)) - for tag in c2: - assert utils._contains_rc(seq2, revhash(tag, K)) - assert not utils._contains_rc(seq1, revhash(tag, K)) From 4648333d06b229be233d04b8d6a2d75a6c230e50 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 6 Nov 2016 17:39:10 -0800 Subject: [PATCH 011/185] Add Cython to dev-packages --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ab69638caf..68e6930e25 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ PYSOURCES=$(filter-out khmer/_version.py, \ SOURCES=$(PYSOURCES) $(CPPSOURCES) setup.py DEVPKGS=pep8==1.6.2 diff_cover autopep8 pylint coverage gcovr pytest \ - pydocstyle screed pyenchant + pydocstyle screed pyenchant cython GCOVRURL=git+https://github.com/nschum/gcovr.git@never-executed-branches VERSION=$(shell ./setup.py version | grep Version | awk '{print $$4}' \ From 3d13cb800edaca9e1183b84affae3cdbc1637181 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 7 Nov 2016 18:10:20 -0800 Subject: [PATCH 012/185] Make ComponentPtr set compare on the component id; fix consume_sequence missing the last k-mer --- lib/partitioning.cc | 46 +++++++++++++++++++++++++++++++-------------- lib/partitioning.hh | 33 +++++++++++++++++++++++++------- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 6362e2e339..6cee89c013 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -12,13 +12,18 @@ using namespace khmer; uint64_t Component::n_created = 0; +bool ComponentPtrCompare::operator() (const ComponentPtr& lhs, + const ComponentPtr& rhs) const { + return *lhs < *rhs; +} -std::ostream& operator<< (std::ostream& stream, Component& comp) { +inline std::ostream& operator<< (std::ostream& stream, Component& comp) { stream << ""; return stream; } + StreamingPartitioner::StreamingPartitioner(Hashtable * graph) : graph(graph), _tag_density(DEFAULT_TAG_DENSITY) { @@ -63,6 +68,9 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) * as the intersect operator. */ //if (auto graphptr = graph.lock()) { +#if(SP_DEBUG) + std::cout << "Consume sequence." << std::endl; +#endif if(graph != NULL) { KmerIterator kmers(seq.c_str(), graph->ksize()); unsigned int since = 1; @@ -79,6 +87,7 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) Kmer kmer = kmers.next(); tags.insert(kmer); //always tag the first k-mer bool is_new_kmer = graph->test_and_set_bits(kmer); + search_from.push(kmer); while(!kmers.done()) { bool kmer_tagged = false; @@ -118,20 +127,19 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) since = 1; } - is_new_kmer = graph->test_and_set_bits(kmer); kmer = kmers.next(); + is_new_kmer = graph->test_and_set_bits(kmer); } tags.insert(kmer); // always tag the last k-mer - if (is_new_kmer || !found_tag_in_territory) { - search_from.push(kmer); - } + search_from.push(kmer); intersection.clear(); #if(DEBUG_SP) std::cout << "Done iterating k-mers" << std::endl; + std::cout << tags.size() << " tags in sequence" << std::endl; #endif // Now search for tagged nodes connected to U. - find_connected_tags(search_from, tags, seen); + find_connected_tags(search_from, tags, seen, false); // Now resolve components. First, get components from existing tags. ComponentPtrSet comps; @@ -154,6 +162,10 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) #endif } +#if(DEBUG_SP) + std::cout << comps.size() << " unique components." << std::endl; +#endif + if (comps.size() == 0) { comp = std::make_shared(); #if(DEBUG_SP) @@ -166,16 +178,21 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) #if(DEBUG_SP) std::cout << "Merge into: " << *comp << std::endl; #endif - if (comps.size() == 1) { // map the new tags to this component - comp->add_tag(tags); - map_tags_to_component(tags, comp); - } else { - merge_components(comp, comps); - } + comp->add_tag(tags); + merge_components(comp, comps); } // (re)map all the tags to the component map_tags_to_component(tags, comp); + +#if(DEBUG_SP) + for (auto tag: tags) { + std::cout << tag << " -> " << *(tag_component_map->get(tag)) << std::endl; + } + for (auto c : *components) { + std::cout << *c << std::endl; + } +#endif } else { throw khmer_ptr_exception("Hashtable has been deleted."); } @@ -183,14 +200,14 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) void StreamingPartitioner::merge_components(ComponentPtr root, - std::set comps) + ComponentPtrSet comps) { #if(DEBUG_SP) std::cout << "Merge components." << std::endl; #endif root->merge(comps); for (auto other : comps) { - if (other == root) { + if (*other == *root) { continue; } #if(DEBUG_SP) @@ -299,6 +316,7 @@ void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, for (unsigned int i = 0; i class GuardedKmerMap { @@ -95,6 +96,21 @@ class GuardedKmerMap { }; +class Component; +typedef std::shared_ptr ComponentPtr; + + +class ComponentPtrCompare { + public: + bool operator() (const ComponentPtr& lhs, const ComponentPtr& rhs) const; +}; + + +class ComponentPtrCompare; +typedef std::set ComponentPtrSet; +typedef GuardedKmerMap GuardedKmerCompMap; + + class Component { private: @@ -115,9 +131,9 @@ class Component { tags.clear(); // maybe not necessary? } - void merge(std::set> other_comps) { + void merge(ComponentPtrSet other_comps) { for (auto other : other_comps) { - if (other.get() == this) { + if (*other == *this) { continue; } this->add_tag(other->tags); @@ -143,14 +159,17 @@ class Component { return n_merges; } - friend std::ostream& operator<< (std::ostream& stream, const Component& comp) ; -}; + friend bool operator==(const Component& lhs, const Component& rhs) { + return lhs.component_id == rhs.component_id; + } + friend bool operator<(const Component& lhs, const Component& rhs) { + return lhs.component_id < rhs.component_id; + } + friend std::ostream& operator<< (std::ostream& stream, const Component& comp); +}; -typedef std::shared_ptr ComponentPtr; -typedef std::set ComponentPtrSet; -typedef GuardedKmerMap GuardedKmerCompMap; class StreamingPartitioner { From 6e7201e55eaf594dfbb6d025e31b7056b6a5cb21 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 7 Nov 2016 18:45:32 -0800 Subject: [PATCH 013/185] Pass sets by reference, map ALL tags from merged components to proper new component --- khmer/_oxli.pxd | 2 +- lib/partitioning.cc | 9 +++++---- lib/partitioning.hh | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index 195b9a2644..12a5bae0b6 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -64,7 +64,7 @@ cdef extern from "partitioning.hh" namespace "khmer": CyStreamingPartitioner(CyHashtable * ) except +MemoryError void consume_sequence(string&) except +MemoryError - void map_tags_to_component(set[HashIntoType], ComponentPtr) + void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) void find_connected_tags(queue[Kmer]&, set[HashIntoType]&, set[HashIntoType]&) except +MemoryError diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 6cee89c013..6391073fec 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -50,7 +50,7 @@ StreamingPartitioner::StreamingPartitioner(Hashtable * graph) : } -void StreamingPartitioner::map_tags_to_component(std::set tags, +void StreamingPartitioner::map_tags_to_component(std::set& tags, ComponentPtr& comp) { for (auto tag: tags) { @@ -178,12 +178,13 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) #if(DEBUG_SP) std::cout << "Merge into: " << *comp << std::endl; #endif - // map the new tags to this component + // map the new tags to this component comp->add_tag(tags); merge_components(comp, comps); } // (re)map all the tags to the component map_tags_to_component(tags, comp); + map_tags_to_component(comp->tags, comp); #if(DEBUG_SP) for (auto tag: tags) { @@ -199,8 +200,8 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) } -void StreamingPartitioner::merge_components(ComponentPtr root, - ComponentPtrSet comps) +void StreamingPartitioner::merge_components(ComponentPtr& root, + ComponentPtrSet& comps) { #if(DEBUG_SP) std::cout << "Merge components." << std::endl; diff --git a/lib/partitioning.hh b/lib/partitioning.hh index b564fe04f2..fbba749245 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -190,7 +190,7 @@ class StreamingPartitioner { explicit StreamingPartitioner(Hashtable * graph); void consume_sequence(const std::string& seq); - void map_tags_to_component(std::set tags, ComponentPtr& comp); + void map_tags_to_component(std::set& tags, ComponentPtr& comp); void find_connected_tags(KmerQueue& node_q, std::set& found_tags, std::set& seen, @@ -200,7 +200,7 @@ class StreamingPartitioner { return components->size(); } - void merge_components(ComponentPtr root, ComponentPtrSet comps); + void merge_components(ComponentPtr& root, ComponentPtrSet& comps); ComponentPtr get_tag_component(HashIntoType tag) const; ComponentPtr get_tag_component(std::string& tag) const; From 0479b8fba5df628f71746e8175844f8ea2af1c5a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 7 Nov 2016 18:46:47 -0800 Subject: [PATCH 014/185] Add coverage guarantee to reads gen --- tests/graph_features.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/graph_features.py b/tests/graph_features.py index 72c610d134..cd5222f46f 100644 --- a/tests/graph_features.py +++ b/tests/graph_features.py @@ -131,8 +131,17 @@ def add_seen(kmer): return ''.join(seq) -def reads(sequence, L=100, N=100): +def reads(sequence, L=100, N=100, dbg_cover=False): positions = list(range(len(sequence) - L)) + if dbg_cover is True: + for start in range(0, len(sequence), K): + read = sequence[start:start+L] + if len(read) < K: + read = sequence[-L:] + yield read + N -= 1 + if N < 0: + return for i in range(N): start = random.choice(positions) yield sequence[start:start + L] From b18d20de439591e8351085bdae569f71d0165b99 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 7 Nov 2016 18:47:33 -0800 Subject: [PATCH 015/185] Add tests for edge cases of k-1 neighbors and components from simulated reads --- tests/test_cython_oxli.py | 99 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/tests/test_cython_oxli.py b/tests/test_cython_oxli.py index 59dbe9037a..3fec6edd5b 100644 --- a/tests/test_cython_oxli.py +++ b/tests/test_cython_oxli.py @@ -19,6 +19,21 @@ def teardown(): utils.cleanup() + +@pytest.fixture +def partitioner(graph): + sp = _oxli.StreamingPartitioner(graph) + return graph, sp + + +@pytest.fixture +def single_component(partitioner, random_sequence): + graph, partitioner = partitioner + sequence = random_sequence() + partitioner.consume_sequence(sequence) + return graph, partitioner, sequence + + class TestStreamingPartitionerBasic: def test_one_component(self, known_sequence): @@ -140,11 +155,13 @@ def test_multi_merge_components(self, random_sequence): sp.consume_sequence(seq1) sp.consume_sequence(seq2) - assert sp.n_components == 2 + sp.consume_sequence(seq3) + assert sp.n_components == 3 - sp.consume_sequence(seq1 + seq2) + sp.consume_sequence(seq1 + seq2 + seq3) assert sp.n_components == 1 + ''' sp.consume_sequence(seq3) assert sp.n_components == 2 @@ -155,7 +172,85 @@ def test_multi_merge_components(self, random_sequence): assert len(comps) == 1 assert comps[0].n_merges == 2 + ''' + + def test_nomerge_k_minus_2_overlap(self, single_component, random_sequence): + '''Test that components are not merged when they have a length K-2 overlap. + ''' + + graph, partitioner, seq = single_component + first = seq[:K-2] + neighbor = random_sequence(exclude=seq) + first + + assert partitioner.n_components == 1 + partitioner.consume_sequence(neighbor) + print(seq, neighbor, graph.assemble_linear_path(seq[:K]), sep='\n') + assert partitioner.n_components == 2 + + @pytest.mark.parametrize("where", ["beginning", "end"]) + def test_merge_k_minus_1_overlap(self, single_component, random_sequence, + where): + '''Test that components are merged when they have a length K-1 overlap. + ''' + + graph, partitioner, seq = single_component + if where == "beginning": + overlap = seq[:K-1] + neighbor = random_sequence(exclude=seq) + overlap + else: + overlap = seq[-K+1:] + neighbor = overlap + random_sequence(exclude=seq) + + assert partitioner.n_components == 1 + partitioner.consume_sequence(neighbor) + path = graph.assemble_linear_path(seq[:K]) + assert partitioner.n_components == 1 + + def test_merge_k_overlap(self, single_component, random_sequence): + '''Test that components are merged when they have a length K overlap. + ''' + + graph, partitioner, seq = single_component + first = seq[:K] + neighbor = random_sequence(exclude=seq) + first + + assert partitioner.n_components == 1 + partitioner.consume_sequence(neighbor) + print(seq, neighbor, graph.assemble_linear_path(seq[:K]), sep='\n') + assert partitioner.n_components == 1 + + + @pytest.mark.parametrize("n_reads", list(range(100, 1001, 100))) + def test_one_component_from_reads(self, random_sequence, n_reads): + seq = random_sequence() + seq_reads = list(reads(seq, dbg_cover=True, N=n_reads)) + + G = khmer.Nodegraph(K, 1e6, 4) + sp = _oxli.StreamingPartitioner(G) + for read in seq_reads: + sp.consume_sequence(read) + + assert sp.n_components == 1 + + @pytest.mark.parametrize("n_components", list(range(1, 10))) + def test_streaming_multicomponents(self, random_sequence, n_components): + # get n_components disconnected sequences + seqs = [] + for _ in range(n_components): + seqs.append(random_sequence(exclude=''.join(seqs))) + seq_reads = [] + for seq in seqs: + seq_reads.extend(list(reads(seq, dbg_cover=True, N=100))) + random.shuffle(seq_reads) + G = khmer.Nodegraph(K, 1e6, 4) + sp = _oxli.StreamingPartitioner(G) + for read in seq_reads: + assert len(read) >= K + sp.consume_sequence(read) + for seq in seqs: + assert G.assemble_linear_path(seq[:K]) == seq + assert sp.n_components == n_components From efab0556820e2f176c5b4d971602434e3c42f7a9 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 7 Nov 2016 23:32:49 -0800 Subject: [PATCH 016/185] Imrpve some code clarity, add tracker for number of components destroyed to help track potential memory leaks --- lib/partitioning.cc | 55 ++++++++++++++++++++------------------------- lib/partitioning.hh | 19 +++++++++------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 6391073fec..06fadf6985 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -12,6 +12,8 @@ using namespace khmer; uint64_t Component::n_created = 0; +uint64_t Component::n_destroyed = 0; + bool ComponentPtrCompare::operator() (const ComponentPtr& lhs, const ComponentPtr& rhs) const { return *lhs < *rhs; @@ -142,8 +144,7 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) find_connected_tags(search_from, tags, seen, false); // Now resolve components. First, get components from existing tags. - ComponentPtrSet comps; - ComponentPtr comp; + ComponentPtrSet found_comps; #if(DEBUG_SP) std::cout << "Get found comps: " << std::endl; #endif @@ -151,11 +152,12 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) #if(DEBUG_SP) std::cout << "Tag: " << tag; #endif + ComponentPtr comp; if ((comp = tag_component_map->get(tag)) != NULL) { #if(DEBUG_SP) std::cout << "->" << *comp; #endif - comps.insert(comp); + found_comps.insert(comp); } #if(DEBUG_SP) std::cout << std::endl; @@ -163,37 +165,30 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) } #if(DEBUG_SP) - std::cout << comps.size() << " unique components." << std::endl; + std::cout << found_comps.size() << " unique components." << std::endl; #endif - if (comps.size() == 0) { - comp = std::make_shared(); + if (found_comps.size() == 0) { + ComponentPtr new_comp = std::make_shared(); #if(DEBUG_SP) - std::cout << "Build new comp: " << *comp << std::endl; + std::cout << "Build new comp: " << *new_comp << std::endl; #endif - components->insert(comp); + components->insert(new_comp); + map_tags_to_component(tags, new_comp); } else { // get the first component - comp = *(comps.begin()); + ComponentPtr root_comp = *(found_comps.begin()); #if(DEBUG_SP) - std::cout << "Merge into: " << *comp << std::endl; + std::cout << "Merge into: " << *root_comp << std::endl; #endif // map the new tags to this component - comp->add_tag(tags); - merge_components(comp, comps); + root_comp->add_tag(tags); + map_tags_to_component(tags, root_comp); + if (found_comps.size() > 1) { + merge_components(root_comp, found_comps); + } } - // (re)map all the tags to the component - map_tags_to_component(tags, comp); - map_tags_to_component(comp->tags, comp); -#if(DEBUG_SP) - for (auto tag: tags) { - std::cout << tag << " -> " << *(tag_component_map->get(tag)) << std::endl; - } - for (auto c : *components) { - std::cout << *c << std::endl; - } -#endif } else { throw khmer_ptr_exception("Hashtable has been deleted."); } @@ -203,19 +198,17 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) void StreamingPartitioner::merge_components(ComponentPtr& root, ComponentPtrSet& comps) { -#if(DEBUG_SP) - std::cout << "Merge components." << std::endl; -#endif - root->merge(comps); for (auto other : comps) { if (*other == *root) { continue; } -#if(DEBUG_SP) - std::cout << "Erase " << *other << std::endl; -#endif - components->erase(other); + root->add_tag(other->tags); // transfer the tags from the other comp + map_tags_to_component(other->tags, root); // set the other's tags to point to root + components->erase(other); // remove other component entirely + } + comps.clear(); // should call destructor on all the merged comps, unless they have + // and active Python wrapper; this leaves them as sole owners } diff --git a/lib/partitioning.hh b/lib/partitioning.hh index fbba749245..c0d8038fd3 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -116,19 +116,19 @@ class Component { private: static uint64_t n_created; - uint64_t n_merges; + static uint64_t n_destroyed; public: const uint64_t component_id; std::set tags; - explicit Component(): component_id(n_created), n_merges(0) { + explicit Component(): component_id(n_created) { n_created++; } ~Component() { - tags.clear(); // maybe not necessary? + n_destroyed++; } void merge(ComponentPtrSet other_comps) { @@ -137,10 +137,17 @@ class Component { continue; } this->add_tag(other->tags); - this->n_merges += other->get_n_merges() + 1; } } + uint64_t get_n_created() const { + return n_created; + } + + uint64_t get_n_destroyed() const { + return n_destroyed; + } + void add_tag(HashIntoType tag) { tags.insert(tag); } @@ -155,10 +162,6 @@ class Component { return tags.size(); } - uint64_t get_n_merges() const { - return n_merges; - } - friend bool operator==(const Component& lhs, const Component& rhs) { return lhs.component_id == rhs.component_id; } From 8d213665dfbf4a4d23b8d6a0f2c4920e8ad3de85 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 7 Nov 2016 23:33:27 -0800 Subject: [PATCH 017/185] Remove n_merges and expose n_destroyed to Python --- khmer/_oxli.pxd | 3 ++- khmer/_oxli.pyx | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index 12a5bae0b6..6cc952f131 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -48,7 +48,8 @@ cdef extern from "partitioning.hh" namespace "khmer": void add_tag(HashIntoType) void add_tag(set[HashIntoType]) uint64_t get_n_tags() const - uint64_t get_n_merges() const + uint64_t get_n_created() const + uint64_t get_n_destroyed() const ctypedef shared_ptr[CyComponent] ComponentPtr ctypedef set[ComponentPtr] ComponentPtrSet diff --git a/khmer/_oxli.pyx b/khmer/_oxli.pyx index f1e4c691f5..ad050676f3 100644 --- a/khmer/_oxli.pyx +++ b/khmer/_oxli.pyx @@ -19,9 +19,13 @@ cdef class Component: def __get__(self): return deref(self._this).component_id - property n_merges: + property _n_created: def __get__(self): - return deref(self._this).get_n_merges() + return deref(self._this).get_n_created() + + property _n_destroyed: + def __get__(self): + return deref(self._this).get_n_destroyed() def __len__(self): return deref(self._this).get_n_tags() @@ -109,8 +113,6 @@ cdef class StreamingPartitioner: else: raise MemoryError("Can't locked underlying Component set") - - property n_components: def __get__(self): return deref(self._this).get_n_components() From fa9fde215fb94f61138f5923d0aca623d409829b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 7 Nov 2016 23:37:02 -0800 Subject: [PATCH 018/185] Manually collect garbage between tests to check for leaked Components --- khmer/_oxli.cpp | 414 ++++++++++++++++++++++---------------- tests/test_cython_oxli.py | 32 +-- 2 files changed, 257 insertions(+), 189 deletions(-) diff --git a/khmer/_oxli.cpp b/khmer/_oxli.cpp index 5bf2ebc039..e2d1a4fb5c 100644 --- a/khmer/_oxli.cpp +++ b/khmer/_oxli.cpp @@ -6,7 +6,7 @@ "define_macros": [ [ "VERSION", - "2.0+522.g1967b74.dirty" + "2.0+531.gb18d20d.dirty" ] ], "depends": [ @@ -716,7 +716,7 @@ struct __pyx_obj_5khmer_5_oxli_Component { }; -/* "khmer/_oxli.pyx":51 +/* "khmer/_oxli.pyx":55 * * * cdef class StreamingPartitioner: # <<<<<<<<<<<<<< @@ -732,7 +732,7 @@ struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner { }; -/* "khmer/_oxli.pyx":29 +/* "khmer/_oxli.pyx":33 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< @@ -746,7 +746,7 @@ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ { }; -/* "khmer/_oxli.pyx":90 +/* "khmer/_oxli.pyx":94 * return Component.create(compptr) * * def components(self): # <<<<<<<<<<<<<< @@ -762,7 +762,7 @@ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components { }; -/* "khmer/_oxli.pyx":101 +/* "khmer/_oxli.pyx":105 * raise MemoryError("Can't locked underlying Component set") * * def tag_components(self): # <<<<<<<<<<<<<< @@ -1177,7 +1177,8 @@ static PyObject *__pyx_n_s_throw; static PyObject *__pyx_kp_s_utf_8; static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ @@ -1352,7 +1353,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct * def __get__(self): * return deref(self._this).component_id # <<<<<<<<<<<<<< * - * property n_merges: + * property _n_created: */ __Pyx_XDECREF(__pyx_r); __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 20, __pyx_L1_error) @@ -1382,40 +1383,40 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct /* "khmer/_oxli.pyx":23 * - * property n_merges: + * property _n_created: * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_merges() + * return deref(self._this).get_n_created() * */ /* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_8n_merges_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_8n_merges_1__get__(PyObject *__pyx_v_self) { +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10_n_created_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10_n_created_1__get__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); /* "khmer/_oxli.pyx":24 - * property n_merges: + * property _n_created: * def __get__(self): - * return deref(self._this).get_n_merges() # <<<<<<<<<<<<<< + * return deref(self._this).get_n_created() # <<<<<<<<<<<<<< * - * def __len__(self): + * property _n_destroyed: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_merges()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 24, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_created()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -1423,16 +1424,16 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __py /* "khmer/_oxli.pyx":23 * - * property n_merges: + * property _n_created: * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_merges() + * return deref(self._this).get_n_created() * */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.Component.n_merges.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("khmer._oxli.Component._n_created.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); @@ -1440,8 +1441,68 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_8n_merges___get__(struct __py return __pyx_r; } -/* "khmer/_oxli.pyx":26 - * return deref(self._this).get_n_merges() +/* "khmer/_oxli.pyx":27 + * + * property _n_destroyed: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_destroyed() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12_n_destroyed_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12_n_destroyed_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli.pyx":28 + * property _n_destroyed: + * def __get__(self): + * return deref(self._this).get_n_destroyed() # <<<<<<<<<<<<<< + * + * def __len__(self): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_destroyed()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 28, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":27 + * + * property _n_destroyed: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_destroyed() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.Component._n_destroyed.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":30 + * return deref(self._this).get_n_destroyed() * * def __len__(self): # <<<<<<<<<<<<<< * return deref(self._this).get_n_tags() @@ -1466,7 +1527,7 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "khmer/_oxli.pyx":27 + /* "khmer/_oxli.pyx":31 * * def __len__(self): * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< @@ -1476,8 +1537,8 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k __pyx_r = (*__pyx_v_self->_this).get_n_tags(); goto __pyx_L0; - /* "khmer/_oxli.pyx":26 - * return deref(self._this).get_n_merges() + /* "khmer/_oxli.pyx":30 + * return deref(self._this).get_n_destroyed() * * def __len__(self): # <<<<<<<<<<<<<< * return deref(self._this).get_n_tags() @@ -1491,7 +1552,7 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k } static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "khmer/_oxli.pyx":29 +/* "khmer/_oxli.pyx":33 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< @@ -1521,7 +1582,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5k if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(1, 29, __pyx_L1_error) + __PYX_ERR(1, 33, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } @@ -1529,7 +1590,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5k __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 29, __pyx_L1_error) + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 33, __pyx_L1_error) __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -1561,9 +1622,9 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 29, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 33, __pyx_L1_error) - /* "khmer/_oxli.pyx":30 + /* "khmer/_oxli.pyx":34 * * def __iter__(self): * it = deref(self._this).tags.begin() # <<<<<<<<<<<<<< @@ -1572,7 +1633,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj */ __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_self->_this).tags.begin(); - /* "khmer/_oxli.pyx":31 + /* "khmer/_oxli.pyx":35 * def __iter__(self): * it = deref(self._this).tags.begin() * while it != deref(self._this).tags.end(): # <<<<<<<<<<<<<< @@ -1583,14 +1644,14 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj __pyx_t_1 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_self->_this).tags.end()) != 0); if (!__pyx_t_1) break; - /* "khmer/_oxli.pyx":32 + /* "khmer/_oxli.pyx":36 * it = deref(self._this).tags.begin() * while it != deref(self._this).tags.end(): * yield deref(it) # <<<<<<<<<<<<<< * inc(it) * */ - __pyx_t_2 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 32, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 36, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; @@ -1600,9 +1661,9 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L6_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 32, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 36, __pyx_L1_error) - /* "khmer/_oxli.pyx":33 + /* "khmer/_oxli.pyx":37 * while it != deref(self._this).tags.end(): * yield deref(it) * inc(it) # <<<<<<<<<<<<<< @@ -1613,7 +1674,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj } if (1); else __pyx_cur_scope = __pyx_cur_scope; - /* "khmer/_oxli.pyx":29 + /* "khmer/_oxli.pyx":33 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< @@ -1635,7 +1696,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj return __pyx_r; } -/* "khmer/_oxli.pyx":35 +/* "khmer/_oxli.pyx":39 * inc(it) * * def __hash__(self): # <<<<<<<<<<<<<< @@ -1661,7 +1722,7 @@ static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5k __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__hash__", 0); - /* "khmer/_oxli.pyx":36 + /* "khmer/_oxli.pyx":40 * * def __hash__(self): * return self._this.get() # <<<<<<<<<<<<<< @@ -1671,7 +1732,7 @@ static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5k __pyx_r = ((uintptr_t)__pyx_v_self->_this.get()); goto __pyx_L0; - /* "khmer/_oxli.pyx":35 + /* "khmer/_oxli.pyx":39 * inc(it) * * def __hash__(self): # <<<<<<<<<<<<<< @@ -1686,7 +1747,7 @@ static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5k return __pyx_r; } -/* "khmer/_oxli.pyx":38 +/* "khmer/_oxli.pyx":42 * return self._this.get() * * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< @@ -1701,7 +1762,7 @@ static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10__richcmp__(PyObject *__pyx PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); - __pyx_v_op = __Pyx_PyInt_From_int(__pyx_arg_op); if (unlikely(!__pyx_v_op)) __PYX_ERR(1, 38, __pyx_L3_error) + __pyx_v_op = __Pyx_PyInt_From_int(__pyx_arg_op); if (unlikely(!__pyx_v_op)) __PYX_ERR(1, 42, __pyx_L3_error) __Pyx_GOTREF(__pyx_v_op); goto __pyx_L4_argument_unpacking_done; __pyx_L3_error:; @@ -1726,20 +1787,20 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_ PyObject *__pyx_t_4 = NULL; __Pyx_RefNannySetupContext("__richcmp__", 0); - /* "khmer/_oxli.pyx":39 + /* "khmer/_oxli.pyx":43 * * def __richcmp__(x, y, op): * if op == 2: # <<<<<<<<<<<<<< * return x.component_id == y.component_id * else: */ - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_v_op, __pyx_int_2, 2, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 39, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_v_op, __pyx_int_2, 2, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 43, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(1, 39, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(1, 43, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "khmer/_oxli.pyx":40 + /* "khmer/_oxli.pyx":44 * def __richcmp__(x, y, op): * if op == 2: * return x.component_id == y.component_id # <<<<<<<<<<<<<< @@ -1747,18 +1808,18 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_ * raise NotImplementedError('Operator not available.') */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_x, __pyx_n_s_component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 40, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_x, __pyx_n_s_component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_y, __pyx_n_s_component_id); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 40, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_y, __pyx_n_s_component_id); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 44, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_RichCompare(__pyx_t_1, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 40, __pyx_L1_error) + __pyx_t_4 = PyObject_RichCompare(__pyx_t_1, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 44, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":39 + /* "khmer/_oxli.pyx":43 * * def __richcmp__(x, y, op): * if op == 2: # <<<<<<<<<<<<<< @@ -1767,7 +1828,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_ */ } - /* "khmer/_oxli.pyx":42 + /* "khmer/_oxli.pyx":46 * return x.component_id == y.component_id * else: * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< @@ -1775,14 +1836,14 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_ * @staticmethod */ /*else*/ { - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_NotImplementedError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 42, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_NotImplementedError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 46, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 42, __pyx_L1_error) + __PYX_ERR(1, 46, __pyx_L1_error) } - /* "khmer/_oxli.pyx":38 + /* "khmer/_oxli.pyx":42 * return self._this.get() * * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< @@ -1803,7 +1864,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_ return __pyx_r; } -/* "khmer/_oxli.pyx":45 +/* "khmer/_oxli.pyx":49 * * @staticmethod * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< @@ -1818,19 +1879,19 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Componen PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("create", 0); - /* "khmer/_oxli.pyx":46 + /* "khmer/_oxli.pyx":50 * @staticmethod * cdef Component create(ComponentPtr ptr): * cdef Component comp = Component() # <<<<<<<<<<<<<< * comp._this = ptr * return comp */ - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_Component), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 46, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_Component), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_comp = ((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_t_1); __pyx_t_1 = 0; - /* "khmer/_oxli.pyx":47 + /* "khmer/_oxli.pyx":51 * cdef Component create(ComponentPtr ptr): * cdef Component comp = Component() * comp._this = ptr # <<<<<<<<<<<<<< @@ -1839,7 +1900,7 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Componen */ __pyx_v_comp->_this = __pyx_v_ptr; - /* "khmer/_oxli.pyx":48 + /* "khmer/_oxli.pyx":52 * cdef Component comp = Component() * comp._this = ptr * return comp # <<<<<<<<<<<<<< @@ -1851,7 +1912,7 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Componen __pyx_r = __pyx_v_comp; goto __pyx_L0; - /* "khmer/_oxli.pyx":45 + /* "khmer/_oxli.pyx":49 * * @staticmethod * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< @@ -1871,7 +1932,7 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Componen return __pyx_r; } -/* "khmer/_oxli.pyx":58 +/* "khmer/_oxli.pyx":62 * cdef CyHashtable * _graph_ptr * * def __cinit__(self, graph): # <<<<<<<<<<<<<< @@ -1904,7 +1965,7 @@ static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__ else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 58, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 62, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { goto __pyx_L5_argtuple_error; @@ -1915,7 +1976,7 @@ static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__ } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 58, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 62, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -1940,16 +2001,16 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ khmer::StreamingPartitioner *__pyx_t_6; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "khmer/_oxli.pyx":59 + /* "khmer/_oxli.pyx":63 * * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< * raise ValueError('Must take an object with Hashtable *') * */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 59, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 63, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 59, __pyx_L1_error) + __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 63, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_4 = (__pyx_t_3 != 0); if (!__pyx_t_4) { @@ -1957,9 +2018,9 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_1 = __pyx_t_4; goto __pyx_L4_bool_binop_done; } - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 59, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 63, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 59, __pyx_L1_error) + __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 63, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_3 = (__pyx_t_4 != 0); __pyx_t_1 = __pyx_t_3; @@ -1967,20 +2028,20 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_3 = ((!__pyx_t_1) != 0); if (__pyx_t_3) { - /* "khmer/_oxli.pyx":60 + /* "khmer/_oxli.pyx":64 * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< * * */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 60, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 64, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 60, __pyx_L1_error) + __PYX_ERR(1, 64, __pyx_L1_error) - /* "khmer/_oxli.pyx":59 + /* "khmer/_oxli.pyx":63 * * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< @@ -1989,7 +2050,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ } - /* "khmer/_oxli.pyx":63 + /* "khmer/_oxli.pyx":67 * * * cdef CyCpHashtable_Object* ptr = graph # <<<<<<<<<<<<<< @@ -1998,7 +2059,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - /* "khmer/_oxli.pyx":64 + /* "khmer/_oxli.pyx":68 * * cdef CyCpHashtable_Object* ptr = graph * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< @@ -2008,7 +2069,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_5 = (*__pyx_v_ptr).hashtable; __pyx_v_self->_graph_ptr = __pyx_t_5; - /* "khmer/_oxli.pyx":66 + /* "khmer/_oxli.pyx":70 * self._graph_ptr = deref(ptr).hashtable * * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) # <<<<<<<<<<<<<< @@ -2019,11 +2080,11 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_6 = new khmer::StreamingPartitioner(__pyx_v_self->_graph_ptr); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 66, __pyx_L1_error) + __PYX_ERR(1, 70, __pyx_L1_error) } __pyx_v_self->_this.reset(__pyx_t_6); - /* "khmer/_oxli.pyx":68 + /* "khmer/_oxli.pyx":72 * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) * * self._tag_component_map = deref(self._this).get_tag_component_map() # <<<<<<<<<<<<<< @@ -2032,7 +2093,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ __pyx_v_self->_tag_component_map = (*__pyx_v_self->_this).get_tag_component_map(); - /* "khmer/_oxli.pyx":69 + /* "khmer/_oxli.pyx":73 * * self._tag_component_map = deref(self._this).get_tag_component_map() * self._components = deref(self._this).get_component_set() # <<<<<<<<<<<<<< @@ -2041,7 +2102,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ __pyx_v_self->_components = (*__pyx_v_self->_this).get_component_set(); - /* "khmer/_oxli.pyx":58 + /* "khmer/_oxli.pyx":62 * cdef CyHashtable * _graph_ptr * * def __cinit__(self, graph): # <<<<<<<<<<<<<< @@ -2061,7 +2122,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ return __pyx_r; } -/* "khmer/_oxli.pyx":71 +/* "khmer/_oxli.pyx":75 * self._components = deref(self._this).get_component_set() * * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< @@ -2090,28 +2151,28 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence std::string __pyx_t_3; __Pyx_RefNannySetupContext("consume_sequence", 0); - /* "khmer/_oxli.pyx":72 + /* "khmer/_oxli.pyx":76 * * def consume_sequence(self, sequence): * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< * * def get_tag_component(self, kmer): */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 72, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 72, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 72, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; try { (*__pyx_v_self->_this).consume_sequence(__pyx_t_3); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 72, __pyx_L1_error) + __PYX_ERR(1, 76, __pyx_L1_error) } - /* "khmer/_oxli.pyx":71 + /* "khmer/_oxli.pyx":75 * self._components = deref(self._this).get_component_set() * * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< @@ -2133,7 +2194,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence return __pyx_r; } -/* "khmer/_oxli.pyx":74 +/* "khmer/_oxli.pyx":78 * deref(self._this).consume_sequence(sequence.encode('utf-8')) * * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< @@ -2164,23 +2225,23 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen int __pyx_t_4; __Pyx_RefNannySetupContext("get_tag_component", 0); - /* "khmer/_oxli.pyx":76 + /* "khmer/_oxli.pyx":80 * def get_tag_component(self, kmer): * cdef ComponentPtr compptr * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< * if compptr == NULL: * return None */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_compptr = (*__pyx_v_self->_this).get_tag_component(__pyx_t_3); - /* "khmer/_oxli.pyx":77 + /* "khmer/_oxli.pyx":81 * cdef ComponentPtr compptr * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) * if compptr == NULL: # <<<<<<<<<<<<<< @@ -2190,7 +2251,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); if (__pyx_t_4) { - /* "khmer/_oxli.pyx":78 + /* "khmer/_oxli.pyx":82 * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) * if compptr == NULL: * return None # <<<<<<<<<<<<<< @@ -2202,7 +2263,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen __pyx_r = Py_None; goto __pyx_L0; - /* "khmer/_oxli.pyx":77 + /* "khmer/_oxli.pyx":81 * cdef ComponentPtr compptr * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) * if compptr == NULL: # <<<<<<<<<<<<<< @@ -2211,7 +2272,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen */ } - /* "khmer/_oxli.pyx":80 + /* "khmer/_oxli.pyx":84 * return None * else: * return Component.create(compptr) # <<<<<<<<<<<<<< @@ -2220,14 +2281,14 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen */ /*else*/ { __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 80, __pyx_L1_error) + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; } - /* "khmer/_oxli.pyx":74 + /* "khmer/_oxli.pyx":78 * deref(self._this).consume_sequence(sequence.encode('utf-8')) * * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< @@ -2247,7 +2308,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen return __pyx_r; } -/* "khmer/_oxli.pyx":82 +/* "khmer/_oxli.pyx":86 * return Component.create(compptr) * * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< @@ -2278,23 +2339,23 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp int __pyx_t_4; __Pyx_RefNannySetupContext("get_nearest_component", 0); - /* "khmer/_oxli.pyx":84 + /* "khmer/_oxli.pyx":88 * def get_nearest_component(self, kmer): * cdef ComponentPtr compptr * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< * if compptr == NULL: * return None */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 84, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 84, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 84, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 88, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_compptr = (*__pyx_v_self->_this).get_nearest_component(__pyx_t_3); - /* "khmer/_oxli.pyx":85 + /* "khmer/_oxli.pyx":89 * cdef ComponentPtr compptr * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) * if compptr == NULL: # <<<<<<<<<<<<<< @@ -2304,7 +2365,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); if (__pyx_t_4) { - /* "khmer/_oxli.pyx":86 + /* "khmer/_oxli.pyx":90 * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) * if compptr == NULL: * return None # <<<<<<<<<<<<<< @@ -2316,7 +2377,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp __pyx_r = Py_None; goto __pyx_L0; - /* "khmer/_oxli.pyx":85 + /* "khmer/_oxli.pyx":89 * cdef ComponentPtr compptr * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) * if compptr == NULL: # <<<<<<<<<<<<<< @@ -2325,7 +2386,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp */ } - /* "khmer/_oxli.pyx":88 + /* "khmer/_oxli.pyx":92 * return None * else: * return Component.create(compptr) # <<<<<<<<<<<<<< @@ -2334,14 +2395,14 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp */ /*else*/ { __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; } - /* "khmer/_oxli.pyx":82 + /* "khmer/_oxli.pyx":86 * return Component.create(compptr) * * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< @@ -2362,7 +2423,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp } static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "khmer/_oxli.pyx":90 +/* "khmer/_oxli.pyx":94 * return Component.create(compptr) * * def components(self): # <<<<<<<<<<<<<< @@ -2392,7 +2453,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struc if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(1, 90, __pyx_L1_error) + __PYX_ERR(1, 94, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } @@ -2400,7 +2461,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struc __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_components, __pyx_n_s_StreamingPartitioner_components, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 90, __pyx_L1_error) + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_components, __pyx_n_s_StreamingPartitioner_components, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 94, __pyx_L1_error) __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -2433,9 +2494,9 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 90, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 94, __pyx_L1_error) - /* "khmer/_oxli.pyx":92 + /* "khmer/_oxli.pyx":96 * def components(self): * cdef shared_ptr[ComponentPtrSet] locked * lockedptr = self._components.lock() # <<<<<<<<<<<<<< @@ -2444,7 +2505,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py */ __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_components.lock(); - /* "khmer/_oxli.pyx":93 + /* "khmer/_oxli.pyx":97 * cdef shared_ptr[ComponentPtrSet] locked * lockedptr = self._components.lock() * if lockedptr: # <<<<<<<<<<<<<< @@ -2454,7 +2515,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); if (__pyx_t_1) { - /* "khmer/_oxli.pyx":94 + /* "khmer/_oxli.pyx":98 * lockedptr = self._components.lock() * if lockedptr: * it = deref(lockedptr).begin() # <<<<<<<<<<<<<< @@ -2463,7 +2524,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py */ __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_lockedptr).begin(); - /* "khmer/_oxli.pyx":95 + /* "khmer/_oxli.pyx":99 * if lockedptr: * it = deref(lockedptr).begin() * while it != deref(lockedptr).end(): # <<<<<<<<<<<<<< @@ -2474,14 +2535,14 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py __pyx_t_2 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_lockedptr).end()) != 0); if (!__pyx_t_2) break; - /* "khmer/_oxli.pyx":96 + /* "khmer/_oxli.pyx":100 * it = deref(lockedptr).begin() * while it != deref(lockedptr).end(): * yield Component.create(deref(it)) # <<<<<<<<<<<<<< * inc(it) * else: */ - __pyx_t_3 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create((*__pyx_cur_scope->__pyx_v_it))); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 96, __pyx_L1_error) + __pyx_t_3 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create((*__pyx_cur_scope->__pyx_v_it))); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_r = __pyx_t_3; __pyx_t_3 = 0; @@ -2491,9 +2552,9 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L7_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 96, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 100, __pyx_L1_error) - /* "khmer/_oxli.pyx":97 + /* "khmer/_oxli.pyx":101 * while it != deref(lockedptr).end(): * yield Component.create(deref(it)) * inc(it) # <<<<<<<<<<<<<< @@ -2503,7 +2564,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py (++__pyx_cur_scope->__pyx_v_it); } - /* "khmer/_oxli.pyx":93 + /* "khmer/_oxli.pyx":97 * cdef shared_ptr[ComponentPtrSet] locked * lockedptr = self._components.lock() * if lockedptr: # <<<<<<<<<<<<<< @@ -2513,7 +2574,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py goto __pyx_L4; } - /* "khmer/_oxli.pyx":99 + /* "khmer/_oxli.pyx":103 * inc(it) * else: * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< @@ -2521,16 +2582,16 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py * def tag_components(self): */ /*else*/ { - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 99, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 103, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 99, __pyx_L1_error) + __PYX_ERR(1, 103, __pyx_L1_error) } __pyx_L4:; if (1); else __pyx_cur_scope = __pyx_cur_scope; - /* "khmer/_oxli.pyx":90 + /* "khmer/_oxli.pyx":94 * return Component.create(compptr) * * def components(self): # <<<<<<<<<<<<<< @@ -2553,7 +2614,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py } static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "khmer/_oxli.pyx":101 +/* "khmer/_oxli.pyx":105 * raise MemoryError("Can't locked underlying Component set") * * def tag_components(self): # <<<<<<<<<<<<<< @@ -2583,7 +2644,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components( if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(1, 101, __pyx_L1_error) + __PYX_ERR(1, 105, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } @@ -2591,7 +2652,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components( __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_tag_components, __pyx_n_s_StreamingPartitioner_tag_compone, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 101, __pyx_L1_error) + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_tag_components, __pyx_n_s_StreamingPartitioner_tag_compone, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 105, __pyx_L1_error) __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -2626,9 +2687,9 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 101, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 105, __pyx_L1_error) - /* "khmer/_oxli.pyx":103 + /* "khmer/_oxli.pyx":107 * def tag_components(self): * cdef shared_ptr[GuardedKmerCompMap] locked * lockedptr = self._tag_component_map.lock() # <<<<<<<<<<<<<< @@ -2637,7 +2698,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py */ __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_tag_component_map.lock(); - /* "khmer/_oxli.pyx":104 + /* "khmer/_oxli.pyx":108 * cdef shared_ptr[GuardedKmerCompMap] locked * lockedptr = self._tag_component_map.lock() * if lockedptr: # <<<<<<<<<<<<<< @@ -2647,7 +2708,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); if (__pyx_t_1) { - /* "khmer/_oxli.pyx":105 + /* "khmer/_oxli.pyx":109 * lockedptr = self._tag_component_map.lock() * if lockedptr: * it = deref(lockedptr).data.begin() # <<<<<<<<<<<<<< @@ -2656,7 +2717,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py */ __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_lockedptr).data.begin(); - /* "khmer/_oxli.pyx":106 + /* "khmer/_oxli.pyx":110 * if lockedptr: * it = deref(lockedptr).data.begin() * while it != deref(lockedptr).data.end(): # <<<<<<<<<<<<<< @@ -2667,18 +2728,18 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py __pyx_t_2 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_lockedptr).data.end()) != 0); if (!__pyx_t_2) break; - /* "khmer/_oxli.pyx":107 + /* "khmer/_oxli.pyx":111 * it = deref(lockedptr).data.begin() * while it != deref(lockedptr).data.end(): * yield deref(it).first, Component.create(deref(it).second) # <<<<<<<<<<<<<< * inc(it) * else: */ - __pyx_t_3 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it).first); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 107, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it).first); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create((*__pyx_cur_scope->__pyx_v_it).second)); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 107, __pyx_L1_error) + __pyx_t_4 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create((*__pyx_cur_scope->__pyx_v_it).second)); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); @@ -2694,9 +2755,9 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L7_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 107, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 111, __pyx_L1_error) - /* "khmer/_oxli.pyx":108 + /* "khmer/_oxli.pyx":112 * while it != deref(lockedptr).data.end(): * yield deref(it).first, Component.create(deref(it).second) * inc(it) # <<<<<<<<<<<<<< @@ -2706,7 +2767,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py (++__pyx_cur_scope->__pyx_v_it); } - /* "khmer/_oxli.pyx":104 + /* "khmer/_oxli.pyx":108 * cdef shared_ptr[GuardedKmerCompMap] locked * lockedptr = self._tag_component_map.lock() * if lockedptr: # <<<<<<<<<<<<<< @@ -2716,24 +2777,24 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py goto __pyx_L4; } - /* "khmer/_oxli.pyx":110 + /* "khmer/_oxli.pyx":114 * inc(it) * else: * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< * - * + * property n_components: */ /*else*/ { - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 110, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 114, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_Raise(__pyx_t_5, 0, 0, 0); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 110, __pyx_L1_error) + __PYX_ERR(1, 114, __pyx_L1_error) } __pyx_L4:; if (1); else __pyx_cur_scope = __pyx_cur_scope; - /* "khmer/_oxli.pyx":101 + /* "khmer/_oxli.pyx":105 * raise MemoryError("Can't locked underlying Component set") * * def tag_components(self): # <<<<<<<<<<<<<< @@ -2757,7 +2818,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py return __pyx_r; } -/* "khmer/_oxli.pyx":115 +/* "khmer/_oxli.pyx":117 * * property n_components: * def __get__(self): # <<<<<<<<<<<<<< @@ -2784,20 +2845,20 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - /* "khmer/_oxli.pyx":116 + /* "khmer/_oxli.pyx":118 * property n_components: * def __get__(self): * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 116, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 118, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":115 + /* "khmer/_oxli.pyx":117 * * property n_components: * def __get__(self): # <<<<<<<<<<<<<< @@ -2903,8 +2964,12 @@ static PyObject *__pyx_getprop_5khmer_5_oxli_9Component_component_id(PyObject *o return __pyx_pw_5khmer_5_oxli_9Component_12component_id_1__get__(o); } -static PyObject *__pyx_getprop_5khmer_5_oxli_9Component_n_merges(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_9Component_8n_merges_1__get__(o); +static PyObject *__pyx_getprop_5khmer_5_oxli_9Component__n_created(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_9Component_10_n_created_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_9Component__n_destroyed(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_9Component_12_n_destroyed_1__get__(o); } static PyMethodDef __pyx_methods_5khmer_5_oxli_Component[] = { @@ -2913,7 +2978,8 @@ static PyMethodDef __pyx_methods_5khmer_5_oxli_Component[] = { static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_Component[] = { {(char *)"component_id", __pyx_getprop_5khmer_5_oxli_9Component_component_id, 0, (char *)0, 0}, - {(char *)"n_merges", __pyx_getprop_5khmer_5_oxli_9Component_n_merges, 0, (char *)0, 0}, + {(char *)"_n_created", __pyx_getprop_5khmer_5_oxli_9Component__n_created, 0, (char *)0, 0}, + {(char *)"_n_destroyed", __pyx_getprop_5khmer_5_oxli_9Component__n_destroyed, 0, (char *)0, 0}, {0, 0, 0, 0, 0} }; @@ -3490,9 +3556,9 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 64, __pyx_L1_error) - __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(1, 42, __pyx_L1_error) - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 60, __pyx_L1_error) + __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 65, __pyx_L1_error) + __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(1, 46, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 64, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -3502,80 +3568,80 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "khmer/_oxli.pyx":42 + /* "khmer/_oxli.pyx":46 * return x.component_id == y.component_id * else: * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< * * @staticmethod */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Operator_not_available); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 42, __pyx_L1_error) + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Operator_not_available); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 46, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); - /* "khmer/_oxli.pyx":60 + /* "khmer/_oxli.pyx":64 * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< * * */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 60, __pyx_L1_error) + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 64, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); - /* "khmer/_oxli.pyx":72 + /* "khmer/_oxli.pyx":76 * * def consume_sequence(self, sequence): * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< * * def get_tag_component(self, kmer): */ - __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 72, __pyx_L1_error) + __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3); - /* "khmer/_oxli.pyx":76 + /* "khmer/_oxli.pyx":80 * def get_tag_component(self, kmer): * cdef ComponentPtr compptr * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< * if compptr == NULL: * return None */ - __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__4); __Pyx_GIVEREF(__pyx_tuple__4); - /* "khmer/_oxli.pyx":84 + /* "khmer/_oxli.pyx":88 * def get_nearest_component(self, kmer): * cdef ComponentPtr compptr * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< * if compptr == NULL: * return None */ - __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 84, __pyx_L1_error) + __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); - /* "khmer/_oxli.pyx":99 + /* "khmer/_oxli.pyx":103 * inc(it) * else: * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< * * def tag_components(self): */ - __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 99, __pyx_L1_error) + __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 103, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); - /* "khmer/_oxli.pyx":110 + /* "khmer/_oxli.pyx":114 * inc(it) * else: * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< * - * + * property n_components: */ - __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 110, __pyx_L1_error) + __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 114, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); __Pyx_RefNannyFinishContext(); @@ -3685,17 +3751,17 @@ PyMODINIT_FUNC PyInit__oxli(void) if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_Component.tp_dict, __pyx_vtabptr_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 10, __pyx_L1_error) if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 10, __pyx_L1_error) __pyx_ptype_5khmer_5_oxli_Component = &__pyx_type_5khmer_5_oxli_Component; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 51, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 55, __pyx_L1_error) __pyx_type_5khmer_5_oxli_StreamingPartitioner.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 51, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 55, __pyx_L1_error) __pyx_ptype_5khmer_5_oxli_StreamingPartitioner = &__pyx_type_5khmer_5_oxli_StreamingPartitioner; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 29, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 33, __pyx_L1_error) __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__.tp_print = 0; __pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = &__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components) < 0) __PYX_ERR(1, 90, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components) < 0) __PYX_ERR(1, 94, __pyx_L1_error) __pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components.tp_print = 0; __pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components = &__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components) < 0) __PYX_ERR(1, 101, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components) < 0) __PYX_ERR(1, 105, __pyx_L1_error) __pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components.tp_print = 0; __pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components = &__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components; /*--- Type import code ---*/ diff --git a/tests/test_cython_oxli.py b/tests/test_cython_oxli.py index 3fec6edd5b..b12c0717cb 100644 --- a/tests/test_cython_oxli.py +++ b/tests/test_cython_oxli.py @@ -1,6 +1,7 @@ from __future__ import print_function from __future__ import absolute_import +import gc import itertools import random @@ -36,6 +37,15 @@ def single_component(partitioner, random_sequence): class TestStreamingPartitionerBasic: + def teardown_method(self, method): + # Force garbage to collect. When Python component objects exist and + # their underlying c++ Component objects are destroyed, the Python + # wrapper becomes the sole owner of the pointer. By manually collecting + # garbage between tests we assure that these objects are freed, and we + # can properly test the _n_destroyed property to make sure there are no + # real memory leaks. + gc.collect() + def test_one_component(self, known_sequence): inpath = utils.get_test_data('random-20-a.fa') @@ -143,7 +153,6 @@ def test_merge_components(self, random_sequence): comps = list(sp.components()) assert len(comps) == 1 - assert comps[0].n_merges == 1 def test_multi_merge_components(self, random_sequence): seq1 = random_sequence() @@ -161,19 +170,6 @@ def test_multi_merge_components(self, random_sequence): sp.consume_sequence(seq1 + seq2 + seq3) assert sp.n_components == 1 - ''' - sp.consume_sequence(seq3) - assert sp.n_components == 2 - - sp.consume_sequence(seq2 + seq3) - assert sp.n_components == 1 - - comps = list(sp.components()) - assert len(comps) == 1 - - assert comps[0].n_merges == 2 - ''' - def test_nomerge_k_minus_2_overlap(self, single_component, random_sequence): '''Test that components are not merged when they have a length K-2 overlap. ''' @@ -234,7 +230,7 @@ def test_one_component_from_reads(self, random_sequence, n_reads): @pytest.mark.parametrize("n_components", list(range(1, 10))) def test_streaming_multicomponents(self, random_sequence, n_components): - # get n_components disconnected sequences + '''Test with many components from reads, and check for memory leaks.''' seqs = [] for _ in range(n_components): seqs.append(random_sequence(exclude=''.join(seqs))) @@ -252,5 +248,11 @@ def test_streaming_multicomponents(self, random_sequence, n_components): sp.consume_sequence(read) for seq in seqs: + # make sure we got the complete component assert G.assemble_linear_path(seq[:K]) == seq assert sp.n_components == n_components + + comps = list(sp.components()) + comp = comps[0] + assert len(comps) == n_components + assert sp.n_components == (comp._n_created - comp._n_destroyed) From 438826f0baa47a13ecb329e6c2b75949e249e6e5 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 10 Nov 2016 16:30:44 -0800 Subject: [PATCH 019/185] Add functions to check average component coverage, write out results --- khmer/_oxli.pxd | 5 +++- khmer/_oxli.pyx | 49 +++++++++++++++++++++++++++++++++++++++ tests/test_cython_oxli.py | 29 +++++++++++++++++++---- 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index 6cc952f131..5c24d1e19c 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -18,6 +18,8 @@ cdef extern from "khmer.hh" namespace "khmer": cdef extern from "hashtable.hh" namespace "khmer": cdef cppclass CyHashtable "khmer::Hashtable": CyHashtable(WordLength) + const BoundedCounterType get_count(const char *) const + const BoundedCounterType get_count(HashIntoType) const cdef extern from "_khmer.hh": @@ -69,7 +71,8 @@ cdef extern from "partitioning.hh" namespace "khmer": void find_connected_tags(queue[Kmer]&, set[HashIntoType]&, set[HashIntoType]&) except +MemoryError - uint64_t get_n_components() + uint64_t get_n_components() const + uint64_t get_n_tags() const ComponentPtr get_tag_component(string&) const ComponentPtr get_nearest_component(string&) const diff --git a/khmer/_oxli.pyx b/khmer/_oxli.pyx index ad050676f3..24db7e97c3 100644 --- a/khmer/_oxli.pyx +++ b/khmer/_oxli.pyx @@ -3,6 +3,7 @@ from cython.operator cimport dereference as deref, preincrement as inc from libc.limits cimport UINT_MAX from libcpp.memory cimport unique_ptr, weak_ptr from libc.stdint cimport uintptr_t +from libc.stdio cimport FILE, fopen, fwrite, fclose, stdout, stderr, fprintf from khmer._khmer import Countgraph from khmer._khmer import Nodegraph @@ -51,6 +52,26 @@ cdef class Component: comp._this = ptr return comp + @staticmethod + cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CyHashtable * graph): + cdef uint64_t n_tags = deref(comp).get_n_tags() + cdef vector[BoundedCounterType] counts + counts = vector[BoundedCounterType](n_tags) + cdef int idx + cdef uint64_t tag + for idx, tag in deref(comp).tags: + counts[idx] = deref(graph).get_count(tag) + return counts + + @staticmethod + cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): + cdef uint64_t n_tags = deref(comp).get_n_tags() + cdef float acc = 0 + cdef uint64_t tag + for tag in deref(comp).tags: + acc += deref(graph).get_count(tag) + return acc / n_tags + cdef class StreamingPartitioner: @@ -58,6 +79,7 @@ cdef class StreamingPartitioner: cdef weak_ptr[ComponentPtrSet] _components cdef weak_ptr[GuardedKmerCompMap] _tag_component_map cdef CyHashtable * _graph_ptr + cdef readonly uint64_t n_consumed def __cinit__(self, graph): if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): @@ -71,9 +93,11 @@ cdef class StreamingPartitioner: self._tag_component_map = deref(self._this).get_tag_component_map() self._components = deref(self._this).get_component_set() + self.n_consumed = 0 def consume_sequence(self, sequence): deref(self._this).consume_sequence(sequence.encode('utf-8')) + self.n_consumed += 1 def get_tag_component(self, kmer): cdef ComponentPtr compptr @@ -113,7 +137,32 @@ cdef class StreamingPartitioner: else: raise MemoryError("Can't locked underlying Component set") + def write_components(self, filename): + cdef FILE* fp + fp = fopen(filename.encode('utf-8'), 'wb') + if fp == NULL: + raise IOError("Can't open file.") + + cdef ComponentPtr cmpptr + cdef shared_ptr[ComponentPtrSet] locked + lockedptr = self._components.lock() + + if lockedptr: + it = deref(lockedptr).begin() + while it != deref(lockedptr).end(): + cmpptr = deref(it) + + fprintf(fp, "%u,%u,%f\n", + deref(cmpptr).component_id, + deref(cmpptr).get_n_tags(), + Component._mean_tag_count(cmpptr, self._graph_ptr)) + inc(it) + fclose(fp) + property n_components: def __get__(self): return deref(self._this).get_n_components() + property n_tags: + def __get__(self): + return deref(self._this).get_n_tags() diff --git a/tests/test_cython_oxli.py b/tests/test_cython_oxli.py index b12c0717cb..c66cebe11e 100644 --- a/tests/test_cython_oxli.py +++ b/tests/test_cython_oxli.py @@ -246,13 +246,34 @@ def test_streaming_multicomponents(self, random_sequence, n_components): for read in seq_reads: assert len(read) >= K sp.consume_sequence(read) - - for seq in seqs: - # make sure we got the complete component - assert G.assemble_linear_path(seq[:K]) == seq assert sp.n_components == n_components comps = list(sp.components()) comp = comps[0] assert len(comps) == n_components assert sp.n_components == (comp._n_created - comp._n_destroyed) + assert sp.n_consumed == len(seq_reads) + + @pytest.mark.parametrize("n_components", list(range(1,101, 20))) + @pytest.mark.parametrize("cov", [1,10,20]) + def test_write_components(self, random_sequence, cov, n_components, tmpdir): + outfn = tmpdir.join('counts.csv') + seqs = [] + for _ in range(n_components): + seqs.append(random_sequence(exclude=''.join(seqs))) + G = khmer.Countgraph(K, 1e6, 4) + sp = _oxli.StreamingPartitioner(G) + + for seq in seqs: + for _ in range(cov): + sp.consume_sequence(seq) + for seq in seqs: + (med, _, _) = G.get_median_count(seq) + assert med == cov + assert sp.n_components == n_components + + sp.write_components(str(outfn)) + results = [line.strip().split(',') for line in outfn.open()] + assert len(results) == n_components + for row in results: + assert abs(float(row[2])-float(cov)) < 2 From b03946b507e99889d8a113a34cd8e1b799c0b514 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 10 Nov 2016 16:31:14 -0800 Subject: [PATCH 020/185] Minor reorganization of control flow, always add at least one tag --- lib/partitioning.cc | 31 +++++++++++++++++++------------ lib/partitioning.hh | 8 ++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 06fadf6985..b7a75cbee7 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -75,7 +75,7 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) #endif if(graph != NULL) { KmerIterator kmers(seq.c_str(), graph->ksize()); - unsigned int since = 1; + unsigned int since = _tag_density / 2 + 1; std::set tags; std::set seen; @@ -86,12 +86,14 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) bool found_tag_in_territory = false; // First check if we overlap any tags - Kmer kmer = kmers.next(); - tags.insert(kmer); //always tag the first k-mer - bool is_new_kmer = graph->test_and_set_bits(kmer); - search_from.push(kmer); - - while(!kmers.done()) { + //Kmer kmer = kmers.next(); + //tags.insert(kmer); //always tag the first k-mer + //bool is_new_kmer = graph->test_and_set_bits(kmer); + //search_from.push(kmer); + + do { + Kmer kmer = kmers.next(); + bool is_new_kmer = graph->test_and_set_bits(kmer); bool kmer_tagged = false; if (is_new_kmer) { @@ -128,13 +130,18 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) tags.insert(kmer); since = 1; } + } while (!kmers.done()); - kmer = kmers.next(); - is_new_kmer = graph->test_and_set_bits(kmer); + if (tags.size() == 0) { + // if no tags are found or seeded, add the middle k-mer + uint32_t start = (seq.length() - graph->ksize()) / 2; + std::string kmer = seq.substr(start, graph->ksize()); + tags.insert(graph->hash_dna(kmer.c_str())); } - tags.insert(kmer); // always tag the last k-mer - search_from.push(kmer); - intersection.clear(); + + //tags.insert(kmer); // always tag the last k-mer + //search_from.push(kmer); + //intersection.clear(); #if(DEBUG_SP) std::cout << "Done iterating k-mers" << std::endl; diff --git a/lib/partitioning.hh b/lib/partitioning.hh index c0d8038fd3..61d47b0e0e 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -93,6 +93,10 @@ class GuardedKmerMap { return get(kmer) != NULL; } + uint64_t size() const { + return data.size(); + } + }; @@ -203,6 +207,10 @@ class StreamingPartitioner { return components->size(); } + uint64_t get_n_tags() const { + return tag_component_map->size(); + } + void merge_components(ComponentPtr& root, ComponentPtrSet& comps); ComponentPtr get_tag_component(HashIntoType tag) const; From 72eeb5aab77a94faf0497216d4b2711f95312681 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 10 Nov 2016 16:33:03 -0800 Subject: [PATCH 021/185] add the partitioning script --- scripts/partition-streaming.py | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 scripts/partition-streaming.py diff --git a/scripts/partition-streaming.py b/scripts/partition-streaming.py new file mode 100644 index 0000000000..29ff7a4ff5 --- /dev/null +++ b/scripts/partition-streaming.py @@ -0,0 +1,51 @@ +from __future__ import print_function + +import sys +import screed +import os +import khmer +import textwrap +from khmer import khmer_args +from khmer.khmer_args import (build_counting_args, add_loadgraph_args, + report_on_config, info, calculate_graphsize, + sanitize_help, check_argument_range) +from khmer import _oxli +import argparse +from khmer.utils import (write_record, broken_paired_reader, ReadBundle) + + +def write_stats(partitioner, folder, n, sample): + sample = os.path.basename(sample) + filename = os.path.join(folder, + '{0}-{1}.stats.csv'.format(sample, n)) + print('# {0}: {1} tags, {2} components.'.format(n, partitioner.n_tags, + partitioner.n_components)) + print(' writing results to file -> {0}'.format(filename)) + partitioner.write_components(filename) + +def main(): + parser = build_counting_args() + parser.add_argument('-stats-dir', default='component_stats') + parser.add_argument('samples', nargs='+') + args = parser.parse_args() + + graph = khmer_args.create_countgraph(args) + partitioner = _oxli.StreamingPartitioner(graph) + + try: + os.mkdir(args.stats_dir) + except OSError as e: + pass + + for sample in args.samples: + print('== Starting {0} =='.format(sample)) + for n, read in enumerate(screed.open(sample)): + if n % 100 == 0: + print (n, '...', sep='') + if n > 0 and n % 10000 == 0: + write_stats(partitioner, args.stats_dir, n, sample) + partitioner.consume_sequence(read.sequence) + write_stats(partitioner, args.stats_dir, n, sample) + +if __name__ == '__main__': + main() From 36597908689215d596f64dfa01856bcba3a4ce4f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 10 Nov 2016 21:11:05 -0800 Subject: [PATCH 022/185] Use build_ext method for cython compilation --- Makefile | 7 ++++++- setup.py | 22 ++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 68e6930e25..4fad666b42 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,7 @@ endif MODEXT=$(shell python -c \ "import sysconfig;print(sysconfig.get_config_var('SO'))") EXTENSION_MODULE = khmer/_khmer$(MODEXT) +CYTHON_MODULE = khmer/_oxli$(MODEXT) PYLINT_TEMPLATE="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" @@ -107,6 +108,9 @@ sharedobj: $(EXTENSION_MODULE) $(EXTENSION_MODULE): $(CPPSOURCES) ./setup.py build_ext --inplace +$(CYTHON_MODULE): $(CPPSOURCES) + ./setup.py build_ext --inplace + coverage-debug: $(CPPSOURCES) export CFLAGS="-pg -fprofile-arcs -ftest-coverage -O0"; ./setup.py \ build_ext --debug --inplace --libraries gcov @@ -127,8 +131,9 @@ clean: FORCE cd lib && $(MAKE) clean || true cd tests && rm -rf khmertest_* || true rm -f $(EXTENSION_MODULE) + rm -f $(CYTHON_MODULE) rm -f khmer/*.pyc lib/*.pyc scripts/*.pyc tests/*.pyc oxli/*.pyc \ - sandbox/*.pyc khmer/__pycache__/* sandbox/__pycache__/* + sandbox/*.pyc khmer/__pycache__/* sandbox/__pycache__/* khmer/_oxli.cpp ./setup.py clean --all || true rm -f coverage-debug rm -Rf .coverage diff --git a/setup.py b/setup.py index 7545086b3c..dd8061b68e 100755 --- a/setup.py +++ b/setup.py @@ -51,19 +51,24 @@ from setuptools import setup from setuptools import Extension -from setuptools.command.build_ext import build_ext as _build_ext from distutils.spawn import spawn from distutils.sysconfig import get_config_vars from distutils.dist import Distribution from distutils.errors import DistutilsPlatformError -from Cython.Build import cythonize - import versioneer ez_setup.use_setuptools(version="3.4.1") CMDCLASS = versioneer.get_cmdclass() +from Cython.Build import cythonize + +try: + from Cython.Distutils import build_ext as _build_ext +except ImportError: + print('Cython not found.', file=sys.stderr) + from setuptools.command.build_ext import build_ext as _build_ext + # strip out -Wstrict-prototypes; a hack suggested by # http://stackoverflow.com/a/9740721 # proper fix coming in http://bugs.python.org/issue1222585 @@ -200,7 +205,8 @@ def build_dir(): CY_EXT_MOD_DICT['depends'].append(path_join("lib", "partitioning.hh")) ''' CY_EXTENSION_MOD = Extension("khmer._oxli", ** CY_EXTENSION_MOD_DICT) -EXTENSION_MODS.extend(cythonize([CY_EXTENSION_MOD])) +#EXTENSION_MODS.extend(cythonize([CY_EXTENSION_MOD])) +EXTENSION_MODS.append(CY_EXTENSION_MOD) SCRIPTS = [] @@ -305,7 +311,9 @@ def run(self): ' configure || bash ./configure --static ) && make -f ' 'Makefile.pic PIC'] spawn(cmd=zcmd, dry_run=self.dry_run) - self.extensions[0].extra_objects.extend( + #self.extensions[0].extra_objects.extend( + for ext in self.extensions: + ext.extra_objects.extend( path_join("third-party", "zlib", bn + ".lo") for bn in [ "adler32", "compress", "crc32", "deflate", "gzclose", "gzlib", "gzread", "gzwrite", "infback", "inffast", @@ -314,7 +322,9 @@ def run(self): bz2cmd = ['bash', '-c', 'cd ' + BZIP2DIR + ' && make -f ' 'Makefile-libbz2_so all'] spawn(cmd=bz2cmd, dry_run=self.dry_run) - self.extensions[0].extra_objects.extend( + #self.extensions[0].extra_objects.extend( + for ext in self.extensions: + ext.extra_objects.extend( path_join("third-party", "bzip2", bn + ".o") for bn in [ "blocksort", "huffman", "crctable", "randtable", "compress", "decompress", "bzlib"]) From 13b957839a44a11fb7da07ecd55b74ac5d826525 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 10 Nov 2016 21:11:45 -0800 Subject: [PATCH 023/185] gcc wants algorithm header for max_element --- khmer/_oxli.pxd | 1 + khmer/_oxli.pyx | 31 ++++++++++++------------------- lib/partitioning.cc | 1 + 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index 5c24d1e19c..bc0323bbb3 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -5,6 +5,7 @@ from libcpp.map cimport map from libcpp.set cimport set from libcpp.queue cimport queue from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr +from libcpp.utility cimport pair from libc.stdint cimport uint32_t, uint8_t, uint64_t diff --git a/khmer/_oxli.pyx b/khmer/_oxli.pyx index 24db7e97c3..9441fb3c12 100644 --- a/khmer/_oxli.pyx +++ b/khmer/_oxli.pyx @@ -32,10 +32,9 @@ cdef class Component: return deref(self._this).get_n_tags() def __iter__(self): - it = deref(self._this).tags.begin() - while it != deref(self._this).tags.end(): - yield deref(it) - inc(it) + cdef HashIntoType tag + for tag in deref(self._this).tags: + yield tag def __hash__(self): return self._this.get() @@ -117,23 +116,21 @@ cdef class StreamingPartitioner: def components(self): cdef shared_ptr[ComponentPtrSet] locked + cdef ComponentPtr cmpptr lockedptr = self._components.lock() if lockedptr: - it = deref(lockedptr).begin() - while it != deref(lockedptr).end(): - yield Component.create(deref(it)) - inc(it) + for cmpptr in deref(lockedptr): + yield Component.create(cmpptr) else: raise MemoryError("Can't locked underlying Component set") def tag_components(self): cdef shared_ptr[GuardedKmerCompMap] locked + cdef pair[HashIntoType,ComponentPtr] cpair lockedptr = self._tag_component_map.lock() if lockedptr: - it = deref(lockedptr).data.begin() - while it != deref(lockedptr).data.end(): - yield deref(it).first, Component.create(deref(it).second) - inc(it) + for cpair in deref(lockedptr).data: + yield cpair.first, Component.create(cpair.second) else: raise MemoryError("Can't locked underlying Component set") @@ -144,19 +141,15 @@ cdef class StreamingPartitioner: raise IOError("Can't open file.") cdef ComponentPtr cmpptr - cdef shared_ptr[ComponentPtrSet] locked + cdef shared_ptr[ComponentPtrSet] lockedptr lockedptr = self._components.lock() if lockedptr: - it = deref(lockedptr).begin() - while it != deref(lockedptr).end(): - cmpptr = deref(it) - - fprintf(fp, "%u,%u,%f\n", + for cmpptr in deref(lockedptr): + fprintf(fp, "%llu,%llu,%f\n", deref(cmpptr).component_id, deref(cmpptr).get_n_tags(), Component._mean_tag_count(cmpptr, self._graph_ptr)) - inc(it) fclose(fp) property n_components: diff --git a/lib/partitioning.cc b/lib/partitioning.cc index b7a75cbee7..591a8ffd1c 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include "hashtable.hh" #include "hashbits.hh" From 1dcd249d2b64058522d23b1b904dad763d9ddff6 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 10 Nov 2016 21:12:19 -0800 Subject: [PATCH 024/185] reduce report interval --- scripts/partition-streaming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/partition-streaming.py b/scripts/partition-streaming.py index 29ff7a4ff5..73ff98d724 100644 --- a/scripts/partition-streaming.py +++ b/scripts/partition-streaming.py @@ -40,7 +40,7 @@ def main(): for sample in args.samples: print('== Starting {0} =='.format(sample)) for n, read in enumerate(screed.open(sample)): - if n % 100 == 0: + if n % 500 == 0: print (n, '...', sep='') if n > 0 and n % 10000 == 0: write_stats(partitioner, args.stats_dir, n, sample) From 19084f85d46ef0e748383dbef52ae7c56b676314 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 13 Nov 2016 17:42:11 -0800 Subject: [PATCH 025/185] Add consume_fasta to c++ layer of partitioner --- lib/partitioning.cc | 31 +++++++++++++++++++++++++++++++ lib/partitioning.hh | 1 + 2 files changed, 32 insertions(+) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 591a8ffd1c..1930594c5e 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -11,6 +11,7 @@ #include "partitioning.hh" using namespace khmer; +using namespace khmer::read_parsers; uint64_t Component::n_created = 0; uint64_t Component::n_destroyed = 0; @@ -63,6 +64,36 @@ void StreamingPartitioner::map_tags_to_component(std::set& tags, } +uint64_t StreamingPartitioner::consume_fasta(const std::string& filename) +{ + std::unique_ptr parser = std::unique_ptr(IParser::get_parser(filename)); + Read read; + uint64_t n_invalid = 0; + uint64_t n_consumed = 0; + + while (!parser->is_complete()) { + if (n_consumed && (n_consumed % 10000 == 0)) { + std::cout << "consumed " << n_consumed << "..." << std::endl; + } + try { + read = parser->get_next_read( ); + } catch (NoMoreReadsAvailable) { + break; + } + + bool is_valid = graph->check_and_normalize_read(read.sequence); + if (is_valid) { + consume_sequence(read.sequence); + } else { + n_invalid++; + } + n_consumed++; + } + + return n_invalid; +} + + void StreamingPartitioner::consume_sequence(const std::string& seq) { /* For the following comments, let G be the set of k-mers diff --git a/lib/partitioning.hh b/lib/partitioning.hh index 61d47b0e0e..99f6702941 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -197,6 +197,7 @@ class StreamingPartitioner { explicit StreamingPartitioner(Hashtable * graph); void consume_sequence(const std::string& seq); + uint64_t consume_fasta(std::string const &filename); void map_tags_to_component(std::set& tags, ComponentPtr& comp); void find_connected_tags(KmerQueue& node_q, std::set& found_tags, From 10071912b4e7d8e75a9dcbcdff6ef89137953cd7 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 13 Nov 2016 17:42:57 -0800 Subject: [PATCH 026/185] Expose consume_fasta and add staticmethod def'd for python for tag_counts --- khmer/_oxli.cpp | 2757 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 2162 insertions(+), 595 deletions(-) diff --git a/khmer/_oxli.cpp b/khmer/_oxli.cpp index e2d1a4fb5c..d6dac5bca4 100644 --- a/khmer/_oxli.cpp +++ b/khmer/_oxli.cpp @@ -1,50 +1,5 @@ /* Generated by Cython 0.25.1 */ -/* BEGIN: Cython Metadata -{ - "distutils": { - "define_macros": [ - [ - "VERSION", - "2.0+531.gb18d20d.dirty" - ] - ], - "depends": [ - "khmer/_khmer.hh" - ], - "extra_compile_args": [ - "-O3", - "-std=c++11", - "-pedantic", - "-arch", - "x86_64", - "-mmacosx-version-min=10.7", - "-stdlib=libc++" - ], - "extra_objects": [ - "build/temp.macosx-10.6-x86_64-3.5/khmer/_khmer.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/read_parsers.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/kmer_hash.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/hashtable.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/hashbits.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/labelhash.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/counting.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/subset.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/read_aligner.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/hllcounter.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/traversal.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/kmer_filters.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/assembler.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/alphabets.o", - "build/temp.macosx-10.6-x86_64-3.5/lib/partitioning.o", - "build/temp.macosx-10.6-x86_64-3.5/third-party/smhasher/MurmurHash3.o" - ], - "language": "c++" - }, - "module_name": "khmer._oxli" -} -END: Cython Metadata */ - #define PY_SSIZE_T_CLEAN #include "Python.h" #ifndef Py_PYTHON_H @@ -475,6 +430,7 @@ static CYTHON_INLINE float __PYX_NAN() { #include "kmer_hash.hh" #include "partitioning.hh" #include +#include #ifdef _OPENMP #include #endif /* _OPENMP */ @@ -702,7 +658,7 @@ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__; struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components; struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components; -/* "khmer/_oxli.pyx":10 +/* "khmer/_oxli.pyx":11 * from khmer._khmer import Nodegraph * * cdef class Component: # <<<<<<<<<<<<<< @@ -716,7 +672,7 @@ struct __pyx_obj_5khmer_5_oxli_Component { }; -/* "khmer/_oxli.pyx":55 +/* "khmer/_oxli.pyx":80 * * * cdef class StreamingPartitioner: # <<<<<<<<<<<<<< @@ -729,57 +685,64 @@ struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner { std::weak_ptr _components; std::weak_ptr _tag_component_map; khmer::Hashtable *_graph_ptr; + uint64_t n_consumed; }; -/* "khmer/_oxli.pyx":33 +/* "khmer/_oxli.pyx":34 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< - * it = deref(self._this).tags.begin() - * while it != deref(self._this).tags.end(): + * cdef HashIntoType tag + * for tag in deref(self._this).tags: */ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ { PyObject_HEAD - std::set ::iterator __pyx_v_it; struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self; + khmer::HashIntoType __pyx_v_tag; + std::set ::iterator __pyx_t_0; + std::set *__pyx_t_1; }; -/* "khmer/_oxli.pyx":94 +/* "khmer/_oxli.pyx":125 * return Component.create(compptr) * * def components(self): # <<<<<<<<<<<<<< * cdef shared_ptr[ComponentPtrSet] locked - * lockedptr = self._components.lock() + * cdef ComponentPtr cmpptr */ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components { PyObject_HEAD - std::set ::iterator __pyx_v_it; + khmer::ComponentPtr __pyx_v_cmpptr; std::shared_ptr __pyx_v_locked; std::shared_ptr __pyx_v_lockedptr; struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self; + std::set ::iterator __pyx_t_0; + khmer::ComponentPtrSet *__pyx_t_1; }; -/* "khmer/_oxli.pyx":105 +/* "khmer/_oxli.pyx":135 * raise MemoryError("Can't locked underlying Component set") * * def tag_components(self): # <<<<<<<<<<<<<< * cdef shared_ptr[GuardedKmerCompMap] locked - * lockedptr = self._tag_component_map.lock() + * cdef pair[HashIntoType,ComponentPtr] cpair */ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components { PyObject_HEAD - std::map ::iterator __pyx_v_it; + std::pair __pyx_v_cpair; std::shared_ptr __pyx_v_locked; std::shared_ptr __pyx_v_lockedptr; struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self; + std::map ::iterator __pyx_t_0; + std::map *__pyx_t_1; }; -/* "khmer/_oxli.pyx":10 +/* "khmer/_oxli.pyx":11 * from khmer._khmer import Nodegraph * * cdef class Component: # <<<<<<<<<<<<<< @@ -789,6 +752,8 @@ struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components { struct __pyx_vtabstruct_5khmer_5_oxli_Component { struct __pyx_obj_5khmer_5_oxli_Component *(*create)(khmer::ComponentPtr); + std::vector (*_tag_counts)(khmer::ComponentPtr, khmer::Hashtable *); + float (*_mean_tag_count)(khmer::ComponentPtr, khmer::Hashtable *); }; static struct __pyx_vtabstruct_5khmer_5_oxli_Component *__pyx_vtabptr_5khmer_5_oxli_Component; @@ -933,9 +898,43 @@ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject /* RaiseException.proto */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); +/* RaiseTooManyValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* RaiseNeedMoreValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* IterFinish.proto */ +static CYTHON_INLINE int __Pyx_IterFinish(void); + +/* UnpackItemEndCheck.proto */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); + +/* WriteUnraisableException.proto */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil); + /* GetModuleGlobalName.proto */ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + PyList_SET_ITEM(list, len, x); + Py_SIZE(list) = len+1; + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + /* IncludeStringH.proto */ #include @@ -948,6 +947,9 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /* ImportFrom.proto */ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); +/* GetNameInClass.proto */ +static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name); + /* CodeObjectCache.proto */ typedef struct { PyCodeObject* code_object; @@ -979,14 +981,65 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value); /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value); +/* CppExceptionConversion.proto */ +#ifndef __Pyx_CppExn2PyErr +#include +#include +#include +#include +static void __Pyx_CppExn2PyErr() { + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one + else + throw; + } catch (const std::bad_alloc& exn) { + PyErr_SetString(PyExc_MemoryError, exn.what()); + } catch (const std::bad_cast& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::bad_typeid& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::domain_error& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::invalid_argument& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::ios_base::failure& exn) { + PyErr_SetString(PyExc_IOError, exn.what()); + } catch (const std::out_of_range& exn) { + PyErr_SetString(PyExc_IndexError, exn.what()); + } catch (const std::overflow_error& exn) { + PyErr_SetString(PyExc_OverflowError, exn.what()); + } catch (const std::range_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::underflow_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } + catch (...) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} +#endif + /* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType(khmer::BoundedCounterType value); /* CIntFromPy.proto */ -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); /* CIntFromPy.proto */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); +static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); /* FetchCommonType.proto */ static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); @@ -1078,6 +1131,8 @@ static int __Pyx_check_binary_version(void); static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Component_create(khmer::ComponentPtr __pyx_v_ptr); /* proto*/ +static std::vector __pyx_f_5khmer_5_oxli_9Component__tag_counts(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph); /* proto*/ +static float __pyx_f_5khmer_5_oxli_9Component__mean_tag_count(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph); /* proto*/ /* Module declarations from 'libcpp' */ @@ -1103,20 +1158,26 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Componen /* Module declarations from 'libc.limits' */ +/* Module declarations from 'libc.stdio' */ + /* Module declarations from 'khmer._oxli' */ static PyTypeObject *__pyx_ptype_5khmer_5_oxli_Component = 0; static PyTypeObject *__pyx_ptype_5khmer_5_oxli_StreamingPartitioner = 0; static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = 0; static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components = 0; static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components = 0; +static PyObject *__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(const std::vector &); /*proto*/ static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ #define __Pyx_MODULE_NAME "khmer._oxli" int __pyx_module_is_main_khmer___oxli = 0; /* Implementation of 'khmer._oxli' */ static PyObject *__pyx_builtin_MemoryError; +static PyObject *__pyx_builtin_staticmethod; static PyObject *__pyx_builtin_NotImplementedError; static PyObject *__pyx_builtin_ValueError; +static PyObject *__pyx_builtin_IOError; +static PyObject *__pyx_builtin_range; static const char __pyx_k_args[] = "args"; static const char __pyx_k_iter[] = "__iter__"; static const char __pyx_k_main[] = "__main__"; @@ -1125,30 +1186,40 @@ static const char __pyx_k_test[] = "__test__"; static const char __pyx_k_close[] = "close"; static const char __pyx_k_graph[] = "graph"; static const char __pyx_k_other[] = "other"; +static const char __pyx_k_range[] = "range"; static const char __pyx_k_throw[] = "throw"; static const char __pyx_k_utf_8[] = "utf-8"; static const char __pyx_k_encode[] = "encode"; static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_IOError[] = "IOError"; static const char __pyx_k_Nodegraph[] = "Nodegraph"; +static const char __pyx_k_component[] = "component"; +static const char __pyx_k_graph_ptr[] = "graph_ptr"; static const char __pyx_k_Countgraph[] = "Countgraph"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_components[] = "components"; static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; +static const char __pyx_k_tag_counts[] = "tag_counts"; static const char __pyx_k_MemoryError[] = "MemoryError"; static const char __pyx_k_khmer__oxli[] = "khmer._oxli"; static const char __pyx_k_component_id[] = "component_id"; static const char __pyx_k_khmer__khmer[] = "khmer._khmer"; +static const char __pyx_k_staticmethod[] = "staticmethod"; static const char __pyx_k_tag_components[] = "tag_components"; +static const char __pyx_k_Can_t_open_file[] = "Can't open file."; static const char __pyx_k_Component___iter[] = "Component.__iter__"; static const char __pyx_k_NotImplementedError[] = "NotImplementedError"; static const char __pyx_k_Operator_not_available[] = "Operator not available."; +static const char __pyx_k_work_khmer_khmer__oxli_pyx[] = "/work/khmer/khmer/_oxli.pyx"; static const char __pyx_k_StreamingPartitioner_components[] = "StreamingPartitioner.components"; static const char __pyx_k_Can_t_locked_underlying_Componen[] = "Can't locked underlying Component set"; static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; static const char __pyx_k_StreamingPartitioner_tag_compone[] = "StreamingPartitioner.tag_components"; static PyObject *__pyx_kp_s_Can_t_locked_underlying_Componen; +static PyObject *__pyx_kp_s_Can_t_open_file; static PyObject *__pyx_n_s_Component___iter; static PyObject *__pyx_n_s_Countgraph; +static PyObject *__pyx_n_s_IOError; static PyObject *__pyx_n_s_MemoryError; static PyObject *__pyx_kp_s_Must_take_an_object_with_Hashtab; static PyObject *__pyx_n_s_Nodegraph; @@ -1159,10 +1230,12 @@ static PyObject *__pyx_n_s_StreamingPartitioner_tag_compone; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_n_s_args; static PyObject *__pyx_n_s_close; +static PyObject *__pyx_n_s_component; static PyObject *__pyx_n_s_component_id; static PyObject *__pyx_n_s_components; static PyObject *__pyx_n_s_encode; static PyObject *__pyx_n_s_graph; +static PyObject *__pyx_n_s_graph_ptr; static PyObject *__pyx_n_s_import; static PyObject *__pyx_n_s_iter; static PyObject *__pyx_n_s_khmer__khmer; @@ -1170,11 +1243,15 @@ static PyObject *__pyx_n_s_khmer__oxli; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_other; static PyObject *__pyx_n_s_pyx_vtable; +static PyObject *__pyx_n_s_range; static PyObject *__pyx_n_s_send; +static PyObject *__pyx_n_s_staticmethod; static PyObject *__pyx_n_s_tag_components; +static PyObject *__pyx_n_s_tag_counts; static PyObject *__pyx_n_s_test; static PyObject *__pyx_n_s_throw; static PyObject *__pyx_kp_s_utf_8; +static PyObject *__pyx_kp_s_work_khmer_khmer__oxli_pyx; static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ @@ -1183,13 +1260,18 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_11tag_counts(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_component, PyObject *__pyx_v_graph); /* proto */ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4consume_fasta(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_13tag_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_16write_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6n_tags___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10n_consumed___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5khmer_5_oxli_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ @@ -1203,8 +1285,13 @@ static PyObject *__pyx_tuple__4; static PyObject *__pyx_tuple__5; static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; +static PyObject *__pyx_tuple__8; +static PyObject *__pyx_tuple__9; +static PyObject *__pyx_tuple__10; +static PyObject *__pyx_tuple__11; +static PyObject *__pyx_codeobj__12; -/* "khmer/_oxli.pyx":14 +/* "khmer/_oxli.pyx":15 * cdef ComponentPtr _this * * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< @@ -1240,7 +1327,7 @@ static int __pyx_pw_5khmer_5_oxli_9Component_1__cinit__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 14, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 15, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -1253,13 +1340,13 @@ static int __pyx_pw_5khmer_5_oxli_9Component_1__cinit__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 14, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 15, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("khmer._oxli.Component.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return -1; __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5khmer_5_oxli_Component, 1, "other", 0))) __PYX_ERR(1, 14, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5khmer_5_oxli_Component, 1, "other", 0))) __PYX_ERR(1, 15, __pyx_L1_error) __pyx_r = __pyx_pf_5khmer_5_oxli_9Component___cinit__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self), __pyx_v_other); /* function exit code */ @@ -1278,7 +1365,7 @@ static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5 int __pyx_t_2; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "khmer/_oxli.pyx":15 + /* "khmer/_oxli.pyx":16 * * def __cinit__(self, Component other=None): * if other is not None: # <<<<<<<<<<<<<< @@ -1289,7 +1376,7 @@ static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5 __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "khmer/_oxli.pyx":16 + /* "khmer/_oxli.pyx":17 * def __cinit__(self, Component other=None): * if other is not None: * self._this.reset(other._this.get()) # <<<<<<<<<<<<<< @@ -1298,7 +1385,7 @@ static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5 */ __pyx_v_self->_this.reset(__pyx_v_other->_this.get()); - /* "khmer/_oxli.pyx":15 + /* "khmer/_oxli.pyx":16 * * def __cinit__(self, Component other=None): * if other is not None: # <<<<<<<<<<<<<< @@ -1307,7 +1394,7 @@ static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5 */ } - /* "khmer/_oxli.pyx":14 + /* "khmer/_oxli.pyx":15 * cdef ComponentPtr _this * * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< @@ -1321,7 +1408,7 @@ static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5 return __pyx_r; } -/* "khmer/_oxli.pyx":19 +/* "khmer/_oxli.pyx":20 * * property component_id: * def __get__(self): # <<<<<<<<<<<<<< @@ -1348,7 +1435,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - /* "khmer/_oxli.pyx":20 + /* "khmer/_oxli.pyx":21 * property component_id: * def __get__(self): * return deref(self._this).component_id # <<<<<<<<<<<<<< @@ -1356,13 +1443,13 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct * property _n_created: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 20, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 21, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":19 + /* "khmer/_oxli.pyx":20 * * property component_id: * def __get__(self): # <<<<<<<<<<<<<< @@ -1381,7 +1468,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct return __pyx_r; } -/* "khmer/_oxli.pyx":23 +/* "khmer/_oxli.pyx":24 * * property _n_created: * def __get__(self): # <<<<<<<<<<<<<< @@ -1408,7 +1495,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct _ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - /* "khmer/_oxli.pyx":24 + /* "khmer/_oxli.pyx":25 * property _n_created: * def __get__(self): * return deref(self._this).get_n_created() # <<<<<<<<<<<<<< @@ -1416,13 +1503,13 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct _ * property _n_destroyed: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_created()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 24, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_created()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":23 + /* "khmer/_oxli.pyx":24 * * property _n_created: * def __get__(self): # <<<<<<<<<<<<<< @@ -1441,7 +1528,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct _ return __pyx_r; } -/* "khmer/_oxli.pyx":27 +/* "khmer/_oxli.pyx":28 * * property _n_destroyed: * def __get__(self): # <<<<<<<<<<<<<< @@ -1468,7 +1555,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(struct PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - /* "khmer/_oxli.pyx":28 + /* "khmer/_oxli.pyx":29 * property _n_destroyed: * def __get__(self): * return deref(self._this).get_n_destroyed() # <<<<<<<<<<<<<< @@ -1476,13 +1563,13 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(struct * def __len__(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_destroyed()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 28, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_destroyed()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 29, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":27 + /* "khmer/_oxli.pyx":28 * * property _n_destroyed: * def __get__(self): # <<<<<<<<<<<<<< @@ -1501,7 +1588,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(struct return __pyx_r; } -/* "khmer/_oxli.pyx":30 +/* "khmer/_oxli.pyx":31 * return deref(self._this).get_n_destroyed() * * def __len__(self): # <<<<<<<<<<<<<< @@ -1527,7 +1614,7 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "khmer/_oxli.pyx":31 + /* "khmer/_oxli.pyx":32 * * def __len__(self): * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< @@ -1537,7 +1624,7 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k __pyx_r = (*__pyx_v_self->_this).get_n_tags(); goto __pyx_L0; - /* "khmer/_oxli.pyx":30 + /* "khmer/_oxli.pyx":31 * return deref(self._this).get_n_destroyed() * * def __len__(self): # <<<<<<<<<<<<<< @@ -1552,12 +1639,12 @@ static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5k } static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "khmer/_oxli.pyx":33 +/* "khmer/_oxli.pyx":34 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< - * it = deref(self._this).tags.begin() - * while it != deref(self._this).tags.end(): + * cdef HashIntoType tag + * for tag in deref(self._this).tags: */ /* Python wrapper */ @@ -1582,7 +1669,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5k if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(1, 33, __pyx_L1_error) + __PYX_ERR(1, 34, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } @@ -1590,7 +1677,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5k __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 33, __pyx_L1_error) + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 34, __pyx_L1_error) __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -1610,8 +1697,10 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj { struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; + std::set ::iterator __pyx_t_1; + std::set *__pyx_t_2; + khmer::HashIntoType __pyx_t_3; + PyObject *__pyx_t_4 = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("None", 0); switch (__pyx_generator->resume_label) { @@ -1622,71 +1711,69 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 33, __pyx_L1_error) - - /* "khmer/_oxli.pyx":34 - * - * def __iter__(self): - * it = deref(self._this).tags.begin() # <<<<<<<<<<<<<< - * while it != deref(self._this).tags.end(): - * yield deref(it) - */ - __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_self->_this).tags.begin(); + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 34, __pyx_L1_error) - /* "khmer/_oxli.pyx":35 + /* "khmer/_oxli.pyx":36 * def __iter__(self): - * it = deref(self._this).tags.begin() - * while it != deref(self._this).tags.end(): # <<<<<<<<<<<<<< - * yield deref(it) - * inc(it) + * cdef HashIntoType tag + * for tag in deref(self._this).tags: # <<<<<<<<<<<<<< + * yield tag + * */ - while (1) { - __pyx_t_1 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_self->_this).tags.end()) != 0); - if (!__pyx_t_1) break; + __pyx_t_2 = &(*__pyx_cur_scope->__pyx_v_self->_this).tags; + __pyx_t_1 = __pyx_t_2->begin(); + for (;;) { + if (!(__pyx_t_1 != __pyx_t_2->end())) break; + __pyx_t_3 = *__pyx_t_1; + ++__pyx_t_1; + __pyx_cur_scope->__pyx_v_tag = __pyx_t_3; - /* "khmer/_oxli.pyx":36 - * it = deref(self._this).tags.begin() - * while it != deref(self._this).tags.end(): - * yield deref(it) # <<<<<<<<<<<<<< - * inc(it) + /* "khmer/_oxli.pyx":37 + * cdef HashIntoType tag + * for tag in deref(self._this).tags: + * yield tag # <<<<<<<<<<<<<< * + * def __hash__(self): */ - __pyx_t_2 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 36, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; + __pyx_t_4 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_cur_scope->__pyx_v_tag); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 37, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_2; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); /* return from generator, yielding value */ __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L6_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 36, __pyx_L1_error) + __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; + __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 37, __pyx_L1_error) - /* "khmer/_oxli.pyx":37 - * while it != deref(self._this).tags.end(): - * yield deref(it) - * inc(it) # <<<<<<<<<<<<<< + /* "khmer/_oxli.pyx":36 + * def __iter__(self): + * cdef HashIntoType tag + * for tag in deref(self._this).tags: # <<<<<<<<<<<<<< + * yield tag * - * def __hash__(self): */ - (++__pyx_cur_scope->__pyx_v_it); } if (1); else __pyx_cur_scope = __pyx_cur_scope; - /* "khmer/_oxli.pyx":33 + /* "khmer/_oxli.pyx":34 * return deref(self._this).get_n_tags() * * def __iter__(self): # <<<<<<<<<<<<<< - * it = deref(self._this).tags.begin() - * while it != deref(self._this).tags.end(): + * cdef HashIntoType tag + * for tag in deref(self._this).tags: */ /* function exit code */ PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); __Pyx_AddTraceback("__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_r = 0; @@ -1697,7 +1784,7 @@ static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObj } /* "khmer/_oxli.pyx":39 - * inc(it) + * yield tag * * def __hash__(self): # <<<<<<<<<<<<<< * return self._this.get() @@ -1733,7 +1820,7 @@ static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5k goto __pyx_L0; /* "khmer/_oxli.pyx":39 - * inc(it) + * yield tag * * def __hash__(self): # <<<<<<<<<<<<<< * return self._this.get() @@ -1905,7 +1992,7 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Componen * comp._this = ptr * return comp # <<<<<<<<<<<<<< * - * + * @staticmethod */ __Pyx_XDECREF(((PyObject *)__pyx_r)); __Pyx_INCREF(((PyObject *)__pyx_v_comp)); @@ -1932,8 +2019,402 @@ static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Componen return __pyx_r; } -/* "khmer/_oxli.pyx":62 - * cdef CyHashtable * _graph_ptr +/* "khmer/_oxli.pyx":55 + * + * @staticmethod + * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CyHashtable * graph): # <<<<<<<<<<<<<< + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef vector[BoundedCounterType] counts + */ + +static std::vector __pyx_f_5khmer_5_oxli_9Component__tag_counts(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph) { + uint64_t __pyx_v_n_tags; + std::vector __pyx_v_counts; + int __pyx_v_idx; + uint64_t __pyx_v_tag; + std::vector __pyx_r; + __Pyx_RefNannyDeclarations + std::vector __pyx_t_1; + std::set ::iterator __pyx_t_2; + std::set *__pyx_t_3; + khmer::HashIntoType __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *(*__pyx_t_9)(PyObject *); + int __pyx_t_10; + uint64_t __pyx_t_11; + __Pyx_RefNannySetupContext("_tag_counts", 0); + + /* "khmer/_oxli.pyx":56 + * @staticmethod + * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CyHashtable * graph): + * cdef uint64_t n_tags = deref(comp).get_n_tags() # <<<<<<<<<<<<<< + * cdef vector[BoundedCounterType] counts + * counts = vector[BoundedCounterType](n_tags) + */ + __pyx_v_n_tags = (*__pyx_v_comp).get_n_tags(); + + /* "khmer/_oxli.pyx":58 + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef vector[BoundedCounterType] counts + * counts = vector[BoundedCounterType](n_tags) # <<<<<<<<<<<<<< + * cdef int idx + * cdef uint64_t tag + */ + try { + __pyx_t_1 = std::vector (__pyx_v_n_tags); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 58, __pyx_L1_error) + } + __pyx_v_counts = __pyx_t_1; + + /* "khmer/_oxli.pyx":61 + * cdef int idx + * cdef uint64_t tag + * for idx, tag in deref(comp).tags: # <<<<<<<<<<<<<< + * counts[idx] = deref(graph).get_count(tag) + * return counts + */ + __pyx_t_3 = &(*__pyx_v_comp).tags; + __pyx_t_2 = __pyx_t_3->begin(); + for (;;) { + if (!(__pyx_t_2 != __pyx_t_3->end())) break; + __pyx_t_4 = *__pyx_t_2; + ++__pyx_t_2; + __pyx_t_5 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { + PyObject* sequence = __pyx_t_5; + #if !CYTHON_COMPILING_IN_PYPY + Py_ssize_t size = Py_SIZE(sequence); + #else + Py_ssize_t size = PySequence_Size(sequence); + #endif + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 61, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); + __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); + } else { + __pyx_t_6 = PyList_GET_ITEM(sequence, 0); + __pyx_t_7 = PyList_GET_ITEM(sequence, 1); + } + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(__pyx_t_7); + #else + __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + #endif + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_8 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_9 = Py_TYPE(__pyx_t_8)->tp_iternext; + index = 0; __pyx_t_6 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_6)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_6); + index = 1; __pyx_t_7 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_7)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < 0) __PYX_ERR(1, 61, __pyx_L1_error) + __pyx_t_9 = NULL; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L6_unpacking_done; + __pyx_L5_unpacking_failed:; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(1, 61, __pyx_L1_error) + __pyx_L6_unpacking_done:; + } + __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_6); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_11 = __Pyx_PyInt_As_uint64_t(__pyx_t_7); if (unlikely((__pyx_t_11 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_v_idx = __pyx_t_10; + __pyx_v_tag = __pyx_t_11; + + /* "khmer/_oxli.pyx":62 + * cdef uint64_t tag + * for idx, tag in deref(comp).tags: + * counts[idx] = deref(graph).get_count(tag) # <<<<<<<<<<<<<< + * return counts + * + */ + (__pyx_v_counts[__pyx_v_idx]) = (*__pyx_v_graph).get_count(__pyx_v_tag); + + /* "khmer/_oxli.pyx":61 + * cdef int idx + * cdef uint64_t tag + * for idx, tag in deref(comp).tags: # <<<<<<<<<<<<<< + * counts[idx] = deref(graph).get_count(tag) + * return counts + */ + } + + /* "khmer/_oxli.pyx":63 + * for idx, tag in deref(comp).tags: + * counts[idx] = deref(graph).get_count(tag) + * return counts # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __pyx_r = __pyx_v_counts; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":55 + * + * @staticmethod + * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CyHashtable * graph): # <<<<<<<<<<<<<< + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef vector[BoundedCounterType] counts + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_WriteUnraisable("khmer._oxli.Component._tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":66 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CyCpHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12tag_counts(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_5khmer_5_oxli_9Component_12tag_counts = {"tag_counts", (PyCFunction)__pyx_pw_5khmer_5_oxli_9Component_12tag_counts, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12tag_counts(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_component = 0; + PyObject *__pyx_v_graph = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("tag_counts (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_component,&__pyx_n_s_graph,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_component)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("tag_counts", 1, 2, 2, 1); __PYX_ERR(1, 66, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "tag_counts") < 0)) __PYX_ERR(1, 66, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + } + __pyx_v_component = ((struct __pyx_obj_5khmer_5_oxli_Component *)values[0]); + __pyx_v_graph = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("tag_counts", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 66, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.Component.tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_component), __pyx_ptype_5khmer_5_oxli_Component, 1, "component", 0))) __PYX_ERR(1, 66, __pyx_L1_error) + __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_11tag_counts(__pyx_v_component, __pyx_v_graph); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_9Component_11tag_counts(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_component, PyObject *__pyx_v_graph) { + khmer::khmer_KHashtable_Object *__pyx_v_graph_ptr; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("tag_counts", 0); + + /* "khmer/_oxli.pyx":67 + * @staticmethod + * def tag_counts(Component component, graph): + * cdef CyCpHashtable_Object* graph_ptr = graph # <<<<<<<<<<<<<< + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + * + */ + __pyx_v_graph_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); + + /* "khmer/_oxli.pyx":68 + * def tag_counts(Component component, graph): + * cdef CyCpHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(__pyx_f_5khmer_5_oxli_9Component__tag_counts(__pyx_v_component->_this, (*__pyx_v_graph_ptr).hashtable)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":66 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CyCpHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.Component.tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":71 + * + * @staticmethod + * cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): # <<<<<<<<<<<<<< + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef float acc = 0 + */ + +static float __pyx_f_5khmer_5_oxli_9Component__mean_tag_count(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph) { + uint64_t __pyx_v_n_tags; + float __pyx_v_acc; + uint64_t __pyx_v_tag; + float __pyx_r; + __Pyx_RefNannyDeclarations + std::set ::iterator __pyx_t_1; + std::set *__pyx_t_2; + khmer::HashIntoType __pyx_t_3; + __Pyx_RefNannySetupContext("_mean_tag_count", 0); + + /* "khmer/_oxli.pyx":72 + * @staticmethod + * cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): + * cdef uint64_t n_tags = deref(comp).get_n_tags() # <<<<<<<<<<<<<< + * cdef float acc = 0 + * cdef uint64_t tag + */ + __pyx_v_n_tags = (*__pyx_v_comp).get_n_tags(); + + /* "khmer/_oxli.pyx":73 + * cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef float acc = 0 # <<<<<<<<<<<<<< + * cdef uint64_t tag + * for tag in deref(comp).tags: + */ + __pyx_v_acc = 0.0; + + /* "khmer/_oxli.pyx":75 + * cdef float acc = 0 + * cdef uint64_t tag + * for tag in deref(comp).tags: # <<<<<<<<<<<<<< + * acc += deref(graph).get_count(tag) + * return acc / n_tags + */ + __pyx_t_2 = &(*__pyx_v_comp).tags; + __pyx_t_1 = __pyx_t_2->begin(); + for (;;) { + if (!(__pyx_t_1 != __pyx_t_2->end())) break; + __pyx_t_3 = *__pyx_t_1; + ++__pyx_t_1; + __pyx_v_tag = __pyx_t_3; + + /* "khmer/_oxli.pyx":76 + * cdef uint64_t tag + * for tag in deref(comp).tags: + * acc += deref(graph).get_count(tag) # <<<<<<<<<<<<<< + * return acc / n_tags + * + */ + __pyx_v_acc = (__pyx_v_acc + ((float)(*__pyx_v_graph).get_count(__pyx_v_tag))); + + /* "khmer/_oxli.pyx":75 + * cdef float acc = 0 + * cdef uint64_t tag + * for tag in deref(comp).tags: # <<<<<<<<<<<<<< + * acc += deref(graph).get_count(tag) + * return acc / n_tags + */ + } + + /* "khmer/_oxli.pyx":77 + * for tag in deref(comp).tags: + * acc += deref(graph).get_count(tag) + * return acc / n_tags # <<<<<<<<<<<<<< + * + * + */ + if (unlikely(((float)__pyx_v_n_tags) == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "float division"); + __PYX_ERR(1, 77, __pyx_L1_error) + } + __pyx_r = (__pyx_v_acc / ((float)__pyx_v_n_tags)); + goto __pyx_L0; + + /* "khmer/_oxli.pyx":71 + * + * @staticmethod + * cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): # <<<<<<<<<<<<<< + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef float acc = 0 + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_WriteUnraisable("khmer._oxli.Component._mean_tag_count", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); + __pyx_r = 0; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":88 + * cdef readonly uint64_t n_consumed * * def __cinit__(self, graph): # <<<<<<<<<<<<<< * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): @@ -1965,7 +2446,7 @@ static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__ else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 62, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 88, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { goto __pyx_L5_argtuple_error; @@ -1976,7 +2457,7 @@ static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__ } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 62, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 88, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2001,16 +2482,16 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ khmer::StreamingPartitioner *__pyx_t_6; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "khmer/_oxli.pyx":63 + /* "khmer/_oxli.pyx":89 * * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< * raise ValueError('Must take an object with Hashtable *') * */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 63, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 63, __pyx_L1_error) + __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 89, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_4 = (__pyx_t_3 != 0); if (!__pyx_t_4) { @@ -2018,9 +2499,9 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_1 = __pyx_t_4; goto __pyx_L4_bool_binop_done; } - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 63, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 63, __pyx_L1_error) + __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 89, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_3 = (__pyx_t_4 != 0); __pyx_t_1 = __pyx_t_3; @@ -2028,20 +2509,20 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_3 = ((!__pyx_t_1) != 0); if (__pyx_t_3) { - /* "khmer/_oxli.pyx":64 + /* "khmer/_oxli.pyx":90 * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< * * */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 64, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 90, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 64, __pyx_L1_error) + __PYX_ERR(1, 90, __pyx_L1_error) - /* "khmer/_oxli.pyx":63 + /* "khmer/_oxli.pyx":89 * * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< @@ -2050,7 +2531,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ } - /* "khmer/_oxli.pyx":67 + /* "khmer/_oxli.pyx":93 * * * cdef CyCpHashtable_Object* ptr = graph # <<<<<<<<<<<<<< @@ -2059,7 +2540,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ */ __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - /* "khmer/_oxli.pyx":68 + /* "khmer/_oxli.pyx":94 * * cdef CyCpHashtable_Object* ptr = graph * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< @@ -2069,7 +2550,7 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_5 = (*__pyx_v_ptr).hashtable; __pyx_v_self->_graph_ptr = __pyx_t_5; - /* "khmer/_oxli.pyx":70 + /* "khmer/_oxli.pyx":96 * self._graph_ptr = deref(ptr).hashtable * * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) # <<<<<<<<<<<<<< @@ -2080,34 +2561,43 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ __pyx_t_6 = new khmer::StreamingPartitioner(__pyx_v_self->_graph_ptr); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 70, __pyx_L1_error) + __PYX_ERR(1, 96, __pyx_L1_error) } __pyx_v_self->_this.reset(__pyx_t_6); - /* "khmer/_oxli.pyx":72 + /* "khmer/_oxli.pyx":98 * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) * * self._tag_component_map = deref(self._this).get_tag_component_map() # <<<<<<<<<<<<<< * self._components = deref(self._this).get_component_set() - * + * self.n_consumed = 0 */ __pyx_v_self->_tag_component_map = (*__pyx_v_self->_this).get_tag_component_map(); - /* "khmer/_oxli.pyx":73 + /* "khmer/_oxli.pyx":99 * * self._tag_component_map = deref(self._this).get_tag_component_map() * self._components = deref(self._this).get_component_set() # <<<<<<<<<<<<<< + * self.n_consumed = 0 * - * def consume_sequence(self, sequence): */ __pyx_v_self->_components = (*__pyx_v_self->_this).get_component_set(); - /* "khmer/_oxli.pyx":62 - * cdef CyHashtable * _graph_ptr + /* "khmer/_oxli.pyx":100 + * self._tag_component_map = deref(self._this).get_tag_component_map() + * self._components = deref(self._this).get_component_set() + * self.n_consumed = 0 # <<<<<<<<<<<<<< * - * def __cinit__(self, graph): # <<<<<<<<<<<<<< - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') + * def consume_sequence(self, sequence): + */ + __pyx_v_self->n_consumed = 0; + + /* "khmer/_oxli.pyx":88 + * cdef readonly uint64_t n_consumed + * + * def __cinit__(self, graph): # <<<<<<<<<<<<<< + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') */ /* function exit code */ @@ -2122,12 +2612,12 @@ static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_ return __pyx_r; } -/* "khmer/_oxli.pyx":75 - * self._components = deref(self._this).get_component_set() +/* "khmer/_oxli.pyx":102 + * self.n_consumed = 0 * * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< * deref(self._this).consume_sequence(sequence.encode('utf-8')) - * + * self.n_consumed += 1 */ /* Python wrapper */ @@ -2151,33 +2641,42 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence std::string __pyx_t_3; __Pyx_RefNannySetupContext("consume_sequence", 0); - /* "khmer/_oxli.pyx":76 + /* "khmer/_oxli.pyx":103 * * def consume_sequence(self, sequence): * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< + * self.n_consumed += 1 * - * def get_tag_component(self, kmer): */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 103, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 103, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; try { (*__pyx_v_self->_this).consume_sequence(__pyx_t_3); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 76, __pyx_L1_error) + __PYX_ERR(1, 103, __pyx_L1_error) } - /* "khmer/_oxli.pyx":75 - * self._components = deref(self._this).get_component_set() + /* "khmer/_oxli.pyx":104 + * def consume_sequence(self, sequence): + * deref(self._this).consume_sequence(sequence.encode('utf-8')) + * self.n_consumed += 1 # <<<<<<<<<<<<<< + * + * def consume_fasta(self, filename): + */ + __pyx_v_self->n_consumed = (__pyx_v_self->n_consumed + 1); + + /* "khmer/_oxli.pyx":102 + * self.n_consumed = 0 * * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< * deref(self._this).consume_sequence(sequence.encode('utf-8')) - * + * self.n_consumed += 1 */ /* function exit code */ @@ -2194,8 +2693,85 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence return __pyx_r; } -/* "khmer/_oxli.pyx":78 - * deref(self._this).consume_sequence(sequence.encode('utf-8')) +/* "khmer/_oxli.pyx":106 + * self.n_consumed += 1 + * + * def consume_fasta(self, filename): # <<<<<<<<<<<<<< + * return deref(self._this).consume_fasta(filename.encode('utf-8')) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5consume_fasta(PyObject *__pyx_v_self, PyObject *__pyx_v_filename); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5consume_fasta(PyObject *__pyx_v_self, PyObject *__pyx_v_filename) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("consume_fasta (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4consume_fasta(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_filename)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4consume_fasta(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + std::string __pyx_t_3; + uint64_t __pyx_t_4; + __Pyx_RefNannySetupContext("consume_fasta", 0); + + /* "khmer/_oxli.pyx":107 + * + * def consume_fasta(self, filename): + * return deref(self._this).consume_fasta(filename.encode('utf-8')) # <<<<<<<<<<<<<< + * + * def get_tag_component(self, kmer): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_filename, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 107, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + try { + __pyx_t_4 = (*__pyx_v_self->_this).consume_fasta(__pyx_t_3); + } catch(...) { + try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } + __PYX_ERR(1, 107, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyInt_From_uint64_t(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":106 + * self.n_consumed += 1 + * + * def consume_fasta(self, filename): # <<<<<<<<<<<<<< + * return deref(self._this).consume_fasta(filename.encode('utf-8')) + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.consume_fasta", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":109 + * return deref(self._this).consume_fasta(filename.encode('utf-8')) * * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< * cdef ComponentPtr compptr @@ -2203,19 +2779,19 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence */ /* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("get_tag_component (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_component(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_tag_component(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { khmer::ComponentPtr __pyx_v_compptr; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations @@ -2225,23 +2801,23 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen int __pyx_t_4; __Pyx_RefNannySetupContext("get_tag_component", 0); - /* "khmer/_oxli.pyx":80 + /* "khmer/_oxli.pyx":111 * def get_tag_component(self, kmer): * cdef ComponentPtr compptr * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< * if compptr == NULL: * return None */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 80, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 80, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 80, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_compptr = (*__pyx_v_self->_this).get_tag_component(__pyx_t_3); - /* "khmer/_oxli.pyx":81 + /* "khmer/_oxli.pyx":112 * cdef ComponentPtr compptr * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) * if compptr == NULL: # <<<<<<<<<<<<<< @@ -2251,7 +2827,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); if (__pyx_t_4) { - /* "khmer/_oxli.pyx":82 + /* "khmer/_oxli.pyx":113 * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) * if compptr == NULL: * return None # <<<<<<<<<<<<<< @@ -2263,7 +2839,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen __pyx_r = Py_None; goto __pyx_L0; - /* "khmer/_oxli.pyx":81 + /* "khmer/_oxli.pyx":112 * cdef ComponentPtr compptr * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) * if compptr == NULL: # <<<<<<<<<<<<<< @@ -2272,7 +2848,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen */ } - /* "khmer/_oxli.pyx":84 + /* "khmer/_oxli.pyx":115 * return None * else: * return Component.create(compptr) # <<<<<<<<<<<<<< @@ -2281,15 +2857,15 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen */ /*else*/ { __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 84, __pyx_L1_error) + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 115, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; } - /* "khmer/_oxli.pyx":78 - * deref(self._this).consume_sequence(sequence.encode('utf-8')) + /* "khmer/_oxli.pyx":109 + * return deref(self._this).consume_fasta(filename.encode('utf-8')) * * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< * cdef ComponentPtr compptr @@ -2308,7 +2884,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen return __pyx_r; } -/* "khmer/_oxli.pyx":86 +/* "khmer/_oxli.pyx":117 * return Component.create(compptr) * * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< @@ -2317,19 +2893,19 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4get_tag_componen */ /* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("get_nearest_component (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_component(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8get_nearest_component(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { khmer::ComponentPtr __pyx_v_compptr; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations @@ -2339,23 +2915,23 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp int __pyx_t_4; __Pyx_RefNannySetupContext("get_nearest_component", 0); - /* "khmer/_oxli.pyx":88 + /* "khmer/_oxli.pyx":119 * def get_nearest_component(self, kmer): * cdef ComponentPtr compptr * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< * if compptr == NULL: * return None */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 119, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 119, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 119, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_compptr = (*__pyx_v_self->_this).get_nearest_component(__pyx_t_3); - /* "khmer/_oxli.pyx":89 + /* "khmer/_oxli.pyx":120 * cdef ComponentPtr compptr * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) * if compptr == NULL: # <<<<<<<<<<<<<< @@ -2365,7 +2941,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); if (__pyx_t_4) { - /* "khmer/_oxli.pyx":90 + /* "khmer/_oxli.pyx":121 * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) * if compptr == NULL: * return None # <<<<<<<<<<<<<< @@ -2377,7 +2953,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp __pyx_r = Py_None; goto __pyx_L0; - /* "khmer/_oxli.pyx":89 + /* "khmer/_oxli.pyx":120 * cdef ComponentPtr compptr * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) * if compptr == NULL: # <<<<<<<<<<<<<< @@ -2386,7 +2962,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp */ } - /* "khmer/_oxli.pyx":92 + /* "khmer/_oxli.pyx":123 * return None * else: * return Component.create(compptr) # <<<<<<<<<<<<<< @@ -2395,14 +2971,14 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp */ /*else*/ { __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 92, __pyx_L1_error) + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; } - /* "khmer/_oxli.pyx":86 + /* "khmer/_oxli.pyx":117 * return Component.create(compptr) * * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< @@ -2421,30 +2997,30 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_nearest_comp __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_12generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "khmer/_oxli.pyx":94 +/* "khmer/_oxli.pyx":125 * return Component.create(compptr) * * def components(self): # <<<<<<<<<<<<<< * cdef shared_ptr[ComponentPtrSet] locked - * lockedptr = self._components.lock() + * cdef ComponentPtr cmpptr */ /* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_11components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_11components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("components (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations @@ -2453,7 +3029,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struc if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(1, 94, __pyx_L1_error) + __PYX_ERR(1, 125, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } @@ -2461,7 +3037,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struc __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_components, __pyx_n_s_StreamingPartitioner_components, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 94, __pyx_L1_error) + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_12generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_components, __pyx_n_s_StreamingPartitioner_components, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 125, __pyx_L1_error) __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -2477,13 +3053,15 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8components(struc return __pyx_r; } -static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_12generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)__pyx_generator->closure); PyObject *__pyx_r = NULL; bool __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; + std::set ::iterator __pyx_t_2; + khmer::ComponentPtrSet *__pyx_t_3; + khmer::ComponentPtr __pyx_t_4; + PyObject *__pyx_t_5 = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("None", 0); switch (__pyx_generator->resume_label) { @@ -2494,116 +3072,114 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 94, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 125, __pyx_L1_error) - /* "khmer/_oxli.pyx":96 - * def components(self): + /* "khmer/_oxli.pyx":128 * cdef shared_ptr[ComponentPtrSet] locked + * cdef ComponentPtr cmpptr * lockedptr = self._components.lock() # <<<<<<<<<<<<<< * if lockedptr: - * it = deref(lockedptr).begin() + * for cmpptr in deref(lockedptr): */ __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_components.lock(); - /* "khmer/_oxli.pyx":97 - * cdef shared_ptr[ComponentPtrSet] locked + /* "khmer/_oxli.pyx":129 + * cdef ComponentPtr cmpptr * lockedptr = self._components.lock() * if lockedptr: # <<<<<<<<<<<<<< - * it = deref(lockedptr).begin() - * while it != deref(lockedptr).end(): + * for cmpptr in deref(lockedptr): + * yield Component.create(cmpptr) */ __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); if (__pyx_t_1) { - /* "khmer/_oxli.pyx":98 + /* "khmer/_oxli.pyx":130 * lockedptr = self._components.lock() * if lockedptr: - * it = deref(lockedptr).begin() # <<<<<<<<<<<<<< - * while it != deref(lockedptr).end(): - * yield Component.create(deref(it)) + * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< + * yield Component.create(cmpptr) + * else: */ - __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_lockedptr).begin(); - - /* "khmer/_oxli.pyx":99 + __pyx_t_3 = &(*__pyx_cur_scope->__pyx_v_lockedptr); + __pyx_t_2 = __pyx_t_3->begin(); + for (;;) { + if (!(__pyx_t_2 != __pyx_t_3->end())) break; + __pyx_t_4 = *__pyx_t_2; + ++__pyx_t_2; + __pyx_cur_scope->__pyx_v_cmpptr = __pyx_t_4; + + /* "khmer/_oxli.pyx":131 * if lockedptr: - * it = deref(lockedptr).begin() - * while it != deref(lockedptr).end(): # <<<<<<<<<<<<<< - * yield Component.create(deref(it)) - * inc(it) - */ - while (1) { - __pyx_t_2 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_lockedptr).end()) != 0); - if (!__pyx_t_2) break; - - /* "khmer/_oxli.pyx":100 - * it = deref(lockedptr).begin() - * while it != deref(lockedptr).end(): - * yield Component.create(deref(it)) # <<<<<<<<<<<<<< - * inc(it) + * for cmpptr in deref(lockedptr): + * yield Component.create(cmpptr) # <<<<<<<<<<<<<< * else: + * raise MemoryError("Can't locked underlying Component set") */ - __pyx_t_3 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create((*__pyx_cur_scope->__pyx_v_it))); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 100, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; + __pyx_t_5 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_cur_scope->__pyx_v_cmpptr)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 131, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); /* return from generator, yielding value */ __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L7_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 100, __pyx_L1_error) + __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; + __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 131, __pyx_L1_error) - /* "khmer/_oxli.pyx":101 - * while it != deref(lockedptr).end(): - * yield Component.create(deref(it)) - * inc(it) # <<<<<<<<<<<<<< + /* "khmer/_oxli.pyx":130 + * lockedptr = self._components.lock() + * if lockedptr: + * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< + * yield Component.create(cmpptr) * else: - * raise MemoryError("Can't locked underlying Component set") */ - (++__pyx_cur_scope->__pyx_v_it); } - /* "khmer/_oxli.pyx":97 - * cdef shared_ptr[ComponentPtrSet] locked + /* "khmer/_oxli.pyx":129 + * cdef ComponentPtr cmpptr * lockedptr = self._components.lock() * if lockedptr: # <<<<<<<<<<<<<< - * it = deref(lockedptr).begin() - * while it != deref(lockedptr).end(): + * for cmpptr in deref(lockedptr): + * yield Component.create(cmpptr) */ goto __pyx_L4; } - /* "khmer/_oxli.pyx":103 - * inc(it) + /* "khmer/_oxli.pyx":133 + * yield Component.create(cmpptr) * else: * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< * * def tag_components(self): */ /*else*/ { - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 103, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 103, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 133, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_Raise(__pyx_t_5, 0, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 133, __pyx_L1_error) } __pyx_L4:; if (1); else __pyx_cur_scope = __pyx_cur_scope; - /* "khmer/_oxli.pyx":94 + /* "khmer/_oxli.pyx":125 * return Component.create(compptr) * * def components(self): # <<<<<<<<<<<<<< * cdef shared_ptr[ComponentPtrSet] locked - * lockedptr = self._components.lock() + * cdef ComponentPtr cmpptr */ /* function exit code */ PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); __Pyx_AddTraceback("components", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_r = 0; @@ -2612,30 +3188,30 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_10generator1(__py __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_15generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "khmer/_oxli.pyx":105 +/* "khmer/_oxli.pyx":135 * raise MemoryError("Can't locked underlying Component set") * * def tag_components(self): # <<<<<<<<<<<<<< * cdef shared_ptr[GuardedKmerCompMap] locked - * lockedptr = self._tag_component_map.lock() + * cdef pair[HashIntoType,ComponentPtr] cpair */ /* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_14tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_14tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("tag_components (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_13tag_components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_13tag_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations @@ -2644,7 +3220,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components( if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(1, 105, __pyx_L1_error) + __PYX_ERR(1, 135, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } @@ -2652,7 +3228,7 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components( __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_tag_components, __pyx_n_s_StreamingPartitioner_tag_compone, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 105, __pyx_L1_error) + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_15generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_tag_components, __pyx_n_s_StreamingPartitioner_tag_compone, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 135, __pyx_L1_error) __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -2668,15 +3244,17 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_11tag_components( return __pyx_r; } -static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_15generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)__pyx_generator->closure); PyObject *__pyx_r = NULL; bool __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; + std::map ::iterator __pyx_t_2; + std::map *__pyx_t_3; + std::pair __pyx_t_4; PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("None", 0); switch (__pyx_generator->resume_label) { @@ -2687,128 +3265,126 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 105, __pyx_L1_error) + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 135, __pyx_L1_error) - /* "khmer/_oxli.pyx":107 - * def tag_components(self): + /* "khmer/_oxli.pyx":138 * cdef shared_ptr[GuardedKmerCompMap] locked + * cdef pair[HashIntoType,ComponentPtr] cpair * lockedptr = self._tag_component_map.lock() # <<<<<<<<<<<<<< * if lockedptr: - * it = deref(lockedptr).data.begin() + * for cpair in deref(lockedptr).data: */ __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_tag_component_map.lock(); - /* "khmer/_oxli.pyx":108 - * cdef shared_ptr[GuardedKmerCompMap] locked + /* "khmer/_oxli.pyx":139 + * cdef pair[HashIntoType,ComponentPtr] cpair * lockedptr = self._tag_component_map.lock() * if lockedptr: # <<<<<<<<<<<<<< - * it = deref(lockedptr).data.begin() - * while it != deref(lockedptr).data.end(): + * for cpair in deref(lockedptr).data: + * yield cpair.first, Component.create(cpair.second) */ __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); if (__pyx_t_1) { - /* "khmer/_oxli.pyx":109 + /* "khmer/_oxli.pyx":140 * lockedptr = self._tag_component_map.lock() * if lockedptr: - * it = deref(lockedptr).data.begin() # <<<<<<<<<<<<<< - * while it != deref(lockedptr).data.end(): - * yield deref(it).first, Component.create(deref(it).second) + * for cpair in deref(lockedptr).data: # <<<<<<<<<<<<<< + * yield cpair.first, Component.create(cpair.second) + * else: */ - __pyx_cur_scope->__pyx_v_it = (*__pyx_cur_scope->__pyx_v_lockedptr).data.begin(); - - /* "khmer/_oxli.pyx":110 + __pyx_t_3 = &(*__pyx_cur_scope->__pyx_v_lockedptr).data; + __pyx_t_2 = __pyx_t_3->begin(); + for (;;) { + if (!(__pyx_t_2 != __pyx_t_3->end())) break; + __pyx_t_4 = *__pyx_t_2; + ++__pyx_t_2; + __pyx_cur_scope->__pyx_v_cpair = __pyx_t_4; + + /* "khmer/_oxli.pyx":141 * if lockedptr: - * it = deref(lockedptr).data.begin() - * while it != deref(lockedptr).data.end(): # <<<<<<<<<<<<<< - * yield deref(it).first, Component.create(deref(it).second) - * inc(it) - */ - while (1) { - __pyx_t_2 = ((__pyx_cur_scope->__pyx_v_it != (*__pyx_cur_scope->__pyx_v_lockedptr).data.end()) != 0); - if (!__pyx_t_2) break; - - /* "khmer/_oxli.pyx":111 - * it = deref(lockedptr).data.begin() - * while it != deref(lockedptr).data.end(): - * yield deref(it).first, Component.create(deref(it).second) # <<<<<<<<<<<<<< - * inc(it) + * for cpair in deref(lockedptr).data: + * yield cpair.first, Component.create(cpair.second) # <<<<<<<<<<<<<< * else: + * raise MemoryError("Can't locked underlying Component set") */ - __pyx_t_3 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_cur_scope->__pyx_v_it).first); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 111, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create((*__pyx_cur_scope->__pyx_v_it).second)); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 111, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 111, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_cur_scope->__pyx_v_cpair.first); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4); - __pyx_t_3 = 0; - __pyx_t_4 = 0; - __pyx_r = __pyx_t_5; + __pyx_t_6 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_cur_scope->__pyx_v_cpair.second)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 141, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 141, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6); __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_r = __pyx_t_7; + __pyx_t_7 = 0; + __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); /* return from generator, yielding value */ __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L7_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 111, __pyx_L1_error) + __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; + __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 141, __pyx_L1_error) - /* "khmer/_oxli.pyx":112 - * while it != deref(lockedptr).data.end(): - * yield deref(it).first, Component.create(deref(it).second) - * inc(it) # <<<<<<<<<<<<<< + /* "khmer/_oxli.pyx":140 + * lockedptr = self._tag_component_map.lock() + * if lockedptr: + * for cpair in deref(lockedptr).data: # <<<<<<<<<<<<<< + * yield cpair.first, Component.create(cpair.second) * else: - * raise MemoryError("Can't locked underlying Component set") */ - (++__pyx_cur_scope->__pyx_v_it); } - /* "khmer/_oxli.pyx":108 - * cdef shared_ptr[GuardedKmerCompMap] locked + /* "khmer/_oxli.pyx":139 + * cdef pair[HashIntoType,ComponentPtr] cpair * lockedptr = self._tag_component_map.lock() * if lockedptr: # <<<<<<<<<<<<<< - * it = deref(lockedptr).data.begin() - * while it != deref(lockedptr).data.end(): + * for cpair in deref(lockedptr).data: + * yield cpair.first, Component.create(cpair.second) */ goto __pyx_L4; } - /* "khmer/_oxli.pyx":114 - * inc(it) + /* "khmer/_oxli.pyx":143 + * yield cpair.first, Component.create(cpair.second) * else: * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< * - * property n_components: + * def write_components(self, filename): */ /*else*/ { - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 114, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_Raise(__pyx_t_5, 0, 0, 0); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 114, __pyx_L1_error) + __pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 143, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_Raise(__pyx_t_7, 0, 0, 0); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_ERR(1, 143, __pyx_L1_error) } __pyx_L4:; if (1); else __pyx_cur_scope = __pyx_cur_scope; - /* "khmer/_oxli.pyx":105 + /* "khmer/_oxli.pyx":135 * raise MemoryError("Can't locked underlying Component set") * * def tag_components(self): # <<<<<<<<<<<<<< * cdef shared_ptr[GuardedKmerCompMap] locked - * lockedptr = self._tag_component_map.lock() + * cdef pair[HashIntoType,ComponentPtr] cpair */ /* function exit code */ PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); __Pyx_AddTraceback("tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_r = 0; @@ -2818,7 +3394,184 @@ static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_13generator2(__py return __pyx_r; } -/* "khmer/_oxli.pyx":117 +/* "khmer/_oxli.pyx":145 + * raise MemoryError("Can't locked underlying Component set") + * + * def write_components(self, filename): # <<<<<<<<<<<<<< + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_17write_components(PyObject *__pyx_v_self, PyObject *__pyx_v_filename); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_17write_components(PyObject *__pyx_v_self, PyObject *__pyx_v_filename) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("write_components (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_16write_components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_filename)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_16write_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename) { + FILE *__pyx_v_fp; + khmer::ComponentPtr __pyx_v_cmpptr; + std::shared_ptr __pyx_v_lockedptr; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + char const *__pyx_t_3; + int __pyx_t_4; + bool __pyx_t_5; + std::set ::iterator __pyx_t_6; + khmer::ComponentPtrSet *__pyx_t_7; + khmer::ComponentPtr __pyx_t_8; + __Pyx_RefNannySetupContext("write_components", 0); + + /* "khmer/_oxli.pyx":147 + * def write_components(self, filename): + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') # <<<<<<<<<<<<<< + * if fp == NULL: + * raise IOError("Can't open file.") + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_filename, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 147, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 147, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __Pyx_PyObject_AsString(__pyx_t_2); if (unlikely((!__pyx_t_3) && PyErr_Occurred())) __PYX_ERR(1, 147, __pyx_L1_error) + __pyx_v_fp = fopen(__pyx_t_3, ((char const *)"wb")); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "khmer/_oxli.pyx":148 + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') + * if fp == NULL: # <<<<<<<<<<<<<< + * raise IOError("Can't open file.") + * + */ + __pyx_t_4 = ((__pyx_v_fp == NULL) != 0); + if (__pyx_t_4) { + + /* "khmer/_oxli.pyx":149 + * fp = fopen(filename.encode('utf-8'), 'wb') + * if fp == NULL: + * raise IOError("Can't open file.") # <<<<<<<<<<<<<< + * + * cdef ComponentPtr cmpptr + */ + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 149, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(1, 149, __pyx_L1_error) + + /* "khmer/_oxli.pyx":148 + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') + * if fp == NULL: # <<<<<<<<<<<<<< + * raise IOError("Can't open file.") + * + */ + } + + /* "khmer/_oxli.pyx":153 + * cdef ComponentPtr cmpptr + * cdef shared_ptr[ComponentPtrSet] lockedptr + * lockedptr = self._components.lock() # <<<<<<<<<<<<<< + * + * if lockedptr: + */ + __pyx_v_lockedptr = __pyx_v_self->_components.lock(); + + /* "khmer/_oxli.pyx":155 + * lockedptr = self._components.lock() + * + * if lockedptr: # <<<<<<<<<<<<<< + * for cmpptr in deref(lockedptr): + * fprintf(fp, "%llu,%llu,%f\n", + */ + __pyx_t_5 = __pyx_v_lockedptr.operator bool(); + if (__pyx_t_5) { + + /* "khmer/_oxli.pyx":156 + * + * if lockedptr: + * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< + * fprintf(fp, "%llu,%llu,%f\n", + * deref(cmpptr).component_id, + */ + __pyx_t_7 = &(*__pyx_v_lockedptr); + __pyx_t_6 = __pyx_t_7->begin(); + for (;;) { + if (!(__pyx_t_6 != __pyx_t_7->end())) break; + __pyx_t_8 = *__pyx_t_6; + ++__pyx_t_6; + __pyx_v_cmpptr = __pyx_t_8; + + /* "khmer/_oxli.pyx":157 + * if lockedptr: + * for cmpptr in deref(lockedptr): + * fprintf(fp, "%llu,%llu,%f\n", # <<<<<<<<<<<<<< + * deref(cmpptr).component_id, + * deref(cmpptr).get_n_tags(), + */ + fprintf(__pyx_v_fp, ((char const *)"%llu,%llu,%f\n"), (*__pyx_v_cmpptr).component_id, (*__pyx_v_cmpptr).get_n_tags(), __pyx_f_5khmer_5_oxli_9Component__mean_tag_count(__pyx_v_cmpptr, __pyx_v_self->_graph_ptr)); + + /* "khmer/_oxli.pyx":156 + * + * if lockedptr: + * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< + * fprintf(fp, "%llu,%llu,%f\n", + * deref(cmpptr).component_id, + */ + } + + /* "khmer/_oxli.pyx":155 + * lockedptr = self._components.lock() + * + * if lockedptr: # <<<<<<<<<<<<<< + * for cmpptr in deref(lockedptr): + * fprintf(fp, "%llu,%llu,%f\n", + */ + } + + /* "khmer/_oxli.pyx":161 + * deref(cmpptr).get_n_tags(), + * Component._mean_tag_count(cmpptr, self._graph_ptr)) + * fclose(fp) # <<<<<<<<<<<<<< + * + * property n_components: + */ + fclose(__pyx_v_fp); + + /* "khmer/_oxli.pyx":145 + * raise MemoryError("Can't locked underlying Component set") + * + * def write_components(self, filename): # <<<<<<<<<<<<<< + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.write_components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":164 * * property n_components: * def __get__(self): # <<<<<<<<<<<<<< @@ -2845,20 +3598,21 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___ PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - /* "khmer/_oxli.pyx":118 + /* "khmer/_oxli.pyx":165 * property n_components: * def __get__(self): * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< * + * property n_tags: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 118, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 165, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "khmer/_oxli.pyx":117 + /* "khmer/_oxli.pyx":164 * * property n_components: * def __get__(self): # <<<<<<<<<<<<<< @@ -2877,40 +3631,200 @@ static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___ return __pyx_r; } -/* "string.from_py":13 +/* "khmer/_oxli.pyx":168 * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * property n_tags: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_tags() */ -static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { - Py_ssize_t __pyx_v_length; - char *__pyx_v_data; - std::string __pyx_r; +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_6n_tags_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_6n_tags_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - char *__pyx_t_1; - __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6n_tags___get__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); - /* "string.from_py":15 - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< - * return string(data, length) - * - */ - __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(2, 15, __pyx_L1_error) - __pyx_v_data = __pyx_t_1; + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "string.from_py":16 - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - * return string(data, length) # <<<<<<<<<<<<<< - * - * - */ - __pyx_r = std::string(__pyx_v_data, __pyx_v_length); +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6n_tags___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli.pyx":169 + * property n_tags: + * def __get__(self): + * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_tags()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 169, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli.pyx":168 + * + * property n_tags: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_tags() + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.n_tags.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli.pyx":86 + * cdef weak_ptr[GuardedKmerCompMap] _tag_component_map + * cdef CyHashtable * _graph_ptr + * cdef readonly uint64_t n_consumed # <<<<<<<<<<<<<< + * + * def __cinit__(self, graph): + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_10n_consumed_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_10n_consumed_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10n_consumed___get__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10n_consumed___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t(__pyx_v_self->n_consumed); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.n_consumed.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "vector.to_py":67 + * + * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") + * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): # <<<<<<<<<<<<<< + * return [X_to_py(v[i]) for i in range(v.size())] + * + */ + +static PyObject *__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(const std::vector &__pyx_v_v) { + size_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + size_t __pyx_t_2; + size_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType", 0); + + /* "vector.to_py":68 + * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") + * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): + * return [X_to_py(v[i]) for i in range(v.size())] # <<<<<<<<<<<<<< + * + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_v_v.size(); + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + __pyx_t_4 = __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "vector.to_py":67 + * + * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") + * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): # <<<<<<<<<<<<<< + * return [X_to_py(v[i]) for i in range(v.size())] + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("vector.to_py.__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { + Py_ssize_t __pyx_v_length; + char *__pyx_v_data; + std::string __pyx_r; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); + + /* "string.from_py":15 + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< + * return string(data, length) + * + */ + __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(2, 15, __pyx_L1_error) + __pyx_v_data = __pyx_t_1; + + /* "string.from_py":16 + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * return string(data, length) # <<<<<<<<<<<<<< + * + * + */ + __pyx_r = std::string(__pyx_v_data, __pyx_v_length); goto __pyx_L0; /* "string.from_py":13 @@ -2973,6 +3887,7 @@ static PyObject *__pyx_getprop_5khmer_5_oxli_9Component__n_destroyed(PyObject *o } static PyMethodDef __pyx_methods_5khmer_5_oxli_Component[] = { + {"tag_counts", (PyCFunction)__pyx_pw_5khmer_5_oxli_9Component_12tag_counts, METH_VARARGS|METH_KEYWORDS, 0}, {0, 0, 0, 0} }; @@ -3097,17 +4012,29 @@ static PyObject *__pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_components return __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(o); } +static PyObject *__pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_tags(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_6n_tags_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_consumed(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_10n_consumed_1__get__(o); +} + static PyMethodDef __pyx_methods_5khmer_5_oxli_StreamingPartitioner[] = { {"consume_sequence", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence, METH_O, 0}, - {"get_tag_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5get_tag_component, METH_O, 0}, - {"get_nearest_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_nearest_component, METH_O, 0}, - {"components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9components, METH_NOARGS, 0}, - {"tag_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12tag_components, METH_NOARGS, 0}, + {"consume_fasta", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5consume_fasta, METH_O, 0}, + {"get_tag_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_tag_component, METH_O, 0}, + {"get_nearest_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9get_nearest_component, METH_O, 0}, + {"components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_11components, METH_NOARGS, 0}, + {"tag_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_14tag_components, METH_NOARGS, 0}, + {"write_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_17write_components, METH_O, 0}, {0, 0, 0, 0} }; static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_StreamingPartitioner[] = { {(char *)"n_components", __pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_components, 0, (char *)0, 0}, + {(char *)"n_tags", __pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_tags, 0, (char *)0, 0}, + {(char *)"n_consumed", __pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_consumed, 0, (char *)0, 0}, {0, 0, 0, 0, 0} }; @@ -3185,14 +4112,14 @@ static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(PyTypeO if (unlikely(!o)) return 0; } p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o); - new((void*)&(p->__pyx_v_it)) std::set ::iterator(); + new((void*)&(p->__pyx_t_0)) std::set ::iterator(); return o; } static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct____iter__(PyObject *o) { struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o; PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_v_it); + __Pyx_call_destructor(p->__pyx_t_0); Py_CLEAR(p->__pyx_v_self); if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__ < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__)))) { __pyx_freelist_5khmer_5_oxli___pyx_scope_struct____iter__[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o); @@ -3293,18 +4220,20 @@ static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components(PyTy if (unlikely(!o)) return 0; } p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o); - new((void*)&(p->__pyx_v_it)) std::set ::iterator(); + new((void*)&(p->__pyx_v_cmpptr)) khmer::ComponentPtr(); new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); + new((void*)&(p->__pyx_t_0)) std::set ::iterator(); return o; } static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_1_components(PyObject *o) { struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o; PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_v_it); + __Pyx_call_destructor(p->__pyx_v_cmpptr); __Pyx_call_destructor(p->__pyx_v_locked); __Pyx_call_destructor(p->__pyx_v_lockedptr); + __Pyx_call_destructor(p->__pyx_t_0); Py_CLEAR(p->__pyx_v_self); if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components)))) { __pyx_freelist_5khmer_5_oxli___pyx_scope_struct_1_components[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o); @@ -3405,18 +4334,20 @@ static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components( if (unlikely(!o)) return 0; } p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o); - new((void*)&(p->__pyx_v_it)) std::map ::iterator(); + new((void*)&(p->__pyx_v_cpair)) std::pair (); new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); + new((void*)&(p->__pyx_t_0)) std::map ::iterator(); return o; } static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyObject *o) { struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o; PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_v_it); + __Pyx_call_destructor(p->__pyx_v_cpair); __Pyx_call_destructor(p->__pyx_v_locked); __Pyx_call_destructor(p->__pyx_v_lockedptr); + __Pyx_call_destructor(p->__pyx_t_0); Py_CLEAR(p->__pyx_v_self); if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components)))) { __pyx_freelist_5khmer_5_oxli___pyx_scope_struct_2_tag_components[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o); @@ -3525,8 +4456,10 @@ static struct PyModuleDef __pyx_moduledef = { static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_Can_t_locked_underlying_Componen, __pyx_k_Can_t_locked_underlying_Componen, sizeof(__pyx_k_Can_t_locked_underlying_Componen), 0, 0, 1, 0}, + {&__pyx_kp_s_Can_t_open_file, __pyx_k_Can_t_open_file, sizeof(__pyx_k_Can_t_open_file), 0, 0, 1, 0}, {&__pyx_n_s_Component___iter, __pyx_k_Component___iter, sizeof(__pyx_k_Component___iter), 0, 0, 1, 1}, {&__pyx_n_s_Countgraph, __pyx_k_Countgraph, sizeof(__pyx_k_Countgraph), 0, 0, 1, 1}, + {&__pyx_n_s_IOError, __pyx_k_IOError, sizeof(__pyx_k_IOError), 0, 0, 1, 1}, {&__pyx_n_s_MemoryError, __pyx_k_MemoryError, sizeof(__pyx_k_MemoryError), 0, 0, 1, 1}, {&__pyx_kp_s_Must_take_an_object_with_Hashtab, __pyx_k_Must_take_an_object_with_Hashtab, sizeof(__pyx_k_Must_take_an_object_with_Hashtab), 0, 0, 1, 0}, {&__pyx_n_s_Nodegraph, __pyx_k_Nodegraph, sizeof(__pyx_k_Nodegraph), 0, 0, 1, 1}, @@ -3537,10 +4470,12 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, + {&__pyx_n_s_component, __pyx_k_component, sizeof(__pyx_k_component), 0, 0, 1, 1}, {&__pyx_n_s_component_id, __pyx_k_component_id, sizeof(__pyx_k_component_id), 0, 0, 1, 1}, {&__pyx_n_s_components, __pyx_k_components, sizeof(__pyx_k_components), 0, 0, 1, 1}, {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, {&__pyx_n_s_graph, __pyx_k_graph, sizeof(__pyx_k_graph), 0, 0, 1, 1}, + {&__pyx_n_s_graph_ptr, __pyx_k_graph_ptr, sizeof(__pyx_k_graph_ptr), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, {&__pyx_n_s_iter, __pyx_k_iter, sizeof(__pyx_k_iter), 0, 0, 1, 1}, {&__pyx_n_s_khmer__khmer, __pyx_k_khmer__khmer, sizeof(__pyx_k_khmer__khmer), 0, 0, 1, 1}, @@ -3548,17 +4483,24 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_other, __pyx_k_other, sizeof(__pyx_k_other), 0, 0, 1, 1}, {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, + {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, + {&__pyx_n_s_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 0, 0, 1, 1}, {&__pyx_n_s_tag_components, __pyx_k_tag_components, sizeof(__pyx_k_tag_components), 0, 0, 1, 1}, + {&__pyx_n_s_tag_counts, __pyx_k_tag_counts, sizeof(__pyx_k_tag_counts), 0, 0, 1, 1}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, + {&__pyx_kp_s_work_khmer_khmer__oxli_pyx, __pyx_k_work_khmer_khmer__oxli_pyx, sizeof(__pyx_k_work_khmer_khmer__oxli_pyx), 0, 0, 1, 0}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 65, __pyx_L1_error) + __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 68, __pyx_L1_error) + __pyx_builtin_staticmethod = __Pyx_GetBuiltinName(__pyx_n_s_staticmethod); if (!__pyx_builtin_staticmethod) __PYX_ERR(1, 65, __pyx_L1_error) __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(1, 46, __pyx_L1_error) - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 64, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 90, __pyx_L1_error) + __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s_IOError); if (!__pyx_builtin_IOError) __PYX_ERR(1, 149, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -3579,71 +4521,116 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); - /* "khmer/_oxli.pyx":64 + /* "khmer/_oxli.pyx":90 * def __cinit__(self, graph): * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< * * */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 64, __pyx_L1_error) + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 90, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); - /* "khmer/_oxli.pyx":76 + /* "khmer/_oxli.pyx":103 * * def consume_sequence(self, sequence): * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< + * self.n_consumed += 1 * - * def get_tag_component(self, kmer): */ - __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 103, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3); - /* "khmer/_oxli.pyx":80 + /* "khmer/_oxli.pyx":107 + * + * def consume_fasta(self, filename): + * return deref(self._this).consume_fasta(filename.encode('utf-8')) # <<<<<<<<<<<<<< + * + * def get_tag_component(self, kmer): + */ + __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__4); + __Pyx_GIVEREF(__pyx_tuple__4); + + /* "khmer/_oxli.pyx":111 * def get_tag_component(self, kmer): * cdef ComponentPtr compptr * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< * if compptr == NULL: * return None */ - __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 80, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__4); - __Pyx_GIVEREF(__pyx_tuple__4); + __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 111, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__5); + __Pyx_GIVEREF(__pyx_tuple__5); - /* "khmer/_oxli.pyx":88 + /* "khmer/_oxli.pyx":119 * def get_nearest_component(self, kmer): * cdef ComponentPtr compptr * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< * if compptr == NULL: * return None */ - __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 88, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__5); - __Pyx_GIVEREF(__pyx_tuple__5); + __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 119, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__6); + __Pyx_GIVEREF(__pyx_tuple__6); - /* "khmer/_oxli.pyx":103 - * inc(it) + /* "khmer/_oxli.pyx":133 + * yield Component.create(cmpptr) * else: * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< * * def tag_components(self): */ - __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 103, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__6); - __Pyx_GIVEREF(__pyx_tuple__6); + __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 133, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__7); + __Pyx_GIVEREF(__pyx_tuple__7); - /* "khmer/_oxli.pyx":114 - * inc(it) + /* "khmer/_oxli.pyx":143 + * yield cpair.first, Component.create(cpair.second) * else: * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< * - * property n_components: + * def write_components(self, filename): */ - __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 114, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__7); - __Pyx_GIVEREF(__pyx_tuple__7); + __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(1, 143, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); + + /* "khmer/_oxli.pyx":147 + * def write_components(self, filename): + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') # <<<<<<<<<<<<<< + * if fp == NULL: + * raise IOError("Can't open file.") + */ + __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 147, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__9); + __Pyx_GIVEREF(__pyx_tuple__9); + + /* "khmer/_oxli.pyx":149 + * fp = fopen(filename.encode('utf-8'), 'wb') + * if fp == NULL: + * raise IOError("Can't open file.") # <<<<<<<<<<<<<< + * + * cdef ComponentPtr cmpptr + */ + __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_s_Can_t_open_file); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(1, 149, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); + + /* "khmer/_oxli.pyx":66 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CyCpHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + __pyx_tuple__11 = PyTuple_Pack(3, __pyx_n_s_component, __pyx_n_s_graph, __pyx_n_s_graph_ptr); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__11); + __Pyx_GIVEREF(__pyx_tuple__11); + __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_work_khmer_khmer__oxli_pyx, __pyx_n_s_tag_counts, 66, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(1, 66, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -3746,22 +4733,24 @@ PyMODINIT_FUNC PyInit__oxli(void) /*--- Type init code ---*/ __pyx_vtabptr_5khmer_5_oxli_Component = &__pyx_vtable_5khmer_5_oxli_Component; __pyx_vtable_5khmer_5_oxli_Component.create = (struct __pyx_obj_5khmer_5_oxli_Component *(*)(khmer::ComponentPtr))__pyx_f_5khmer_5_oxli_9Component_create; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 10, __pyx_L1_error) + __pyx_vtable_5khmer_5_oxli_Component._tag_counts = (std::vector (*)(khmer::ComponentPtr, khmer::Hashtable *))__pyx_f_5khmer_5_oxli_9Component__tag_counts; + __pyx_vtable_5khmer_5_oxli_Component._mean_tag_count = (float (*)(khmer::ComponentPtr, khmer::Hashtable *))__pyx_f_5khmer_5_oxli_9Component__mean_tag_count; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 11, __pyx_L1_error) __pyx_type_5khmer_5_oxli_Component.tp_print = 0; - if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_Component.tp_dict, __pyx_vtabptr_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 10, __pyx_L1_error) - if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 10, __pyx_L1_error) + if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_Component.tp_dict, __pyx_vtabptr_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 11, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 11, __pyx_L1_error) __pyx_ptype_5khmer_5_oxli_Component = &__pyx_type_5khmer_5_oxli_Component; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 55, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 80, __pyx_L1_error) __pyx_type_5khmer_5_oxli_StreamingPartitioner.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 55, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 80, __pyx_L1_error) __pyx_ptype_5khmer_5_oxli_StreamingPartitioner = &__pyx_type_5khmer_5_oxli_StreamingPartitioner; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 33, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 34, __pyx_L1_error) __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__.tp_print = 0; __pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = &__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components) < 0) __PYX_ERR(1, 94, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components) < 0) __PYX_ERR(1, 125, __pyx_L1_error) __pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components.tp_print = 0; __pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components = &__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components) < 0) __PYX_ERR(1, 105, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components) < 0) __PYX_ERR(1, 135, __pyx_L1_error) __pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components.tp_print = 0; __pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components = &__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components; /*--- Type import code ---*/ @@ -3772,47 +4761,105 @@ PyMODINIT_FUNC PyInit__oxli(void) if (__Pyx_patch_abc() < 0) __PYX_ERR(1, 1, __pyx_L1_error) #endif - /* "khmer/_oxli.pyx":7 - * from libc.stdint cimport uintptr_t + /* "khmer/_oxli.pyx":8 + * from libc.stdio cimport FILE, fopen, fwrite, fclose, stdout, stderr, fprintf * * from khmer._khmer import Countgraph # <<<<<<<<<<<<<< * from khmer._khmer import Nodegraph * */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_Countgraph); __Pyx_GIVEREF(__pyx_n_s_Countgraph); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_Countgraph); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(1, 7, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(1, 8, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "khmer/_oxli.pyx":8 + /* "khmer/_oxli.pyx":9 * * from khmer._khmer import Countgraph * from khmer._khmer import Nodegraph # <<<<<<<<<<<<<< * * cdef class Component: */ - __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_Nodegraph); __Pyx_GIVEREF(__pyx_n_s_Nodegraph); PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Nodegraph); - __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "khmer/_oxli.pyx":66 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CyCpHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5khmer_5_oxli_9Component_12tag_counts, NULL, __pyx_n_s_khmer__oxli); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + + /* "khmer/_oxli.pyx":65 + * return counts + * + * @staticmethod # <<<<<<<<<<<<<< + * def tag_counts(Component component, graph): + * cdef CyCpHashtable_Object* graph_ptr = graph + */ + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_staticmethod, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5khmer_5_oxli_Component->tp_dict, __pyx_n_s_tag_counts, __pyx_t_1) < 0) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5khmer_5_oxli_Component); + + /* "khmer/_oxli.pyx":66 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CyCpHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + __pyx_t_1 = __Pyx_GetNameInClass((PyObject *)__pyx_ptype_5khmer_5_oxli_Component, __pyx_n_s_tag_counts); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + + /* "khmer/_oxli.pyx":65 + * return counts + * + * @staticmethod # <<<<<<<<<<<<<< + * def tag_counts(Component component, graph): + * cdef CyCpHashtable_Object* graph_ptr = graph + */ + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_staticmethod, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5khmer_5_oxli_Component->tp_dict, __pyx_n_s_tag_counts, __pyx_t_1) < 0) __PYX_ERR(1, 66, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5khmer_5_oxli_Component); /* "khmer/_oxli.pyx":1 * import cython # <<<<<<<<<<<<<< @@ -4348,6 +5395,108 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject } #endif +/* RaiseTooManyValuesToUnpack */ + static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* RaiseNeedMoreValuesToUnpack */ + static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* IterFinish */ + static CYTHON_INLINE int __Pyx_IterFinish(void) { +#if CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = PyThreadState_GET(); + PyObject* exc_type = tstate->curexc_type; + if (unlikely(exc_type)) { + if (likely(exc_type == PyExc_StopIteration) || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) { + PyObject *exc_value, *exc_tb; + exc_value = tstate->curexc_value; + exc_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + Py_DECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + return 0; + } else { + return -1; + } + } + return 0; +#else + if (unlikely(PyErr_Occurred())) { + if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) { + PyErr_Clear(); + return 0; + } else { + return -1; + } + } + return 0; +#endif +} + +/* UnpackItemEndCheck */ + static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { + if (unlikely(retval)) { + Py_DECREF(retval); + __Pyx_RaiseTooManyValuesError(expected); + return -1; + } else { + return __Pyx_IterFinish(); + } + return 0; +} + +/* WriteUnraisableException */ + static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, + CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, + int full_traceback, CYTHON_UNUSED int nogil) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_PyThreadState_declare +#ifdef WITH_THREAD + PyGILState_STATE state; + if (nogil) + state = PyGILState_Ensure(); +#ifdef _MSC_VER + else state = (PyGILState_STATE)-1; +#endif +#endif + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + if (full_traceback) { + Py_XINCREF(old_exc); + Py_XINCREF(old_val); + Py_XINCREF(old_tb); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + PyErr_PrintEx(1); + } + #if PY_MAJOR_VERSION < 3 + ctx = PyString_FromString(name); + #else + ctx = PyUnicode_FromString(name); + #endif + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } +#ifdef WITH_THREAD + if (nogil) + PyGILState_Release(state); +#endif +} + /* GetModuleGlobalName */ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { PyObject *result; @@ -4472,6 +5621,15 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject return value; } +/* GetNameInClass */ + static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name) { + PyObject *result; + result = __Pyx_PyObject_GetAttrStr(nmspace, name); + if (!result) + result = __Pyx_GetModuleGlobalName(name); + return result; +} + /* CodeObjectCache */ static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; @@ -4726,37 +5884,6 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } } -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - /* CIntFromPyVerify */ #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) @@ -4779,20 +5906,51 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, return (target_type) value;\ } +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType(khmer::BoundedCounterType value) { + const khmer::BoundedCounterType neg_one = (khmer::BoundedCounterType) -1, const_zero = (khmer::BoundedCounterType) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(khmer::BoundedCounterType) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(khmer::BoundedCounterType) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::BoundedCounterType) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(khmer::BoundedCounterType) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::BoundedCounterType) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(khmer::BoundedCounterType), + little, !is_unsigned); + } +} + /* CIntFromPy */ - static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { - const long neg_one = (long) -1, const_zero = (long) 0; + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { - if (sizeof(long) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { goto raise_neg_overflow; } - return (long) val; + return (int) val; } } else #endif @@ -4801,32 +5959,32 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { - case 0: return (long) 0; - case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { - return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); } } break; case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { - return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); } } break; case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { - return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); } } break; @@ -4840,86 +5998,86 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, { int result = PyObject_RichCompareBool(x, Py_False, Py_LT); if (unlikely(result < 0)) - return (long) -1; + return (int) -1; if (unlikely(result == 1)) goto raise_neg_overflow; } #endif - if (sizeof(long) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { - case 0: return (long) 0; - case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) case -2: - if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case -3: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case -4: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; } #endif - if (sizeof(long) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } @@ -4928,7 +6086,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else - long val; + int val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { @@ -4948,40 +6106,40 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, return val; } #endif - return (long) -1; + return (int) -1; } } else { - long val; + int val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (long) -1; - val = __Pyx_PyInt_As_long(tmp); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); Py_DECREF(tmp); return val; } raise_overflow: PyErr_SetString(PyExc_OverflowError, - "value too large to convert to long"); - return (long) -1; + "value too large to convert to int"); + return (int) -1; raise_neg_overflow: PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to long"); - return (long) -1; + "can't convert negative value to int"); + return (int) -1; } /* CIntFromPy */ - static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) -1, const_zero = (int) 0; + static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *x) { + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { - if (sizeof(int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + if (sizeof(uint64_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { goto raise_neg_overflow; } - return (int) val; + return (uint64_t) val; } } else #endif @@ -4990,32 +6148,32 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { - case 0: return (int) 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 0: return (uint64_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, digits[0]) case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { - return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 2 * PyLong_SHIFT) { + return (uint64_t) (((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); } } break; case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { - return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 3 * PyLong_SHIFT) { + return (uint64_t) (((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); } } break; case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { - return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 4 * PyLong_SHIFT) { + return (uint64_t) (((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); } } break; @@ -5029,86 +6187,86 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, { int result = PyObject_RichCompareBool(x, Py_False, Py_LT); if (unlikely(result < 0)) - return (int) -1; + return (uint64_t) -1; if (unlikely(result == 1)) goto raise_neg_overflow; } #endif - if (sizeof(int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + if (sizeof(uint64_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { - case 0: return (int) 0; - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case 0: return (uint64_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(uint64_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, +digits[0]) case -2: - if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) - 1 > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { + return (uint64_t) ((((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case -3: - if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { + return (uint64_t) ((((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case -4: - if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { + return (uint64_t) ((((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; } #endif - if (sizeof(int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + if (sizeof(uint64_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } @@ -5117,7 +6275,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else - int val; + uint64_t val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { @@ -5137,24 +6295,433 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, return val; } #endif - return (int) -1; + return (uint64_t) -1; } } else { - int val; + uint64_t val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (int) -1; - val = __Pyx_PyInt_As_int(tmp); + if (!tmp) return (uint64_t) -1; + val = __Pyx_PyInt_As_uint64_t(tmp); Py_DECREF(tmp); return val; } raise_overflow: PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int) -1; + "value too large to convert to uint64_t"); + return (uint64_t) -1; raise_neg_overflow: PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to int"); - return (int) -1; + "can't convert negative value to uint64_t"); + return (uint64_t) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (size_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) + case -2: + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } +#endif + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + size_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (size_t) -1; + } + } else { + size_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case -2: + if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (long) -1; + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (long) -1; + val = __Pyx_PyInt_As_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; } /* FetchCommonType */ From 9d000076980305f00e28fe01c6460aaf505b00e8 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 13 Nov 2016 17:43:16 -0800 Subject: [PATCH 027/185] consume_fasta dec and start porting Traversal --- khmer/_oxli.pxd | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index bc0323bbb3..b60f72e81d 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -41,6 +41,14 @@ cdef extern from "kmer_hash.hh" namespace "khmer": bool is_forward() const + + +cdef extern from "traversal.hh": + cdef cppclass CyLeftNodeGatherer "khmer::NodeGatherer<0>": + CyLeftNodeGatherer(CyHashtable *) + void push_filter() + + cdef extern from "partitioning.hh" namespace "khmer": cdef cppclass CyComponent "khmer::Component": @@ -68,6 +76,7 @@ cdef extern from "partitioning.hh" namespace "khmer": CyStreamingPartitioner(CyHashtable * ) except +MemoryError void consume_sequence(string&) except +MemoryError + uint64_t consume_fasta(string&) except +MemoryError void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) void find_connected_tags(queue[Kmer]&, set[HashIntoType]&, From b50ae1b21f9a0008c25cd9b2552462becf095d05 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 13 Nov 2016 17:48:17 -0800 Subject: [PATCH 028/185] Tag counts and consume fasta exposure --- khmer/_oxli.pyx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/khmer/_oxli.pyx b/khmer/_oxli.pyx index 9441fb3c12..95bb8aae74 100644 --- a/khmer/_oxli.pyx +++ b/khmer/_oxli.pyx @@ -62,6 +62,11 @@ cdef class Component: counts[idx] = deref(graph).get_count(tag) return counts + @staticmethod + def tag_counts(Component component, graph): + cdef CyCpHashtable_Object* graph_ptr = graph + return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + @staticmethod cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): cdef uint64_t n_tags = deref(comp).get_n_tags() @@ -98,6 +103,9 @@ cdef class StreamingPartitioner: deref(self._this).consume_sequence(sequence.encode('utf-8')) self.n_consumed += 1 + def consume_fasta(self, filename): + return deref(self._this).consume_fasta(filename.encode('utf-8')) + def get_tag_component(self, kmer): cdef ComponentPtr compptr compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) From 3cbb8d04121950bb3f2edde4cf9a1a25301c9b16 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 13 Nov 2016 18:02:34 -0800 Subject: [PATCH 029/185] Add a pop_filter method to Traverser --- lib/traversal.cc | 6 ++++++ lib/traversal.hh | 1 + 2 files changed, 7 insertions(+) diff --git a/lib/traversal.cc b/lib/traversal.cc index 51d27855f0..dc2de55131 100644 --- a/lib/traversal.cc +++ b/lib/traversal.cc @@ -213,6 +213,12 @@ void Traverser::push_filter(KmerFilter filter) right_gatherer.push_filter(filter); } +KmerFilter Traverser::pop_filter() +{ + left_gatherer.pop_filter(); + return right_gatherer.pop_filter(); +} + unsigned int Traverser::traverse(const Kmer& node, KmerQueue& node_q) const diff --git a/lib/traversal.hh b/lib/traversal.hh index dc2fb1506d..6eab0f3e52 100644 --- a/lib/traversal.hh +++ b/lib/traversal.hh @@ -218,6 +218,7 @@ public: KmerFilter filter); void push_filter(KmerFilter filter); + KmerFilter pop_filter(); unsigned int traverse(const Kmer& node, KmerQueue& node_q) const; From 8fe4326c6d03ca83e6301906127bbb4dddc3edfb Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 13 Nov 2016 18:02:59 -0800 Subject: [PATCH 030/185] Expose Traverser in pxd --- khmer/_oxli.cpp | 11 ++++++----- khmer/_oxli.pxd | 17 +++++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/khmer/_oxli.cpp b/khmer/_oxli.cpp index d6dac5bca4..24f68dae40 100644 --- a/khmer/_oxli.cpp +++ b/khmer/_oxli.cpp @@ -428,6 +428,7 @@ static CYTHON_INLINE float __PYX_NAN() { #include "hashtable.hh" #include "_khmer.hh" #include "kmer_hash.hh" +#include "traversal.hh" #include "partitioning.hh" #include #include @@ -1210,8 +1211,8 @@ static const char __pyx_k_Can_t_open_file[] = "Can't open file."; static const char __pyx_k_Component___iter[] = "Component.__iter__"; static const char __pyx_k_NotImplementedError[] = "NotImplementedError"; static const char __pyx_k_Operator_not_available[] = "Operator not available."; -static const char __pyx_k_work_khmer_khmer__oxli_pyx[] = "/work/khmer/khmer/_oxli.pyx"; static const char __pyx_k_StreamingPartitioner_components[] = "StreamingPartitioner.components"; +static const char __pyx_k_Users_camille_w_khmer_khmer__ox[] = "/Users/camille/w/khmer/khmer/_oxli.pyx"; static const char __pyx_k_Can_t_locked_underlying_Componen[] = "Can't locked underlying Component set"; static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; static const char __pyx_k_StreamingPartitioner_tag_compone[] = "StreamingPartitioner.tag_components"; @@ -1227,6 +1228,7 @@ static PyObject *__pyx_n_s_NotImplementedError; static PyObject *__pyx_kp_s_Operator_not_available; static PyObject *__pyx_n_s_StreamingPartitioner_components; static PyObject *__pyx_n_s_StreamingPartitioner_tag_compone; +static PyObject *__pyx_kp_s_Users_camille_w_khmer_khmer__ox; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_n_s_args; static PyObject *__pyx_n_s_close; @@ -1251,7 +1253,6 @@ static PyObject *__pyx_n_s_tag_counts; static PyObject *__pyx_n_s_test; static PyObject *__pyx_n_s_throw; static PyObject *__pyx_kp_s_utf_8; -static PyObject *__pyx_kp_s_work_khmer_khmer__oxli_pyx; static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ @@ -4467,6 +4468,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_Operator_not_available, __pyx_k_Operator_not_available, sizeof(__pyx_k_Operator_not_available), 0, 0, 1, 0}, {&__pyx_n_s_StreamingPartitioner_components, __pyx_k_StreamingPartitioner_components, sizeof(__pyx_k_StreamingPartitioner_components), 0, 0, 1, 1}, {&__pyx_n_s_StreamingPartitioner_tag_compone, __pyx_k_StreamingPartitioner_tag_compone, sizeof(__pyx_k_StreamingPartitioner_tag_compone), 0, 0, 1, 1}, + {&__pyx_kp_s_Users_camille_w_khmer_khmer__ox, __pyx_k_Users_camille_w_khmer_khmer__ox, sizeof(__pyx_k_Users_camille_w_khmer_khmer__ox), 0, 0, 1, 0}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, @@ -4491,11 +4493,10 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, - {&__pyx_kp_s_work_khmer_khmer__oxli_pyx, __pyx_k_work_khmer_khmer__oxli_pyx, sizeof(__pyx_k_work_khmer_khmer__oxli_pyx), 0, 0, 1, 0}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 68, __pyx_L1_error) + __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 85, __pyx_L1_error) __pyx_builtin_staticmethod = __Pyx_GetBuiltinName(__pyx_n_s_staticmethod); if (!__pyx_builtin_staticmethod) __PYX_ERR(1, 65, __pyx_L1_error) __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(1, 46, __pyx_L1_error) __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 90, __pyx_L1_error) @@ -4630,7 +4631,7 @@ static int __Pyx_InitCachedConstants(void) { __pyx_tuple__11 = PyTuple_Pack(3, __pyx_n_s_component, __pyx_n_s_graph, __pyx_n_s_graph_ptr); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(1, 66, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__11); __Pyx_GIVEREF(__pyx_tuple__11); - __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_work_khmer_khmer__oxli_pyx, __pyx_n_s_tag_counts, 66, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(1, 66, __pyx_L1_error) + __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_camille_w_khmer_khmer__ox, __pyx_n_s_tag_counts, 66, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(1, 66, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; diff --git a/khmer/_oxli.pxd b/khmer/_oxli.pxd index b60f72e81d..983c9033ff 100644 --- a/khmer/_oxli.pxd +++ b/khmer/_oxli.pxd @@ -14,6 +14,7 @@ cdef extern from "khmer.hh" namespace "khmer": ctypedef unsigned char WordLength ctypedef unsigned short int BoundedCounterType ctypedef queue[Kmer] KmerQueue + ctypedef bool (*KmerFilter) (Kmer kmer) cdef extern from "hashtable.hh" namespace "khmer": @@ -41,12 +42,20 @@ cdef extern from "kmer_hash.hh" namespace "khmer": bool is_forward() const - - cdef extern from "traversal.hh": - cdef cppclass CyLeftNodeGatherer "khmer::NodeGatherer<0>": + cdef cppclass CyTraverser "khmer::Traverser": CyLeftNodeGatherer(CyHashtable *) - void push_filter() + + void push_filter(KmerFilter) + KmerFilter pop_filter() + + uint32_t traverse(const Kmer&, KmerQueue&) const + uint32_t traverse_left(const Kmer&, KmerQueue&) const + uint32_t traverse_right(const Kmer&, KmerQueue&) const + + uint32_t degree(const Kmer&) const + uint32_t degree_left(const Kmer&) const + uint32_t degree_right(const Kmer&) const cdef extern from "partitioning.hh" namespace "khmer": From aae706c91791a8d1fcef7c2477c73a05e6550962 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 14 Nov 2016 02:14:52 -0800 Subject: [PATCH 031/185] Reorganize oxli files --- khmer/{ => _oxli}/_oxli.cpp | 0 khmer/{ => _oxli}/_oxli.pxd | 0 khmer/{ => _oxli}/_oxli.pyx | 0 setup.py | 42 ++++++++++++++++++------------------- 4 files changed, 20 insertions(+), 22 deletions(-) rename khmer/{ => _oxli}/_oxli.cpp (100%) rename khmer/{ => _oxli}/_oxli.pxd (100%) rename khmer/{ => _oxli}/_oxli.pyx (100%) diff --git a/khmer/_oxli.cpp b/khmer/_oxli/_oxli.cpp similarity index 100% rename from khmer/_oxli.cpp rename to khmer/_oxli/_oxli.cpp diff --git a/khmer/_oxli.pxd b/khmer/_oxli/_oxli.pxd similarity index 100% rename from khmer/_oxli.pxd rename to khmer/_oxli/_oxli.pxd diff --git a/khmer/_oxli.pyx b/khmer/_oxli/_oxli.pyx similarity index 100% rename from khmer/_oxli.pyx rename to khmer/_oxli/_oxli.pyx diff --git a/setup.py b/setup.py index dd8061b68e..f7b25764f1 100755 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ import ez_setup +import glob import os import sys from os import listdir as os_listdir @@ -165,7 +166,7 @@ def build_dir(): EXTRA_COMPILE_ARGS = ['-O3', '-std=c++11', '-pedantic'] EXTRA_LINK_ARGS = [] -if sys.platform == 'darwin': +if sys.platform == 'darwin' and 'clang' in os.getenv('CC', 'cc'): # force 64bit only builds EXTRA_COMPILE_ARGS.extend(['-arch', 'x86_64', '-mmacosx-version-min=10.7', '-stdlib=libc++']) @@ -186,27 +187,24 @@ def build_dir(): EXTENSION_MODS = [Extension("khmer._khmer", ** CP_EXTENSION_MOD_DICT)] -CY_EXTENSION_MOD_DICT = \ - { - "sources": ["khmer/_oxli.pyx"], - "extra_compile_args": EXTRA_COMPILE_ARGS, - "extra_link_args": EXTRA_LINK_ARGS, - "extra_objects": [path_join(build_dir(), splitext(p)[0]+'.o') for p in SOURCES], - "depends": [], - "language": "c++", - "define_macros": [("VERSION", versioneer.get_version()), ], - } - -''' -CY_EXT_MOD_DICT = dict(EXTENSION_MOD_DICT) -CY_EXT_MOD_DICT['sources'].insert(0, "khmer/_oxli.pyx") -CY_EXT_MOD_DICT['sources'].insert(1, "khmer/_khmer.cc") -CY_EXT_MOD_DICT['sources'].append(path_join("lib", "partitioning.cc")) -CY_EXT_MOD_DICT['depends'].append(path_join("lib", "partitioning.hh")) -''' -CY_EXTENSION_MOD = Extension("khmer._oxli", ** CY_EXTENSION_MOD_DICT) -#EXTENSION_MODS.extend(cythonize([CY_EXTENSION_MOD])) -EXTENSION_MODS.append(CY_EXTENSION_MOD) +for cython_ext in glob.glob(os.path.join("khmer", "_oxli", "*.pyx")): + print('ext for', cython_ext) + CY_EXTENSION_MOD_DICT = \ + { + "sources": [cython_ext], + "extra_compile_args": EXTRA_COMPILE_ARGS, + "extra_link_args": EXTRA_LINK_ARGS, + "extra_objects": [path_join(build_dir(), splitext(p)[0]+'.o') for p in SOURCES], + "depends": [], + "include_dirs": ["khmer", "lib"], + "language": "c++", + "define_macros": [("VERSION", versioneer.get_version()), ], + } + + ext_name = "khmer._oxli.{0}".format(splitext(os.path.basename(cython_ext))[0]) + print('Extension name:', ext_name) + CY_EXTENSION_MOD = Extension(ext_name, ** CY_EXTENSION_MOD_DICT) + EXTENSION_MODS.append(CY_EXTENSION_MOD) SCRIPTS = [] From a99a482fd420a2eac55764a4c3acb959a9aea5b3 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 14 Nov 2016 02:16:21 -0800 Subject: [PATCH 032/185] Reorganization of Cython wrappers --- MANIFEST.in | 1 + Makefile | 2 +- khmer/_oxli/__init__.py | 0 khmer/_oxli/_oxli.cpp | 7994 +--------------------------------- khmer/_oxli/_oxli.pxd | 80 +- khmer/_oxli/_oxli.pyx | 188 +- khmer/_oxli/hashing.pyx | 42 + khmer/_oxli/partitioning.pxd | 12 + khmer/_oxli/partitioning.pyx | 181 + khmer/_oxli/traversal.pxd | 19 + khmer/_oxli/traversal.pyx | 0 11 files changed, 303 insertions(+), 8216 deletions(-) create mode 100644 khmer/_oxli/__init__.py create mode 100644 khmer/_oxli/hashing.pyx create mode 100644 khmer/_oxli/partitioning.pxd create mode 100644 khmer/_oxli/partitioning.pyx create mode 100644 khmer/_oxli/traversal.pxd create mode 100644 khmer/_oxli/traversal.pyx diff --git a/MANIFEST.in b/MANIFEST.in index 9afd173435..7162f8de8e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include ChangeLog ez_setup.py IDEAS Makefile README.rst setup.cfg include versioneer.py MANIFEST.in CITATION CONTRIBUTING.md Doxyfile.in include LICENSE TODO .ycm_extra_conf.py recursive-include lib *.hh *.cc [Mm]akefile* get_version.py +recursive-include khmer *.hh *.cc *.cpp recursive-include third-party *.cc *.1 *.xsl README* sample* words* *.sh *.c recursive-include third-party manual* [Mm]akefile* *.pl *.dsp CHANGES *.txt *.h recursive-include third-party ChangeLog FAQ INDEX configure *.xsl diff --git a/Makefile b/Makefile index 4fad666b42..0fe56127cb 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,7 @@ clean: FORCE rm -f $(EXTENSION_MODULE) rm -f $(CYTHON_MODULE) rm -f khmer/*.pyc lib/*.pyc scripts/*.pyc tests/*.pyc oxli/*.pyc \ - sandbox/*.pyc khmer/__pycache__/* sandbox/__pycache__/* khmer/_oxli.cpp + sandbox/*.pyc khmer/__pycache__/* sandbox/__pycache__/* khmer/_oxli/*.cpp ./setup.py clean --all || true rm -f coverage-debug rm -Rf .coverage diff --git a/khmer/_oxli/__init__.py b/khmer/_oxli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/khmer/_oxli/_oxli.cpp b/khmer/_oxli/_oxli.cpp index 24f68dae40..06f2230dd7 100644 --- a/khmer/_oxli/_oxli.cpp +++ b/khmer/_oxli/_oxli.cpp @@ -1,7993 +1 @@ -/* Generated by Cython 0.25.1 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) - #error Cython requires Python 2.6+ or Python 3.2+. -#else -#define CYTHON_ABI "0_25_1" -#include -#ifndef offsetof - #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#endif -#ifndef DL_IMPORT - #define DL_IMPORT(t) t -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#ifndef HAVE_LONG_LONG - #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) - #define HAVE_LONG_LONG - #endif -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef Py_HUGE_VAL - #define Py_HUGE_VAL HUGE_VAL -#endif -#ifdef PYPY_VERSION - #define CYTHON_COMPILING_IN_PYPY 1 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #undef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 0 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #undef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #undef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 1 - #undef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 0 - #undef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 0 - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 -#elif defined(PYSTON_VERSION) - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 1 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 -#else - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 1 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #if PY_MAJOR_VERSION < 3 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #elif !defined(CYTHON_USE_ASYNC_SLOTS) - #define CYTHON_USE_ASYNC_SLOTS 1 - #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) - #define CYTHON_USE_PYLONG_INTERNALS 1 - #endif - #ifndef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 1 - #endif - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #if PY_VERSION_HEX < 0x030300F0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #elif !defined(CYTHON_USE_UNICODE_WRITER) - #define CYTHON_USE_UNICODE_WRITER 1 - #endif - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #ifndef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 1 - #endif - #ifndef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 1 - #endif -#endif -#if !defined(CYTHON_FAST_PYCCALL) -#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) -#endif -#if CYTHON_USE_PYLONG_INTERNALS - #include "longintrepr.h" - #undef SHIFT - #undef BASE - #undef MASK -#endif -#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) - #define Py_OptimizeFlag 0 -#endif -#define __PYX_BUILD_PY_SSIZE_T "n" -#define CYTHON_FORMAT_SSIZE_T "z" -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyClass_Type -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyType_Type -#endif -#ifndef Py_TPFLAGS_CHECKTYPES - #define Py_TPFLAGS_CHECKTYPES 0 -#endif -#ifndef Py_TPFLAGS_HAVE_INDEX - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#ifndef Py_TPFLAGS_HAVE_NEWBUFFER - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#ifndef Py_TPFLAGS_HAVE_FINALIZE - #define Py_TPFLAGS_HAVE_FINALIZE 0 -#endif -#ifndef METH_FASTCALL - #define METH_FASTCALL 0x80 - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); -#else - #define __Pyx_PyCFunctionFast _PyCFunctionFast -#endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) -#else -#define __Pyx_PyFastCFunction_Check(func) 0 -#endif -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) - #define CYTHON_PEP393_ENABLED 1 - #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ - 0 : _PyUnicode_Ready((PyObject *)(op))) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) - #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) - #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) -#else - #define CYTHON_PEP393_ENABLED 0 - #define PyUnicode_1BYTE_KIND 1 - #define PyUnicode_2BYTE_KIND 2 - #define PyUnicode_4BYTE_KIND 4 - #define __Pyx_PyUnicode_READY(op) (0) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) - #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) -#endif -#if CYTHON_COMPILING_IN_PYPY - #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) -#else - #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ - PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) - #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) - #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) - #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) - #define PyObject_Malloc(s) PyMem_Malloc(s) - #define PyObject_Free(p) PyMem_Free(p) - #define PyObject_Realloc(p) PyMem_Realloc(p) -#endif -#if CYTHON_COMPILING_IN_PYSTON - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) -#else - #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) -#endif -#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) -#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) -#else - #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) -#endif -#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) - #define PyObject_ASCII(o) PyObject_Repr(o) -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyStringObject PyUnicodeObject - #define PyString_Type PyUnicode_Type - #define PyString_Check PyUnicode_Check - #define PyString_CheckExact PyUnicode_CheckExact -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) - #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) -#else - #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) - #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) -#endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) -#endif -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) -#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) -#if PY_MAJOR_VERSION >= 3 - #define PyIntObject PyLongObject - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define PyNumber_Int PyNumber_Long -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBoolObject PyLongObject -#endif -#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY - #ifndef PyUnicode_InternFromString - #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) - #endif -#endif -#if PY_VERSION_HEX < 0x030200A4 - typedef long Py_hash_t; - #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong -#else - #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) -#endif -#if CYTHON_USE_ASYNC_SLOTS - #if PY_VERSION_HEX >= 0x030500B1 - #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods - #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) - #else - typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; - } __Pyx_PyAsyncMethodsStruct; - #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) - #endif -#else - #define __Pyx_PyType_AsAsync(obj) NULL -#endif -#ifndef CYTHON_RESTRICT - #if defined(__GNUC__) - #define CYTHON_RESTRICT __restrict__ - #elif defined(_MSC_VER) && _MSC_VER >= 1400 - #define CYTHON_RESTRICT __restrict - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_RESTRICT restrict - #else - #define CYTHON_RESTRICT - #endif -#endif -#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) - -#ifndef __cplusplus - #error "Cython files generated with the C++ option must be compiled with a C++ compiler." -#endif -#ifndef CYTHON_INLINE - #define CYTHON_INLINE inline -#endif -template -void __Pyx_call_destructor(T& x) { - x.~T(); -} -template -class __Pyx_FakeReference { - public: - __Pyx_FakeReference() : ptr(NULL) { } - __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } - T *operator->() { return ptr; } - operator T&() { return *ptr; } - template bool operator ==(U other) { return *ptr == other; }; - template bool operator !=(U other) { return *ptr != other; }; - private: - T *ptr; -}; - -#if defined(WIN32) || defined(MS_WINDOWS) - #define _USE_MATH_DEFINES -#endif -#include -#ifdef NAN -#define __PYX_NAN() ((float) NAN) -#else -static CYTHON_INLINE float __PYX_NAN() { - float value; - memset(&value, 0xFF, sizeof(value)); - return value; -} -#endif -#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) -#define __Pyx_truncl trunc -#else -#define __Pyx_truncl truncl -#endif - - -#define __PYX_ERR(f_index, lineno, Ln_error) \ -{ \ - __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ -} - -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) -#endif - -#ifndef __PYX_EXTERN_C - #ifdef __cplusplus - #define __PYX_EXTERN_C extern "C" - #else - #define __PYX_EXTERN_C extern - #endif -#endif - -#define __PYX_HAVE__khmer___oxli -#define __PYX_HAVE_API__khmer___oxli -#include -#include -#include "ios" -#include "new" -#include "stdexcept" -#include "typeinfo" -#include -#include -#include -#include -#include -#include -#include -#include "khmer.hh" -#include "hashtable.hh" -#include "_khmer.hh" -#include "kmer_hash.hh" -#include "traversal.hh" -#include "partitioning.hh" -#include -#include -#ifdef _OPENMP -#include -#endif /* _OPENMP */ - -#ifdef PYREX_WITHOUT_ASSERTIONS -#define CYTHON_WITHOUT_ASSERTIONS -#endif - -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif -typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; - const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; - -#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 -#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 -#define __PYX_DEFAULT_STRING_ENCODING "" -#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString -#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#define __Pyx_uchar_cast(c) ((unsigned char)c) -#define __Pyx_long_cast(x) ((long)x) -#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ - (sizeof(type) < sizeof(Py_ssize_t)) ||\ - (sizeof(type) > sizeof(Py_ssize_t) &&\ - likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX) &&\ - (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ - v == (type)PY_SSIZE_T_MIN))) ||\ - (sizeof(type) == sizeof(Py_ssize_t) &&\ - (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX))) ) -#if defined (__cplusplus) && __cplusplus >= 201103L - #include - #define __Pyx_sst_abs(value) std::abs(value) -#elif SIZEOF_INT >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) abs(value) -#elif SIZEOF_LONG >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) labs(value) -#elif defined (_MSC_VER) && defined (_M_X64) - #define __Pyx_sst_abs(value) _abs64(value) -#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define __Pyx_sst_abs(value) llabs(value) -#elif defined (__GNUC__) - #define __Pyx_sst_abs(value) __builtin_llabs(value) -#else - #define __Pyx_sst_abs(value) ((value<0) ? -value : value) -#endif -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); -#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) -#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); -#if PY_MAJOR_VERSION < 3 - #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#else - #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize -#endif -#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) -#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) -#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) -#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) -#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -#if PY_MAJOR_VERSION < 3 -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) -{ - const Py_UNICODE *u_end = u; - while (*u_end++) ; - return (size_t)(u_end - u - 1); -} -#else -#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen -#endif -#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) -#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode -#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode -#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) -#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) -#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -#if CYTHON_ASSUME_SAFE_MACROS -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) -#else -#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) -#endif -#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) -#if PY_MAJOR_VERSION >= 3 -#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) -#else -#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) -#endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII -static int __Pyx_sys_getdefaultencoding_not_ascii; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - PyObject* ascii_chars_u = NULL; - PyObject* ascii_chars_b = NULL; - const char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - if (strcmp(default_encoding_c, "ascii") == 0) { - __Pyx_sys_getdefaultencoding_not_ascii = 0; - } else { - char ascii_chars[128]; - int c; - for (c = 0; c < 128; c++) { - ascii_chars[c] = c; - } - __Pyx_sys_getdefaultencoding_not_ascii = 1; - ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); - if (!ascii_chars_u) goto bad; - ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); - if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { - PyErr_Format( - PyExc_ValueError, - "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", - default_encoding_c); - goto bad; - } - Py_DECREF(ascii_chars_u); - Py_DECREF(ascii_chars_b); - } - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - Py_XDECREF(ascii_chars_u); - Py_XDECREF(ascii_chars_b); - return -1; -} -#endif -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) -#else -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT -static char* __PYX_DEFAULT_STRING_ENCODING; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); - if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; - strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - return -1; -} -#endif -#endif - - -/* Test for GCC > 2.95 */ -#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) - #define likely(x) __builtin_expect(!!(x), 1) - #define unlikely(x) __builtin_expect(!!(x), 0) -#else /* !__GNUC__ or GCC < 2.95 */ - #define likely(x) (x) - #define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_d; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static PyObject *__pyx_empty_bytes; -static PyObject *__pyx_empty_unicode; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; - - -static const char *__pyx_f[] = { - "khmer/_oxli.pxd", - "khmer/_oxli.pyx", - "stringsource", -}; - -/*--- Type declarations ---*/ -struct __pyx_obj_5khmer_5_oxli_Component; -struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner; -struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__; -struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components; -struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components; - -/* "khmer/_oxli.pyx":11 - * from khmer._khmer import Nodegraph - * - * cdef class Component: # <<<<<<<<<<<<<< - * - * cdef ComponentPtr _this - */ -struct __pyx_obj_5khmer_5_oxli_Component { - PyObject_HEAD - struct __pyx_vtabstruct_5khmer_5_oxli_Component *__pyx_vtab; - khmer::ComponentPtr _this; -}; - - -/* "khmer/_oxli.pyx":80 - * - * - * cdef class StreamingPartitioner: # <<<<<<<<<<<<<< - * - * cdef unique_ptr[CyStreamingPartitioner] _this - */ -struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner { - PyObject_HEAD - std::unique_ptr _this; - std::weak_ptr _components; - std::weak_ptr _tag_component_map; - khmer::Hashtable *_graph_ptr; - uint64_t n_consumed; -}; - - -/* "khmer/_oxli.pyx":34 - * return deref(self._this).get_n_tags() - * - * def __iter__(self): # <<<<<<<<<<<<<< - * cdef HashIntoType tag - * for tag in deref(self._this).tags: - */ -struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ { - PyObject_HEAD - struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self; - khmer::HashIntoType __pyx_v_tag; - std::set ::iterator __pyx_t_0; - std::set *__pyx_t_1; -}; - - -/* "khmer/_oxli.pyx":125 - * return Component.create(compptr) - * - * def components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[ComponentPtrSet] locked - * cdef ComponentPtr cmpptr - */ -struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components { - PyObject_HEAD - khmer::ComponentPtr __pyx_v_cmpptr; - std::shared_ptr __pyx_v_locked; - std::shared_ptr __pyx_v_lockedptr; - struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self; - std::set ::iterator __pyx_t_0; - khmer::ComponentPtrSet *__pyx_t_1; -}; - - -/* "khmer/_oxli.pyx":135 - * raise MemoryError("Can't locked underlying Component set") - * - * def tag_components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[GuardedKmerCompMap] locked - * cdef pair[HashIntoType,ComponentPtr] cpair - */ -struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components { - PyObject_HEAD - std::pair __pyx_v_cpair; - std::shared_ptr __pyx_v_locked; - std::shared_ptr __pyx_v_lockedptr; - struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self; - std::map ::iterator __pyx_t_0; - std::map *__pyx_t_1; -}; - - - -/* "khmer/_oxli.pyx":11 - * from khmer._khmer import Nodegraph - * - * cdef class Component: # <<<<<<<<<<<<<< - * - * cdef ComponentPtr _this - */ - -struct __pyx_vtabstruct_5khmer_5_oxli_Component { - struct __pyx_obj_5khmer_5_oxli_Component *(*create)(khmer::ComponentPtr); - std::vector (*_tag_counts)(khmer::ComponentPtr, khmer::Hashtable *); - float (*_mean_tag_count)(khmer::ComponentPtr, khmer::Hashtable *); -}; -static struct __pyx_vtabstruct_5khmer_5_oxli_Component *__pyx_vtabptr_5khmer_5_oxli_Component; - -/* --- Runtime support code (head) --- */ -/* Refnanny.proto */ -#ifndef CYTHON_REFNANNY - #define CYTHON_REFNANNY 0 -#endif -#if CYTHON_REFNANNY - typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); - void (*FinishContext)(void**); - } __Pyx_RefNannyAPIStruct; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); - #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; -#ifdef WITH_THREAD - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - if (acquire_gil) {\ - PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - PyGILState_Release(__pyx_gilstate_save);\ - } else {\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - } -#else - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) -#endif - #define __Pyx_RefNannyFinishContext()\ - __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) -#else - #define __Pyx_RefNannyDeclarations - #define __Pyx_RefNannySetupContext(name, acquire_gil) - #define __Pyx_RefNannyFinishContext() - #define __Pyx_INCREF(r) Py_INCREF(r) - #define __Pyx_DECREF(r) Py_DECREF(r) - #define __Pyx_GOTREF(r) - #define __Pyx_GIVEREF(r) - #define __Pyx_XINCREF(r) Py_XINCREF(r) - #define __Pyx_XDECREF(r) Py_XDECREF(r) - #define __Pyx_XGOTREF(r) - #define __Pyx_XGIVEREF(r) -#endif -#define __Pyx_XDECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_XDECREF(tmp);\ - } while (0) -#define __Pyx_DECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_DECREF(tmp);\ - } while (0) -#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) -#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) - -/* PyObjectGetAttrStr.proto */ -#if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { - PyTypeObject* tp = Py_TYPE(obj); - if (likely(tp->tp_getattro)) - return tp->tp_getattro(obj, attr_name); -#if PY_MAJOR_VERSION < 3 - if (likely(tp->tp_getattr)) - return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); -#endif - return PyObject_GetAttr(obj, attr_name); -} -#else -#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) -#endif - -/* GetBuiltinName.proto */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name); - -/* RaiseDoubleKeywords.proto */ -static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); - -/* ParseKeywords.proto */ -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ - const char* function_name); - -/* RaiseArgTupleInvalid.proto */ -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); - -/* ArgTypeTest.proto */ -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact); - -/* PyIntBinop.proto */ -#if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); -#else -#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ - PyObject_RichCompare(op1, op2, Py_EQ) - #endif - -/* PyObjectCall.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); -#else -#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) -#endif - -/* PyThreadStateGet.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; -#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); -#else -#define __Pyx_PyThreadState_declare -#define __Pyx_PyThreadState_assign -#endif - -/* PyErrFetchRestore.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) -#endif - -/* RaiseException.proto */ -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); - -/* RaiseTooManyValuesToUnpack.proto */ -static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); - -/* RaiseNeedMoreValuesToUnpack.proto */ -static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); - -/* IterFinish.proto */ -static CYTHON_INLINE int __Pyx_IterFinish(void); - -/* UnpackItemEndCheck.proto */ -static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); - -/* WriteUnraisableException.proto */ -static void __Pyx_WriteUnraisable(const char *name, int clineno, - int lineno, const char *filename, - int full_traceback, int nogil); - -/* GetModuleGlobalName.proto */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); - -/* ListCompAppend.proto */ -#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS -static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { - PyListObject* L = (PyListObject*) list; - Py_ssize_t len = Py_SIZE(list); - if (likely(L->allocated > len)) { - Py_INCREF(x); - PyList_SET_ITEM(list, len, x); - Py_SIZE(list) = len+1; - return 0; - } - return PyList_Append(list, x); -} -#else -#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) -#endif - -/* IncludeStringH.proto */ -#include - -/* SetVTable.proto */ -static int __Pyx_SetVtable(PyObject *dict, void *vtable); - -/* Import.proto */ -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); - -/* ImportFrom.proto */ -static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); - -/* GetNameInClass.proto */ -static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name); - -/* CodeObjectCache.proto */ -typedef struct { - PyCodeObject* code_object; - int code_line; -} __Pyx_CodeObjectCacheEntry; -struct __Pyx_CodeObjectCache { - int count; - int max_count; - __Pyx_CodeObjectCacheEntry* entries; -}; -static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); -static PyCodeObject *__pyx_find_code_object(int code_line); -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); - -/* AddTraceback.proto */ -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename); - -/* None.proto */ -#include - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value); - -/* CppExceptionConversion.proto */ -#ifndef __Pyx_CppExn2PyErr -#include -#include -#include -#include -static void __Pyx_CppExn2PyErr() { - try { - if (PyErr_Occurred()) - ; // let the latest Python exn pass through and ignore the current one - else - throw; - } catch (const std::bad_alloc& exn) { - PyErr_SetString(PyExc_MemoryError, exn.what()); - } catch (const std::bad_cast& exn) { - PyErr_SetString(PyExc_TypeError, exn.what()); - } catch (const std::bad_typeid& exn) { - PyErr_SetString(PyExc_TypeError, exn.what()); - } catch (const std::domain_error& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::invalid_argument& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::ios_base::failure& exn) { - PyErr_SetString(PyExc_IOError, exn.what()); - } catch (const std::out_of_range& exn) { - PyErr_SetString(PyExc_IndexError, exn.what()); - } catch (const std::overflow_error& exn) { - PyErr_SetString(PyExc_OverflowError, exn.what()); - } catch (const std::range_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::underflow_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::exception& exn) { - PyErr_SetString(PyExc_RuntimeError, exn.what()); - } - catch (...) - { - PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); - } -} -#endif - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType(khmer::BoundedCounterType value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); - -/* FetchCommonType.proto */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); - -/* SwapException.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); -#endif - -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/* PyFunctionFastCall.proto */ -#if CYTHON_FAST_PYCALL -#define __Pyx_PyFunction_FastCall(func, args, nargs)\ - __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) -#endif -#endif - -/* PyObjectCallMethO.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - -/* PyObjectCallMethod1.proto */ -static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); - -/* CoroutineBase.proto */ -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); -typedef struct { - PyObject_HEAD - __pyx_coroutine_body_t body; - PyObject *closure; - PyObject *exc_type; - PyObject *exc_value; - PyObject *exc_traceback; - PyObject *gi_weakreflist; - PyObject *classobj; - PyObject *yieldfrom; - PyObject *gi_name; - PyObject *gi_qualname; - PyObject *gi_modulename; - int resume_label; - char is_running; -} __pyx_CoroutineObject; -static __pyx_CoroutineObject *__Pyx__Coroutine_New( - PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *closure, - PyObject *name, PyObject *qualname, PyObject *module_name); -static int __Pyx_Coroutine_clear(PyObject *self); -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); -#else -#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) -#endif - -/* PatchModuleWithCoroutine.proto */ -static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code); - -/* PatchGeneratorABC.proto */ -static int __Pyx_patch_abc(void); - -/* Generator.proto */ -#define __Pyx_Generator_USED -static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) -#define __Pyx_Generator_New(body, closure, name, qualname, module_name)\ - __Pyx__Coroutine_New(__pyx_GeneratorType, body, closure, name, qualname, module_name) -static PyObject *__Pyx_Generator_Next(PyObject *self); -static int __pyx_Generator_init(void); - -/* CheckBinaryVersion.proto */ -static int __Pyx_check_binary_version(void); - -/* InitStrings.proto */ -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); - -static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Component_create(khmer::ComponentPtr __pyx_v_ptr); /* proto*/ -static std::vector __pyx_f_5khmer_5_oxli_9Component__tag_counts(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph); /* proto*/ -static float __pyx_f_5khmer_5_oxli_9Component__mean_tag_count(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph); /* proto*/ - -/* Module declarations from 'libcpp' */ - -/* Module declarations from 'libc.string' */ - -/* Module declarations from 'libcpp.string' */ - -/* Module declarations from 'libcpp.vector' */ - -/* Module declarations from 'libcpp.utility' */ - -/* Module declarations from 'libcpp.map' */ - -/* Module declarations from 'libcpp.set' */ - -/* Module declarations from 'libcpp.queue' */ - -/* Module declarations from 'libcpp.memory' */ - -/* Module declarations from 'libc.stdint' */ - -/* Module declarations from 'cython' */ - -/* Module declarations from 'libc.limits' */ - -/* Module declarations from 'libc.stdio' */ - -/* Module declarations from 'khmer._oxli' */ -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_Component = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_StreamingPartitioner = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components = 0; -static PyObject *__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(const std::vector &); /*proto*/ -static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ -#define __Pyx_MODULE_NAME "khmer._oxli" -int __pyx_module_is_main_khmer___oxli = 0; - -/* Implementation of 'khmer._oxli' */ -static PyObject *__pyx_builtin_MemoryError; -static PyObject *__pyx_builtin_staticmethod; -static PyObject *__pyx_builtin_NotImplementedError; -static PyObject *__pyx_builtin_ValueError; -static PyObject *__pyx_builtin_IOError; -static PyObject *__pyx_builtin_range; -static const char __pyx_k_args[] = "args"; -static const char __pyx_k_iter[] = "__iter__"; -static const char __pyx_k_main[] = "__main__"; -static const char __pyx_k_send[] = "send"; -static const char __pyx_k_test[] = "__test__"; -static const char __pyx_k_close[] = "close"; -static const char __pyx_k_graph[] = "graph"; -static const char __pyx_k_other[] = "other"; -static const char __pyx_k_range[] = "range"; -static const char __pyx_k_throw[] = "throw"; -static const char __pyx_k_utf_8[] = "utf-8"; -static const char __pyx_k_encode[] = "encode"; -static const char __pyx_k_import[] = "__import__"; -static const char __pyx_k_IOError[] = "IOError"; -static const char __pyx_k_Nodegraph[] = "Nodegraph"; -static const char __pyx_k_component[] = "component"; -static const char __pyx_k_graph_ptr[] = "graph_ptr"; -static const char __pyx_k_Countgraph[] = "Countgraph"; -static const char __pyx_k_ValueError[] = "ValueError"; -static const char __pyx_k_components[] = "components"; -static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; -static const char __pyx_k_tag_counts[] = "tag_counts"; -static const char __pyx_k_MemoryError[] = "MemoryError"; -static const char __pyx_k_khmer__oxli[] = "khmer._oxli"; -static const char __pyx_k_component_id[] = "component_id"; -static const char __pyx_k_khmer__khmer[] = "khmer._khmer"; -static const char __pyx_k_staticmethod[] = "staticmethod"; -static const char __pyx_k_tag_components[] = "tag_components"; -static const char __pyx_k_Can_t_open_file[] = "Can't open file."; -static const char __pyx_k_Component___iter[] = "Component.__iter__"; -static const char __pyx_k_NotImplementedError[] = "NotImplementedError"; -static const char __pyx_k_Operator_not_available[] = "Operator not available."; -static const char __pyx_k_StreamingPartitioner_components[] = "StreamingPartitioner.components"; -static const char __pyx_k_Users_camille_w_khmer_khmer__ox[] = "/Users/camille/w/khmer/khmer/_oxli.pyx"; -static const char __pyx_k_Can_t_locked_underlying_Componen[] = "Can't locked underlying Component set"; -static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; -static const char __pyx_k_StreamingPartitioner_tag_compone[] = "StreamingPartitioner.tag_components"; -static PyObject *__pyx_kp_s_Can_t_locked_underlying_Componen; -static PyObject *__pyx_kp_s_Can_t_open_file; -static PyObject *__pyx_n_s_Component___iter; -static PyObject *__pyx_n_s_Countgraph; -static PyObject *__pyx_n_s_IOError; -static PyObject *__pyx_n_s_MemoryError; -static PyObject *__pyx_kp_s_Must_take_an_object_with_Hashtab; -static PyObject *__pyx_n_s_Nodegraph; -static PyObject *__pyx_n_s_NotImplementedError; -static PyObject *__pyx_kp_s_Operator_not_available; -static PyObject *__pyx_n_s_StreamingPartitioner_components; -static PyObject *__pyx_n_s_StreamingPartitioner_tag_compone; -static PyObject *__pyx_kp_s_Users_camille_w_khmer_khmer__ox; -static PyObject *__pyx_n_s_ValueError; -static PyObject *__pyx_n_s_args; -static PyObject *__pyx_n_s_close; -static PyObject *__pyx_n_s_component; -static PyObject *__pyx_n_s_component_id; -static PyObject *__pyx_n_s_components; -static PyObject *__pyx_n_s_encode; -static PyObject *__pyx_n_s_graph; -static PyObject *__pyx_n_s_graph_ptr; -static PyObject *__pyx_n_s_import; -static PyObject *__pyx_n_s_iter; -static PyObject *__pyx_n_s_khmer__khmer; -static PyObject *__pyx_n_s_khmer__oxli; -static PyObject *__pyx_n_s_main; -static PyObject *__pyx_n_s_other; -static PyObject *__pyx_n_s_pyx_vtable; -static PyObject *__pyx_n_s_range; -static PyObject *__pyx_n_s_send; -static PyObject *__pyx_n_s_staticmethod; -static PyObject *__pyx_n_s_tag_components; -static PyObject *__pyx_n_s_tag_counts; -static PyObject *__pyx_n_s_test; -static PyObject *__pyx_n_s_throw; -static PyObject *__pyx_kp_s_utf_8; -static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ -static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ -static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_11tag_counts(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_component, PyObject *__pyx_v_graph); /* proto */ -static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4consume_fasta(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_13tag_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_16write_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6n_tags___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10n_consumed___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_int_2; -static PyObject *__pyx_tuple_; -static PyObject *__pyx_tuple__2; -static PyObject *__pyx_tuple__3; -static PyObject *__pyx_tuple__4; -static PyObject *__pyx_tuple__5; -static PyObject *__pyx_tuple__6; -static PyObject *__pyx_tuple__7; -static PyObject *__pyx_tuple__8; -static PyObject *__pyx_tuple__9; -static PyObject *__pyx_tuple__10; -static PyObject *__pyx_tuple__11; -static PyObject *__pyx_codeobj__12; - -/* "khmer/_oxli.pyx":15 - * cdef ComponentPtr _this - * - * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< - * if other is not None: - * self._this.reset(other._this.get()) - */ - -/* Python wrapper */ -static int __pyx_pw_5khmer_5_oxli_9Component_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5khmer_5_oxli_9Component_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_other,0}; - PyObject* values[1] = {0}; - values[0] = (PyObject *)((struct __pyx_obj_5khmer_5_oxli_Component *)Py_None); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_other); - if (value) { values[0] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 15, __pyx_L3_error) - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_other = ((struct __pyx_obj_5khmer_5_oxli_Component *)values[0]); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 15, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.Component.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5khmer_5_oxli_Component, 1, "other", 0))) __PYX_ERR(1, 15, __pyx_L1_error) - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component___cinit__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self), __pyx_v_other); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5khmer_5_oxli_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_other) { - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "khmer/_oxli.pyx":16 - * - * def __cinit__(self, Component other=None): - * if other is not None: # <<<<<<<<<<<<<< - * self._this.reset(other._this.get()) - * - */ - __pyx_t_1 = (((PyObject *)__pyx_v_other) != Py_None); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "khmer/_oxli.pyx":17 - * def __cinit__(self, Component other=None): - * if other is not None: - * self._this.reset(other._this.get()) # <<<<<<<<<<<<<< - * - * property component_id: - */ - __pyx_v_self->_this.reset(__pyx_v_other->_this.get()); - - /* "khmer/_oxli.pyx":16 - * - * def __cinit__(self, Component other=None): - * if other is not None: # <<<<<<<<<<<<<< - * self._this.reset(other._this.get()) - * - */ - } - - /* "khmer/_oxli.pyx":15 - * cdef ComponentPtr _this - * - * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< - * if other is not None: - * self._this.reset(other._this.get()) - */ - - /* function exit code */ - __pyx_r = 0; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":20 - * - * property component_id: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).component_id - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12component_id_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12component_id_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli.pyx":21 - * property component_id: - * def __get__(self): - * return deref(self._this).component_id # <<<<<<<<<<<<<< - * - * property _n_created: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 21, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":20 - * - * property component_id: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).component_id - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.Component.component_id.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":24 - * - * property _n_created: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_created() - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10_n_created_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10_n_created_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli.pyx":25 - * property _n_created: - * def __get__(self): - * return deref(self._this).get_n_created() # <<<<<<<<<<<<<< - * - * property _n_destroyed: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_created()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 25, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":24 - * - * property _n_created: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_created() - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.Component._n_created.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":28 - * - * property _n_destroyed: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_destroyed() - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12_n_destroyed_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12_n_destroyed_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_12_n_destroyed___get__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli.pyx":29 - * property _n_destroyed: - * def __get__(self): - * return deref(self._this).get_n_destroyed() # <<<<<<<<<<<<<< - * - * def __len__(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_destroyed()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 29, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":28 - * - * property _n_destroyed: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_destroyed() - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.Component._n_destroyed.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":31 - * return deref(self._this).get_n_destroyed() - * - * def __len__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_tags() - * - */ - -/* Python wrapper */ -static Py_ssize_t __pyx_pw_5khmer_5_oxli_9Component_3__len__(PyObject *__pyx_v_self); /*proto*/ -static Py_ssize_t __pyx_pw_5khmer_5_oxli_9Component_3__len__(PyObject *__pyx_v_self) { - Py_ssize_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_2__len__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static Py_ssize_t __pyx_pf_5khmer_5_oxli_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { - Py_ssize_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__len__", 0); - - /* "khmer/_oxli.pyx":32 - * - * def __len__(self): - * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< - * - * def __iter__(self): - */ - __pyx_r = (*__pyx_v_self->_this).get_n_tags(); - goto __pyx_L0; - - /* "khmer/_oxli.pyx":31 - * return deref(self._this).get_n_destroyed() - * - * def __len__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_tags() - * - */ - - /* function exit code */ - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ - -/* "khmer/_oxli.pyx":34 - * return deref(self._this).get_n_tags() - * - * def __iter__(self): # <<<<<<<<<<<<<< - * cdef HashIntoType tag - * for tag in deref(self._this).tags: - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_5__iter__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_5__iter__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_4__iter__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *__pyx_cur_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__iter__", 0); - __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(__pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(1, 34, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 34, __pyx_L1_error) - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("khmer._oxli.Component.__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_gb_5khmer_5_oxli_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)__pyx_generator->closure); - PyObject *__pyx_r = NULL; - std::set ::iterator __pyx_t_1; - std::set *__pyx_t_2; - khmer::HashIntoType __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L6_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 34, __pyx_L1_error) - - /* "khmer/_oxli.pyx":36 - * def __iter__(self): - * cdef HashIntoType tag - * for tag in deref(self._this).tags: # <<<<<<<<<<<<<< - * yield tag - * - */ - __pyx_t_2 = &(*__pyx_cur_scope->__pyx_v_self->_this).tags; - __pyx_t_1 = __pyx_t_2->begin(); - for (;;) { - if (!(__pyx_t_1 != __pyx_t_2->end())) break; - __pyx_t_3 = *__pyx_t_1; - ++__pyx_t_1; - __pyx_cur_scope->__pyx_v_tag = __pyx_t_3; - - /* "khmer/_oxli.pyx":37 - * cdef HashIntoType tag - * for tag in deref(self._this).tags: - * yield tag # <<<<<<<<<<<<<< - * - * def __hash__(self): - */ - __pyx_t_4 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_cur_scope->__pyx_v_tag); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 37, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; - __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_2; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L6_resume_from_yield:; - __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; - __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 37, __pyx_L1_error) - - /* "khmer/_oxli.pyx":36 - * def __iter__(self): - * cdef HashIntoType tag - * for tag in deref(self._this).tags: # <<<<<<<<<<<<<< - * yield tag - * - */ - } - if (1); else __pyx_cur_scope = __pyx_cur_scope; - - /* "khmer/_oxli.pyx":34 - * return deref(self._this).get_n_tags() - * - * def __iter__(self): # <<<<<<<<<<<<<< - * cdef HashIntoType tag - * for tag in deref(self._this).tags: - */ - - /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_XDECREF(__pyx_r); __pyx_r = 0; - __pyx_generator->resume_label = -1; - __Pyx_Coroutine_clear((PyObject*)__pyx_generator); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":39 - * yield tag - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return self._this.get() - * - */ - -/* Python wrapper */ -static Py_hash_t __pyx_pw_5khmer_5_oxli_9Component_8__hash__(PyObject *__pyx_v_self); /*proto*/ -static Py_hash_t __pyx_pw_5khmer_5_oxli_9Component_8__hash__(PyObject *__pyx_v_self) { - Py_hash_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_7__hash__(((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static Py_hash_t __pyx_pf_5khmer_5_oxli_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_self) { - Py_hash_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__", 0); - - /* "khmer/_oxli.pyx":40 - * - * def __hash__(self): - * return self._this.get() # <<<<<<<<<<<<<< - * - * def __richcmp__(x, y, op): - */ - __pyx_r = ((uintptr_t)__pyx_v_self->_this.get()); - goto __pyx_L0; - - /* "khmer/_oxli.pyx":39 - * yield tag - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return self._this.get() - * - */ - - /* function exit code */ - __pyx_L0:; - if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":42 - * return self._this.get() - * - * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< - * if op == 2: - * return x.component_id == y.component_id - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_arg_op); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_10__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_arg_op) { - PyObject *__pyx_v_op = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); - __pyx_v_op = __Pyx_PyInt_From_int(__pyx_arg_op); if (unlikely(!__pyx_v_op)) __PYX_ERR(1, 42, __pyx_L3_error) - __Pyx_GOTREF(__pyx_v_op); - goto __pyx_L4_argument_unpacking_done; - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.Component.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), ((PyObject *)__pyx_v_op)); - - /* function exit code */ - __Pyx_XDECREF(__pyx_v_op); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannySetupContext("__richcmp__", 0); - - /* "khmer/_oxli.pyx":43 - * - * def __richcmp__(x, y, op): - * if op == 2: # <<<<<<<<<<<<<< - * return x.component_id == y.component_id - * else: - */ - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_v_op, __pyx_int_2, 2, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 43, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(1, 43, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (__pyx_t_2) { - - /* "khmer/_oxli.pyx":44 - * def __richcmp__(x, y, op): - * if op == 2: - * return x.component_id == y.component_id # <<<<<<<<<<<<<< - * else: - * raise NotImplementedError('Operator not available.') - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_x, __pyx_n_s_component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_y, __pyx_n_s_component_id); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 44, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_RichCompare(__pyx_t_1, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 44, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":43 - * - * def __richcmp__(x, y, op): - * if op == 2: # <<<<<<<<<<<<<< - * return x.component_id == y.component_id - * else: - */ - } - - /* "khmer/_oxli.pyx":46 - * return x.component_id == y.component_id - * else: - * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< - * - * @staticmethod - */ - /*else*/ { - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_NotImplementedError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 46, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 46, __pyx_L1_error) - } - - /* "khmer/_oxli.pyx":42 - * return self._this.get() - * - * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< - * if op == 2: - * return x.component_id == y.component_id - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("khmer._oxli.Component.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":49 - * - * @staticmethod - * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< - * cdef Component comp = Component() - * comp._this = ptr - */ - -static struct __pyx_obj_5khmer_5_oxli_Component *__pyx_f_5khmer_5_oxli_9Component_create(khmer::ComponentPtr __pyx_v_ptr) { - struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_comp = 0; - struct __pyx_obj_5khmer_5_oxli_Component *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("create", 0); - - /* "khmer/_oxli.pyx":50 - * @staticmethod - * cdef Component create(ComponentPtr ptr): - * cdef Component comp = Component() # <<<<<<<<<<<<<< - * comp._this = ptr - * return comp - */ - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_Component), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 50, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_comp = ((struct __pyx_obj_5khmer_5_oxli_Component *)__pyx_t_1); - __pyx_t_1 = 0; - - /* "khmer/_oxli.pyx":51 - * cdef Component create(ComponentPtr ptr): - * cdef Component comp = Component() - * comp._this = ptr # <<<<<<<<<<<<<< - * return comp - * - */ - __pyx_v_comp->_this = __pyx_v_ptr; - - /* "khmer/_oxli.pyx":52 - * cdef Component comp = Component() - * comp._this = ptr - * return comp # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __Pyx_XDECREF(((PyObject *)__pyx_r)); - __Pyx_INCREF(((PyObject *)__pyx_v_comp)); - __pyx_r = __pyx_v_comp; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":49 - * - * @staticmethod - * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< - * cdef Component comp = Component() - * comp._this = ptr - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.Component.create", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF((PyObject *)__pyx_v_comp); - __Pyx_XGIVEREF((PyObject *)__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":55 - * - * @staticmethod - * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CyHashtable * graph): # <<<<<<<<<<<<<< - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef vector[BoundedCounterType] counts - */ - -static std::vector __pyx_f_5khmer_5_oxli_9Component__tag_counts(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph) { - uint64_t __pyx_v_n_tags; - std::vector __pyx_v_counts; - int __pyx_v_idx; - uint64_t __pyx_v_tag; - std::vector __pyx_r; - __Pyx_RefNannyDeclarations - std::vector __pyx_t_1; - std::set ::iterator __pyx_t_2; - std::set *__pyx_t_3; - khmer::HashIntoType __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - PyObject *(*__pyx_t_9)(PyObject *); - int __pyx_t_10; - uint64_t __pyx_t_11; - __Pyx_RefNannySetupContext("_tag_counts", 0); - - /* "khmer/_oxli.pyx":56 - * @staticmethod - * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CyHashtable * graph): - * cdef uint64_t n_tags = deref(comp).get_n_tags() # <<<<<<<<<<<<<< - * cdef vector[BoundedCounterType] counts - * counts = vector[BoundedCounterType](n_tags) - */ - __pyx_v_n_tags = (*__pyx_v_comp).get_n_tags(); - - /* "khmer/_oxli.pyx":58 - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef vector[BoundedCounterType] counts - * counts = vector[BoundedCounterType](n_tags) # <<<<<<<<<<<<<< - * cdef int idx - * cdef uint64_t tag - */ - try { - __pyx_t_1 = std::vector (__pyx_v_n_tags); - } catch(...) { - __Pyx_CppExn2PyErr(); - __PYX_ERR(1, 58, __pyx_L1_error) - } - __pyx_v_counts = __pyx_t_1; - - /* "khmer/_oxli.pyx":61 - * cdef int idx - * cdef uint64_t tag - * for idx, tag in deref(comp).tags: # <<<<<<<<<<<<<< - * counts[idx] = deref(graph).get_count(tag) - * return counts - */ - __pyx_t_3 = &(*__pyx_v_comp).tags; - __pyx_t_2 = __pyx_t_3->begin(); - for (;;) { - if (!(__pyx_t_2 != __pyx_t_3->end())) break; - __pyx_t_4 = *__pyx_t_2; - ++__pyx_t_2; - __pyx_t_5 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 61, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { - PyObject* sequence = __pyx_t_5; - #if !CYTHON_COMPILING_IN_PYPY - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - __PYX_ERR(1, 61, __pyx_L1_error) - } - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - if (likely(PyTuple_CheckExact(sequence))) { - __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); - __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); - } else { - __pyx_t_6 = PyList_GET_ITEM(sequence, 0); - __pyx_t_7 = PyList_GET_ITEM(sequence, 1); - } - __Pyx_INCREF(__pyx_t_6); - __Pyx_INCREF(__pyx_t_7); - #else - __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 61, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 61, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - #endif - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - } else { - Py_ssize_t index = -1; - __pyx_t_8 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 61, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_9 = Py_TYPE(__pyx_t_8)->tp_iternext; - index = 0; __pyx_t_6 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_6)) goto __pyx_L5_unpacking_failed; - __Pyx_GOTREF(__pyx_t_6); - index = 1; __pyx_t_7 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_7)) goto __pyx_L5_unpacking_failed; - __Pyx_GOTREF(__pyx_t_7); - if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < 0) __PYX_ERR(1, 61, __pyx_L1_error) - __pyx_t_9 = NULL; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - goto __pyx_L6_unpacking_done; - __pyx_L5_unpacking_failed:; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_9 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); - __PYX_ERR(1, 61, __pyx_L1_error) - __pyx_L6_unpacking_done:; - } - __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_6); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_11 = __Pyx_PyInt_As_uint64_t(__pyx_t_7); if (unlikely((__pyx_t_11 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_v_idx = __pyx_t_10; - __pyx_v_tag = __pyx_t_11; - - /* "khmer/_oxli.pyx":62 - * cdef uint64_t tag - * for idx, tag in deref(comp).tags: - * counts[idx] = deref(graph).get_count(tag) # <<<<<<<<<<<<<< - * return counts - * - */ - (__pyx_v_counts[__pyx_v_idx]) = (*__pyx_v_graph).get_count(__pyx_v_tag); - - /* "khmer/_oxli.pyx":61 - * cdef int idx - * cdef uint64_t tag - * for idx, tag in deref(comp).tags: # <<<<<<<<<<<<<< - * counts[idx] = deref(graph).get_count(tag) - * return counts - */ - } - - /* "khmer/_oxli.pyx":63 - * for idx, tag in deref(comp).tags: - * counts[idx] = deref(graph).get_count(tag) - * return counts # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __pyx_r = __pyx_v_counts; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":55 - * - * @staticmethod - * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CyHashtable * graph): # <<<<<<<<<<<<<< - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef vector[BoundedCounterType] counts - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_WriteUnraisable("khmer._oxli.Component._tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":66 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CyCpHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12tag_counts(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyMethodDef __pyx_mdef_5khmer_5_oxli_9Component_12tag_counts = {"tag_counts", (PyCFunction)__pyx_pw_5khmer_5_oxli_9Component_12tag_counts, METH_VARARGS|METH_KEYWORDS, 0}; -static PyObject *__pyx_pw_5khmer_5_oxli_9Component_12tag_counts(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_component = 0; - PyObject *__pyx_v_graph = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("tag_counts (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_component,&__pyx_n_s_graph,0}; - PyObject* values[2] = {0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_component)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("tag_counts", 1, 2, 2, 1); __PYX_ERR(1, 66, __pyx_L3_error) - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "tag_counts") < 0)) __PYX_ERR(1, 66, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - } - __pyx_v_component = ((struct __pyx_obj_5khmer_5_oxli_Component *)values[0]); - __pyx_v_graph = values[1]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("tag_counts", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 66, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.Component.tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_component), __pyx_ptype_5khmer_5_oxli_Component, 1, "component", 0))) __PYX_ERR(1, 66, __pyx_L1_error) - __pyx_r = __pyx_pf_5khmer_5_oxli_9Component_11tag_counts(__pyx_v_component, __pyx_v_graph); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_9Component_11tag_counts(struct __pyx_obj_5khmer_5_oxli_Component *__pyx_v_component, PyObject *__pyx_v_graph) { - khmer::khmer_KHashtable_Object *__pyx_v_graph_ptr; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("tag_counts", 0); - - /* "khmer/_oxli.pyx":67 - * @staticmethod - * def tag_counts(Component component, graph): - * cdef CyCpHashtable_Object* graph_ptr = graph # <<<<<<<<<<<<<< - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - * - */ - __pyx_v_graph_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - - /* "khmer/_oxli.pyx":68 - * def tag_counts(Component component, graph): - * cdef CyCpHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(__pyx_f_5khmer_5_oxli_9Component__tag_counts(__pyx_v_component->_this, (*__pyx_v_graph_ptr).hashtable)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 68, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":66 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CyCpHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.Component.tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":71 - * - * @staticmethod - * cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): # <<<<<<<<<<<<<< - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef float acc = 0 - */ - -static float __pyx_f_5khmer_5_oxli_9Component__mean_tag_count(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph) { - uint64_t __pyx_v_n_tags; - float __pyx_v_acc; - uint64_t __pyx_v_tag; - float __pyx_r; - __Pyx_RefNannyDeclarations - std::set ::iterator __pyx_t_1; - std::set *__pyx_t_2; - khmer::HashIntoType __pyx_t_3; - __Pyx_RefNannySetupContext("_mean_tag_count", 0); - - /* "khmer/_oxli.pyx":72 - * @staticmethod - * cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): - * cdef uint64_t n_tags = deref(comp).get_n_tags() # <<<<<<<<<<<<<< - * cdef float acc = 0 - * cdef uint64_t tag - */ - __pyx_v_n_tags = (*__pyx_v_comp).get_n_tags(); - - /* "khmer/_oxli.pyx":73 - * cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef float acc = 0 # <<<<<<<<<<<<<< - * cdef uint64_t tag - * for tag in deref(comp).tags: - */ - __pyx_v_acc = 0.0; - - /* "khmer/_oxli.pyx":75 - * cdef float acc = 0 - * cdef uint64_t tag - * for tag in deref(comp).tags: # <<<<<<<<<<<<<< - * acc += deref(graph).get_count(tag) - * return acc / n_tags - */ - __pyx_t_2 = &(*__pyx_v_comp).tags; - __pyx_t_1 = __pyx_t_2->begin(); - for (;;) { - if (!(__pyx_t_1 != __pyx_t_2->end())) break; - __pyx_t_3 = *__pyx_t_1; - ++__pyx_t_1; - __pyx_v_tag = __pyx_t_3; - - /* "khmer/_oxli.pyx":76 - * cdef uint64_t tag - * for tag in deref(comp).tags: - * acc += deref(graph).get_count(tag) # <<<<<<<<<<<<<< - * return acc / n_tags - * - */ - __pyx_v_acc = (__pyx_v_acc + ((float)(*__pyx_v_graph).get_count(__pyx_v_tag))); - - /* "khmer/_oxli.pyx":75 - * cdef float acc = 0 - * cdef uint64_t tag - * for tag in deref(comp).tags: # <<<<<<<<<<<<<< - * acc += deref(graph).get_count(tag) - * return acc / n_tags - */ - } - - /* "khmer/_oxli.pyx":77 - * for tag in deref(comp).tags: - * acc += deref(graph).get_count(tag) - * return acc / n_tags # <<<<<<<<<<<<<< - * - * - */ - if (unlikely(((float)__pyx_v_n_tags) == 0)) { - PyErr_SetString(PyExc_ZeroDivisionError, "float division"); - __PYX_ERR(1, 77, __pyx_L1_error) - } - __pyx_r = (__pyx_v_acc / ((float)__pyx_v_n_tags)); - goto __pyx_L0; - - /* "khmer/_oxli.pyx":71 - * - * @staticmethod - * cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): # <<<<<<<<<<<<<< - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef float acc = 0 - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_WriteUnraisable("khmer._oxli.Component._mean_tag_count", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); - __pyx_r = 0; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":88 - * cdef readonly uint64_t n_consumed - * - * def __cinit__(self, graph): # <<<<<<<<<<<<<< - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') - */ - -/* Python wrapper */ -static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_graph = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_graph,0}; - PyObject* values[1] = {0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 88, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - } - __pyx_v_graph = values[0]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 88, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), __pyx_v_graph); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5khmer_5_oxli_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph) { - khmer::khmer_KHashtable_Object *__pyx_v_ptr; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - int __pyx_t_3; - int __pyx_t_4; - khmer::Hashtable *__pyx_t_5; - khmer::StreamingPartitioner *__pyx_t_6; - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "khmer/_oxli.pyx":89 - * - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< - * raise ValueError('Must take an object with Hashtable *') - * - */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 89, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(1, 89, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_4 = (__pyx_t_3 != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_1 = __pyx_t_4; - goto __pyx_L4_bool_binop_done; - } - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 89, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(1, 89, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = (__pyx_t_4 != 0); - __pyx_t_1 = __pyx_t_3; - __pyx_L4_bool_binop_done:; - __pyx_t_3 = ((!__pyx_t_1) != 0); - if (__pyx_t_3) { - - /* "khmer/_oxli.pyx":90 - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< - * - * - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 90, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 90, __pyx_L1_error) - - /* "khmer/_oxli.pyx":89 - * - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< - * raise ValueError('Must take an object with Hashtable *') - * - */ - } - - /* "khmer/_oxli.pyx":93 - * - * - * cdef CyCpHashtable_Object* ptr = graph # <<<<<<<<<<<<<< - * self._graph_ptr = deref(ptr).hashtable - * - */ - __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - - /* "khmer/_oxli.pyx":94 - * - * cdef CyCpHashtable_Object* ptr = graph - * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< - * - * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) - */ - __pyx_t_5 = (*__pyx_v_ptr).hashtable; - __pyx_v_self->_graph_ptr = __pyx_t_5; - - /* "khmer/_oxli.pyx":96 - * self._graph_ptr = deref(ptr).hashtable - * - * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) # <<<<<<<<<<<<<< - * - * self._tag_component_map = deref(self._this).get_tag_component_map() - */ - try { - __pyx_t_6 = new khmer::StreamingPartitioner(__pyx_v_self->_graph_ptr); - } catch(...) { - try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 96, __pyx_L1_error) - } - __pyx_v_self->_this.reset(__pyx_t_6); - - /* "khmer/_oxli.pyx":98 - * self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) - * - * self._tag_component_map = deref(self._this).get_tag_component_map() # <<<<<<<<<<<<<< - * self._components = deref(self._this).get_component_set() - * self.n_consumed = 0 - */ - __pyx_v_self->_tag_component_map = (*__pyx_v_self->_this).get_tag_component_map(); - - /* "khmer/_oxli.pyx":99 - * - * self._tag_component_map = deref(self._this).get_tag_component_map() - * self._components = deref(self._this).get_component_set() # <<<<<<<<<<<<<< - * self.n_consumed = 0 - * - */ - __pyx_v_self->_components = (*__pyx_v_self->_this).get_component_set(); - - /* "khmer/_oxli.pyx":100 - * self._tag_component_map = deref(self._this).get_tag_component_map() - * self._components = deref(self._this).get_component_set() - * self.n_consumed = 0 # <<<<<<<<<<<<<< - * - * def consume_sequence(self, sequence): - */ - __pyx_v_self->n_consumed = 0; - - /* "khmer/_oxli.pyx":88 - * cdef readonly uint64_t n_consumed - * - * def __cinit__(self, graph): # <<<<<<<<<<<<<< - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":102 - * self.n_consumed = 0 - * - * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< - * deref(self._this).consume_sequence(sequence.encode('utf-8')) - * self.n_consumed += 1 - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence(PyObject *__pyx_v_self, PyObject *__pyx_v_sequence); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence(PyObject *__pyx_v_self, PyObject *__pyx_v_sequence) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("consume_sequence (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_sequence)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - std::string __pyx_t_3; - __Pyx_RefNannySetupContext("consume_sequence", 0); - - /* "khmer/_oxli.pyx":103 - * - * def consume_sequence(self, sequence): - * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< - * self.n_consumed += 1 - * - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 103, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 103, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - try { - (*__pyx_v_self->_this).consume_sequence(__pyx_t_3); - } catch(...) { - try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 103, __pyx_L1_error) - } - - /* "khmer/_oxli.pyx":104 - * def consume_sequence(self, sequence): - * deref(self._this).consume_sequence(sequence.encode('utf-8')) - * self.n_consumed += 1 # <<<<<<<<<<<<<< - * - * def consume_fasta(self, filename): - */ - __pyx_v_self->n_consumed = (__pyx_v_self->n_consumed + 1); - - /* "khmer/_oxli.pyx":102 - * self.n_consumed = 0 - * - * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< - * deref(self._this).consume_sequence(sequence.encode('utf-8')) - * self.n_consumed += 1 - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.consume_sequence", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":106 - * self.n_consumed += 1 - * - * def consume_fasta(self, filename): # <<<<<<<<<<<<<< - * return deref(self._this).consume_fasta(filename.encode('utf-8')) - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5consume_fasta(PyObject *__pyx_v_self, PyObject *__pyx_v_filename); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5consume_fasta(PyObject *__pyx_v_self, PyObject *__pyx_v_filename) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("consume_fasta (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4consume_fasta(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_filename)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_4consume_fasta(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - std::string __pyx_t_3; - uint64_t __pyx_t_4; - __Pyx_RefNannySetupContext("consume_fasta", 0); - - /* "khmer/_oxli.pyx":107 - * - * def consume_fasta(self, filename): - * return deref(self._this).consume_fasta(filename.encode('utf-8')) # <<<<<<<<<<<<<< - * - * def get_tag_component(self, kmer): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_filename, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 107, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 107, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - try { - __pyx_t_4 = (*__pyx_v_self->_this).consume_fasta(__pyx_t_3); - } catch(...) { - try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(1, 107, __pyx_L1_error) - } - __pyx_t_2 = __Pyx_PyInt_From_uint64_t(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":106 - * self.n_consumed += 1 - * - * def consume_fasta(self, filename): # <<<<<<<<<<<<<< - * return deref(self._this).consume_fasta(filename.encode('utf-8')) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.consume_fasta", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":109 - * return deref(self._this).consume_fasta(filename.encode('utf-8')) - * - * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("get_tag_component (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_tag_component(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6get_tag_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { - khmer::ComponentPtr __pyx_v_compptr; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - std::string __pyx_t_3; - int __pyx_t_4; - __Pyx_RefNannySetupContext("get_tag_component", 0); - - /* "khmer/_oxli.pyx":111 - * def get_tag_component(self, kmer): - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if compptr == NULL: - * return None - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 111, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 111, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_compptr = (*__pyx_v_self->_this).get_tag_component(__pyx_t_3); - - /* "khmer/_oxli.pyx":112 - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if compptr == NULL: # <<<<<<<<<<<<<< - * return None - * else: - */ - __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); - if (__pyx_t_4) { - - /* "khmer/_oxli.pyx":113 - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if compptr == NULL: - * return None # <<<<<<<<<<<<<< - * else: - * return Component.create(compptr) - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(Py_None); - __pyx_r = Py_None; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":112 - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if compptr == NULL: # <<<<<<<<<<<<<< - * return None - * else: - */ - } - - /* "khmer/_oxli.pyx":115 - * return None - * else: - * return Component.create(compptr) # <<<<<<<<<<<<<< - * - * def get_nearest_component(self, kmer): - */ - /*else*/ { - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 115, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - } - - /* "khmer/_oxli.pyx":109 - * return deref(self._this).consume_fasta(filename.encode('utf-8')) - * - * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.get_tag_component", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":117 - * return Component.create(compptr) - * - * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("get_nearest_component (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8get_nearest_component(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_8get_nearest_component(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { - khmer::ComponentPtr __pyx_v_compptr; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - std::string __pyx_t_3; - int __pyx_t_4; - __Pyx_RefNannySetupContext("get_nearest_component", 0); - - /* "khmer/_oxli.pyx":119 - * def get_nearest_component(self, kmer): - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if compptr == NULL: - * return None - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 119, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 119, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 119, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_compptr = (*__pyx_v_self->_this).get_nearest_component(__pyx_t_3); - - /* "khmer/_oxli.pyx":120 - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if compptr == NULL: # <<<<<<<<<<<<<< - * return None - * else: - */ - __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); - if (__pyx_t_4) { - - /* "khmer/_oxli.pyx":121 - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if compptr == NULL: - * return None # <<<<<<<<<<<<<< - * else: - * return Component.create(compptr) - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(Py_None); - __pyx_r = Py_None; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":120 - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if compptr == NULL: # <<<<<<<<<<<<<< - * return None - * else: - */ - } - - /* "khmer/_oxli.pyx":123 - * return None - * else: - * return Component.create(compptr) # <<<<<<<<<<<<<< - * - * def components(self): - */ - /*else*/ { - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 123, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - } - - /* "khmer/_oxli.pyx":117 - * return Component.create(compptr) - * - * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.get_nearest_component", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_12generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ - -/* "khmer/_oxli.pyx":125 - * return Component.create(compptr) - * - * def components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[ComponentPtrSet] locked - * cdef ComponentPtr cmpptr - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_11components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_11components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("components (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *__pyx_cur_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("components", 0); - __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components(__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(1, 125, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_12generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_components, __pyx_n_s_StreamingPartitioner_components, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 125, __pyx_L1_error) - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_12generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)__pyx_generator->closure); - PyObject *__pyx_r = NULL; - bool __pyx_t_1; - std::set ::iterator __pyx_t_2; - khmer::ComponentPtrSet *__pyx_t_3; - khmer::ComponentPtr __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L7_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 125, __pyx_L1_error) - - /* "khmer/_oxli.pyx":128 - * cdef shared_ptr[ComponentPtrSet] locked - * cdef ComponentPtr cmpptr - * lockedptr = self._components.lock() # <<<<<<<<<<<<<< - * if lockedptr: - * for cmpptr in deref(lockedptr): - */ - __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_components.lock(); - - /* "khmer/_oxli.pyx":129 - * cdef ComponentPtr cmpptr - * lockedptr = self._components.lock() - * if lockedptr: # <<<<<<<<<<<<<< - * for cmpptr in deref(lockedptr): - * yield Component.create(cmpptr) - */ - __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); - if (__pyx_t_1) { - - /* "khmer/_oxli.pyx":130 - * lockedptr = self._components.lock() - * if lockedptr: - * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< - * yield Component.create(cmpptr) - * else: - */ - __pyx_t_3 = &(*__pyx_cur_scope->__pyx_v_lockedptr); - __pyx_t_2 = __pyx_t_3->begin(); - for (;;) { - if (!(__pyx_t_2 != __pyx_t_3->end())) break; - __pyx_t_4 = *__pyx_t_2; - ++__pyx_t_2; - __pyx_cur_scope->__pyx_v_cmpptr = __pyx_t_4; - - /* "khmer/_oxli.pyx":131 - * if lockedptr: - * for cmpptr in deref(lockedptr): - * yield Component.create(cmpptr) # <<<<<<<<<<<<<< - * else: - * raise MemoryError("Can't locked underlying Component set") - */ - __pyx_t_5 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_cur_scope->__pyx_v_cmpptr)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 131, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_r = __pyx_t_5; - __pyx_t_5 = 0; - __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L7_resume_from_yield:; - __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; - __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 131, __pyx_L1_error) - - /* "khmer/_oxli.pyx":130 - * lockedptr = self._components.lock() - * if lockedptr: - * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< - * yield Component.create(cmpptr) - * else: - */ - } - - /* "khmer/_oxli.pyx":129 - * cdef ComponentPtr cmpptr - * lockedptr = self._components.lock() - * if lockedptr: # <<<<<<<<<<<<<< - * for cmpptr in deref(lockedptr): - * yield Component.create(cmpptr) - */ - goto __pyx_L4; - } - - /* "khmer/_oxli.pyx":133 - * yield Component.create(cmpptr) - * else: - * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< - * - * def tag_components(self): - */ - /*else*/ { - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 133, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_Raise(__pyx_t_5, 0, 0, 0); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 133, __pyx_L1_error) - } - __pyx_L4:; - if (1); else __pyx_cur_scope = __pyx_cur_scope; - - /* "khmer/_oxli.pyx":125 - * return Component.create(compptr) - * - * def components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[ComponentPtrSet] locked - * cdef ComponentPtr cmpptr - */ - - /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_5); - __Pyx_AddTraceback("components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_XDECREF(__pyx_r); __pyx_r = 0; - __pyx_generator->resume_label = -1; - __Pyx_Coroutine_clear((PyObject*)__pyx_generator); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_15generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ - -/* "khmer/_oxli.pyx":135 - * raise MemoryError("Can't locked underlying Component set") - * - * def tag_components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[GuardedKmerCompMap] locked - * cdef pair[HashIntoType,ComponentPtr] cpair - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_14tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_14tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("tag_components (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_13tag_components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_13tag_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *__pyx_cur_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("tag_components", 0); - __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components(__pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(1, 135, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_20StreamingPartitioner_15generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_tag_components, __pyx_n_s_StreamingPartitioner_tag_compone, __pyx_n_s_khmer__oxli); if (unlikely(!gen)) __PYX_ERR(1, 135, __pyx_L1_error) - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_gb_5khmer_5_oxli_20StreamingPartitioner_15generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)__pyx_generator->closure); - PyObject *__pyx_r = NULL; - bool __pyx_t_1; - std::map ::iterator __pyx_t_2; - std::map *__pyx_t_3; - std::pair __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L7_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 135, __pyx_L1_error) - - /* "khmer/_oxli.pyx":138 - * cdef shared_ptr[GuardedKmerCompMap] locked - * cdef pair[HashIntoType,ComponentPtr] cpair - * lockedptr = self._tag_component_map.lock() # <<<<<<<<<<<<<< - * if lockedptr: - * for cpair in deref(lockedptr).data: - */ - __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_tag_component_map.lock(); - - /* "khmer/_oxli.pyx":139 - * cdef pair[HashIntoType,ComponentPtr] cpair - * lockedptr = self._tag_component_map.lock() - * if lockedptr: # <<<<<<<<<<<<<< - * for cpair in deref(lockedptr).data: - * yield cpair.first, Component.create(cpair.second) - */ - __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); - if (__pyx_t_1) { - - /* "khmer/_oxli.pyx":140 - * lockedptr = self._tag_component_map.lock() - * if lockedptr: - * for cpair in deref(lockedptr).data: # <<<<<<<<<<<<<< - * yield cpair.first, Component.create(cpair.second) - * else: - */ - __pyx_t_3 = &(*__pyx_cur_scope->__pyx_v_lockedptr).data; - __pyx_t_2 = __pyx_t_3->begin(); - for (;;) { - if (!(__pyx_t_2 != __pyx_t_3->end())) break; - __pyx_t_4 = *__pyx_t_2; - ++__pyx_t_2; - __pyx_cur_scope->__pyx_v_cpair = __pyx_t_4; - - /* "khmer/_oxli.pyx":141 - * if lockedptr: - * for cpair in deref(lockedptr).data: - * yield cpair.first, Component.create(cpair.second) # <<<<<<<<<<<<<< - * else: - * raise MemoryError("Can't locked underlying Component set") - */ - __pyx_t_5 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_cur_scope->__pyx_v_cpair.first); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 141, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = ((PyObject *)__pyx_f_5khmer_5_oxli_9Component_create(__pyx_cur_scope->__pyx_v_cpair.second)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 141, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 141, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_GIVEREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6); - __pyx_t_5 = 0; - __pyx_t_6 = 0; - __pyx_r = __pyx_t_7; - __pyx_t_7 = 0; - __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L7_resume_from_yield:; - __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; - __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(1, 141, __pyx_L1_error) - - /* "khmer/_oxli.pyx":140 - * lockedptr = self._tag_component_map.lock() - * if lockedptr: - * for cpair in deref(lockedptr).data: # <<<<<<<<<<<<<< - * yield cpair.first, Component.create(cpair.second) - * else: - */ - } - - /* "khmer/_oxli.pyx":139 - * cdef pair[HashIntoType,ComponentPtr] cpair - * lockedptr = self._tag_component_map.lock() - * if lockedptr: # <<<<<<<<<<<<<< - * for cpair in deref(lockedptr).data: - * yield cpair.first, Component.create(cpair.second) - */ - goto __pyx_L4; - } - - /* "khmer/_oxli.pyx":143 - * yield cpair.first, Component.create(cpair.second) - * else: - * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< - * - * def write_components(self, filename): - */ - /*else*/ { - __pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 143, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_Raise(__pyx_t_7, 0, 0, 0); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __PYX_ERR(1, 143, __pyx_L1_error) - } - __pyx_L4:; - if (1); else __pyx_cur_scope = __pyx_cur_scope; - - /* "khmer/_oxli.pyx":135 - * raise MemoryError("Can't locked underlying Component set") - * - * def tag_components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[GuardedKmerCompMap] locked - * cdef pair[HashIntoType,ComponentPtr] cpair - */ - - /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_AddTraceback("tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_XDECREF(__pyx_r); __pyx_r = 0; - __pyx_generator->resume_label = -1; - __Pyx_Coroutine_clear((PyObject*)__pyx_generator); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":145 - * raise MemoryError("Can't locked underlying Component set") - * - * def write_components(self, filename): # <<<<<<<<<<<<<< - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_17write_components(PyObject *__pyx_v_self, PyObject *__pyx_v_filename); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_17write_components(PyObject *__pyx_v_self, PyObject *__pyx_v_filename) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("write_components (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_16write_components(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_filename)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_16write_components(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename) { - FILE *__pyx_v_fp; - khmer::ComponentPtr __pyx_v_cmpptr; - std::shared_ptr __pyx_v_lockedptr; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - char const *__pyx_t_3; - int __pyx_t_4; - bool __pyx_t_5; - std::set ::iterator __pyx_t_6; - khmer::ComponentPtrSet *__pyx_t_7; - khmer::ComponentPtr __pyx_t_8; - __Pyx_RefNannySetupContext("write_components", 0); - - /* "khmer/_oxli.pyx":147 - * def write_components(self, filename): - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') # <<<<<<<<<<<<<< - * if fp == NULL: - * raise IOError("Can't open file.") - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_filename, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 147, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 147, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_PyObject_AsString(__pyx_t_2); if (unlikely((!__pyx_t_3) && PyErr_Occurred())) __PYX_ERR(1, 147, __pyx_L1_error) - __pyx_v_fp = fopen(__pyx_t_3, ((char const *)"wb")); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "khmer/_oxli.pyx":148 - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') - * if fp == NULL: # <<<<<<<<<<<<<< - * raise IOError("Can't open file.") - * - */ - __pyx_t_4 = ((__pyx_v_fp == NULL) != 0); - if (__pyx_t_4) { - - /* "khmer/_oxli.pyx":149 - * fp = fopen(filename.encode('utf-8'), 'wb') - * if fp == NULL: - * raise IOError("Can't open file.") # <<<<<<<<<<<<<< - * - * cdef ComponentPtr cmpptr - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 149, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 149, __pyx_L1_error) - - /* "khmer/_oxli.pyx":148 - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') - * if fp == NULL: # <<<<<<<<<<<<<< - * raise IOError("Can't open file.") - * - */ - } - - /* "khmer/_oxli.pyx":153 - * cdef ComponentPtr cmpptr - * cdef shared_ptr[ComponentPtrSet] lockedptr - * lockedptr = self._components.lock() # <<<<<<<<<<<<<< - * - * if lockedptr: - */ - __pyx_v_lockedptr = __pyx_v_self->_components.lock(); - - /* "khmer/_oxli.pyx":155 - * lockedptr = self._components.lock() - * - * if lockedptr: # <<<<<<<<<<<<<< - * for cmpptr in deref(lockedptr): - * fprintf(fp, "%llu,%llu,%f\n", - */ - __pyx_t_5 = __pyx_v_lockedptr.operator bool(); - if (__pyx_t_5) { - - /* "khmer/_oxli.pyx":156 - * - * if lockedptr: - * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< - * fprintf(fp, "%llu,%llu,%f\n", - * deref(cmpptr).component_id, - */ - __pyx_t_7 = &(*__pyx_v_lockedptr); - __pyx_t_6 = __pyx_t_7->begin(); - for (;;) { - if (!(__pyx_t_6 != __pyx_t_7->end())) break; - __pyx_t_8 = *__pyx_t_6; - ++__pyx_t_6; - __pyx_v_cmpptr = __pyx_t_8; - - /* "khmer/_oxli.pyx":157 - * if lockedptr: - * for cmpptr in deref(lockedptr): - * fprintf(fp, "%llu,%llu,%f\n", # <<<<<<<<<<<<<< - * deref(cmpptr).component_id, - * deref(cmpptr).get_n_tags(), - */ - fprintf(__pyx_v_fp, ((char const *)"%llu,%llu,%f\n"), (*__pyx_v_cmpptr).component_id, (*__pyx_v_cmpptr).get_n_tags(), __pyx_f_5khmer_5_oxli_9Component__mean_tag_count(__pyx_v_cmpptr, __pyx_v_self->_graph_ptr)); - - /* "khmer/_oxli.pyx":156 - * - * if lockedptr: - * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< - * fprintf(fp, "%llu,%llu,%f\n", - * deref(cmpptr).component_id, - */ - } - - /* "khmer/_oxli.pyx":155 - * lockedptr = self._components.lock() - * - * if lockedptr: # <<<<<<<<<<<<<< - * for cmpptr in deref(lockedptr): - * fprintf(fp, "%llu,%llu,%f\n", - */ - } - - /* "khmer/_oxli.pyx":161 - * deref(cmpptr).get_n_tags(), - * Component._mean_tag_count(cmpptr, self._graph_ptr)) - * fclose(fp) # <<<<<<<<<<<<<< - * - * property n_components: - */ - fclose(__pyx_v_fp); - - /* "khmer/_oxli.pyx":145 - * raise MemoryError("Can't locked underlying Component set") - * - * def write_components(self, filename): # <<<<<<<<<<<<<< - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.write_components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":164 - * - * property n_components: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_components() - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli.pyx":165 - * property n_components: - * def __get__(self): - * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< - * - * property n_tags: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 165, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":164 - * - * property n_components: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_components() - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.n_components.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":168 - * - * property n_tags: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_tags() - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_6n_tags_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_6n_tags_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6n_tags___get__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_6n_tags___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli.pyx":169 - * property n_tags: - * def __get__(self): - * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_tags()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 169, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli.pyx":168 - * - * property n_tags: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_tags() - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.n_tags.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli.pyx":86 - * cdef weak_ptr[GuardedKmerCompMap] _tag_component_map - * cdef CyHashtable * _graph_ptr - * cdef readonly uint64_t n_consumed # <<<<<<<<<<<<<< - * - * def __cinit__(self, graph): - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_10n_consumed_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_10n_consumed_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10n_consumed___get__(((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_20StreamingPartitioner_10n_consumed___get__(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t(__pyx_v_self->n_consumed); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 86, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.StreamingPartitioner.n_consumed.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "vector.to_py":67 - * - * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") - * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): # <<<<<<<<<<<<<< - * return [X_to_py(v[i]) for i in range(v.size())] - * - */ - -static PyObject *__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(const std::vector &__pyx_v_v) { - size_t __pyx_v_i; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - size_t __pyx_t_2; - size_t __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannySetupContext("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType", 0); - - /* "vector.to_py":68 - * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") - * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): - * return [X_to_py(v[i]) for i in range(v.size())] # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 68, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_v_v.size(); - for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { - __pyx_v_i = __pyx_t_3; - __pyx_t_4 = __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 68, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(2, 68, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - } - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "vector.to_py":67 - * - * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") - * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): # <<<<<<<<<<<<<< - * return [X_to_py(v[i]) for i in range(v.size())] - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("vector.to_py.__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - -static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { - Py_ssize_t __pyx_v_length; - char *__pyx_v_data; - std::string __pyx_r; - __Pyx_RefNannyDeclarations - char *__pyx_t_1; - __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); - - /* "string.from_py":15 - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< - * return string(data, length) - * - */ - __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(2, 15, __pyx_L1_error) - __pyx_v_data = __pyx_t_1; - - /* "string.from_py":16 - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - * return string(data, length) # <<<<<<<<<<<<<< - * - * - */ - __pyx_r = std::string(__pyx_v_data, __pyx_v_length); - goto __pyx_L0; - - /* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static struct __pyx_vtabstruct_5khmer_5_oxli_Component __pyx_vtable_5khmer_5_oxli_Component; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_Component(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_Component *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5khmer_5_oxli_Component *)o); - p->__pyx_vtab = __pyx_vtabptr_5khmer_5_oxli_Component; - new((void*)&(p->_this)) khmer::ComponentPtr(); - if (unlikely(__pyx_pw_5khmer_5_oxli_9Component_1__cinit__(o, a, k) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_Component(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_Component *p = (struct __pyx_obj_5khmer_5_oxli_Component *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - __Pyx_call_destructor(p->_this); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_9Component_component_id(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_9Component_12component_id_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_9Component__n_created(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_9Component_10_n_created_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_9Component__n_destroyed(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_9Component_12_n_destroyed_1__get__(o); -} - -static PyMethodDef __pyx_methods_5khmer_5_oxli_Component[] = { - {"tag_counts", (PyCFunction)__pyx_pw_5khmer_5_oxli_9Component_12tag_counts, METH_VARARGS|METH_KEYWORDS, 0}, - {0, 0, 0, 0} -}; - -static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_Component[] = { - {(char *)"component_id", __pyx_getprop_5khmer_5_oxli_9Component_component_id, 0, (char *)0, 0}, - {(char *)"_n_created", __pyx_getprop_5khmer_5_oxli_9Component__n_created, 0, (char *)0, 0}, - {(char *)"_n_destroyed", __pyx_getprop_5khmer_5_oxli_9Component__n_destroyed, 0, (char *)0, 0}, - {0, 0, 0, 0, 0} -}; - -static PySequenceMethods __pyx_tp_as_sequence_Component = { - __pyx_pw_5khmer_5_oxli_9Component_3__len__, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Component = { - __pyx_pw_5khmer_5_oxli_9Component_3__len__, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyTypeObject __pyx_type_5khmer_5_oxli_Component = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.Component", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_Component), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_Component, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - &__pyx_tp_as_sequence_Component, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Component, /*tp_as_mapping*/ - __pyx_pw_5khmer_5_oxli_9Component_8__hash__, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - __pyx_pw_5khmer_5_oxli_9Component_10__richcmp__, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - __pyx_pw_5khmer_5_oxli_9Component_5__iter__, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5khmer_5_oxli_Component, /*tp_methods*/ - 0, /*tp_members*/ - __pyx_getsets_5khmer_5_oxli_Component, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_Component, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)o); - new((void*)&(p->_this)) std::unique_ptr (); - new((void*)&(p->_components)) std::weak_ptr (); - new((void*)&(p->_tag_component_map)) std::weak_ptr (); - if (unlikely(__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_1__cinit__(o, a, k) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_StreamingPartitioner(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *p = (struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - __Pyx_call_destructor(p->_this); - __Pyx_call_destructor(p->_components); - __Pyx_call_destructor(p->_tag_component_map); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_components(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_12n_components_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_tags(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_6n_tags_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_consumed(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_20StreamingPartitioner_10n_consumed_1__get__(o); -} - -static PyMethodDef __pyx_methods_5khmer_5_oxli_StreamingPartitioner[] = { - {"consume_sequence", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_3consume_sequence, METH_O, 0}, - {"consume_fasta", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_5consume_fasta, METH_O, 0}, - {"get_tag_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_7get_tag_component, METH_O, 0}, - {"get_nearest_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_9get_nearest_component, METH_O, 0}, - {"components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_11components, METH_NOARGS, 0}, - {"tag_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_14tag_components, METH_NOARGS, 0}, - {"write_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_20StreamingPartitioner_17write_components, METH_O, 0}, - {0, 0, 0, 0} -}; - -static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_StreamingPartitioner[] = { - {(char *)"n_components", __pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_components, 0, (char *)0, 0}, - {(char *)"n_tags", __pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_tags, 0, (char *)0, 0}, - {(char *)"n_consumed", __pyx_getprop_5khmer_5_oxli_20StreamingPartitioner_n_consumed, 0, (char *)0, 0}, - {0, 0, 0, 0, 0} -}; - -static PyTypeObject __pyx_type_5khmer_5_oxli_StreamingPartitioner = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.StreamingPartitioner", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_StreamingPartitioner, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5khmer_5_oxli_StreamingPartitioner, /*tp_methods*/ - 0, /*tp_members*/ - __pyx_getsets_5khmer_5_oxli_StreamingPartitioner, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_StreamingPartitioner, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *__pyx_freelist_5khmer_5_oxli___pyx_scope_struct____iter__[8]; -static int __pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__ = 0; - -static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p; - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__ > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__)))) { - o = (PyObject*)__pyx_freelist_5khmer_5_oxli___pyx_scope_struct____iter__[--__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__]; - memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__)); - (void) PyObject_INIT(o, t); - PyObject_GC_Track(o); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o); - new((void*)&(p->__pyx_t_0)) std::set ::iterator(); - return o; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct____iter__(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o; - PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_t_0); - Py_CLEAR(p->__pyx_v_self); - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__ < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__)))) { - __pyx_freelist_5khmer_5_oxli___pyx_scope_struct____iter__[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct____iter__++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static int __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct____iter__(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o; - if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct____iter__(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__ *)o; - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_Component *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyTypeObject __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__ = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.__pyx_scope_struct____iter__", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct____iter__), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct____iter__, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct____iter__, /*tp_traverse*/ - __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct____iter__, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli___pyx_scope_struct____iter__, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *__pyx_freelist_5khmer_5_oxli___pyx_scope_struct_1_components[8]; -static int __pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components = 0; - -static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p; - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components)))) { - o = (PyObject*)__pyx_freelist_5khmer_5_oxli___pyx_scope_struct_1_components[--__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components]; - memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components)); - (void) PyObject_INIT(o, t); - PyObject_GC_Track(o); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o); - new((void*)&(p->__pyx_v_cmpptr)) khmer::ComponentPtr(); - new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); - new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); - new((void*)&(p->__pyx_t_0)) std::set ::iterator(); - return o; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_1_components(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o; - PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_v_cmpptr); - __Pyx_call_destructor(p->__pyx_v_locked); - __Pyx_call_destructor(p->__pyx_v_lockedptr); - __Pyx_call_destructor(p->__pyx_t_0); - Py_CLEAR(p->__pyx_v_self); - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components)))) { - __pyx_freelist_5khmer_5_oxli___pyx_scope_struct_1_components[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_1_components++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static int __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct_1_components(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o; - if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct_1_components(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components *)o; - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyTypeObject __pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.__pyx_scope_struct_1_components", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_1_components), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_1_components, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct_1_components, /*tp_traverse*/ - __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct_1_components, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_1_components, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *__pyx_freelist_5khmer_5_oxli___pyx_scope_struct_2_tag_components[8]; -static int __pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components = 0; - -static PyObject *__pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p; - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components)))) { - o = (PyObject*)__pyx_freelist_5khmer_5_oxli___pyx_scope_struct_2_tag_components[--__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components]; - memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components)); - (void) PyObject_INIT(o, t); - PyObject_GC_Track(o); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - p = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o); - new((void*)&(p->__pyx_v_cpair)) std::pair (); - new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); - new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); - new((void*)&(p->__pyx_t_0)) std::map ::iterator(); - return o; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o; - PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_v_cpair); - __Pyx_call_destructor(p->__pyx_v_locked); - __Pyx_call_destructor(p->__pyx_v_lockedptr); - __Pyx_call_destructor(p->__pyx_t_0); - Py_CLEAR(p->__pyx_v_self); - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components)))) { - __pyx_freelist_5khmer_5_oxli___pyx_scope_struct_2_tag_components[__pyx_freecount_5khmer_5_oxli___pyx_scope_struct_2_tag_components++] = ((struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static int __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o; - if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct_2_tag_components(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components *)o; - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_StreamingPartitioner *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyTypeObject __pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.__pyx_scope_struct_2_tag_components", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli___pyx_scope_struct_2_tag_components), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli___pyx_scope_struct_2_tag_components, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - __pyx_tp_traverse_5khmer_5_oxli___pyx_scope_struct_2_tag_components, /*tp_traverse*/ - __pyx_tp_clear_5khmer_5_oxli___pyx_scope_struct_2_tag_components, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli___pyx_scope_struct_2_tag_components, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyMethodDef __pyx_methods[] = { - {0, 0, 0, 0} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - #if PY_VERSION_HEX < 0x03020000 - { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, - #else - PyModuleDef_HEAD_INIT, - #endif - "_oxli", - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp_s_Can_t_locked_underlying_Componen, __pyx_k_Can_t_locked_underlying_Componen, sizeof(__pyx_k_Can_t_locked_underlying_Componen), 0, 0, 1, 0}, - {&__pyx_kp_s_Can_t_open_file, __pyx_k_Can_t_open_file, sizeof(__pyx_k_Can_t_open_file), 0, 0, 1, 0}, - {&__pyx_n_s_Component___iter, __pyx_k_Component___iter, sizeof(__pyx_k_Component___iter), 0, 0, 1, 1}, - {&__pyx_n_s_Countgraph, __pyx_k_Countgraph, sizeof(__pyx_k_Countgraph), 0, 0, 1, 1}, - {&__pyx_n_s_IOError, __pyx_k_IOError, sizeof(__pyx_k_IOError), 0, 0, 1, 1}, - {&__pyx_n_s_MemoryError, __pyx_k_MemoryError, sizeof(__pyx_k_MemoryError), 0, 0, 1, 1}, - {&__pyx_kp_s_Must_take_an_object_with_Hashtab, __pyx_k_Must_take_an_object_with_Hashtab, sizeof(__pyx_k_Must_take_an_object_with_Hashtab), 0, 0, 1, 0}, - {&__pyx_n_s_Nodegraph, __pyx_k_Nodegraph, sizeof(__pyx_k_Nodegraph), 0, 0, 1, 1}, - {&__pyx_n_s_NotImplementedError, __pyx_k_NotImplementedError, sizeof(__pyx_k_NotImplementedError), 0, 0, 1, 1}, - {&__pyx_kp_s_Operator_not_available, __pyx_k_Operator_not_available, sizeof(__pyx_k_Operator_not_available), 0, 0, 1, 0}, - {&__pyx_n_s_StreamingPartitioner_components, __pyx_k_StreamingPartitioner_components, sizeof(__pyx_k_StreamingPartitioner_components), 0, 0, 1, 1}, - {&__pyx_n_s_StreamingPartitioner_tag_compone, __pyx_k_StreamingPartitioner_tag_compone, sizeof(__pyx_k_StreamingPartitioner_tag_compone), 0, 0, 1, 1}, - {&__pyx_kp_s_Users_camille_w_khmer_khmer__ox, __pyx_k_Users_camille_w_khmer_khmer__ox, sizeof(__pyx_k_Users_camille_w_khmer_khmer__ox), 0, 0, 1, 0}, - {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, - {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, - {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, - {&__pyx_n_s_component, __pyx_k_component, sizeof(__pyx_k_component), 0, 0, 1, 1}, - {&__pyx_n_s_component_id, __pyx_k_component_id, sizeof(__pyx_k_component_id), 0, 0, 1, 1}, - {&__pyx_n_s_components, __pyx_k_components, sizeof(__pyx_k_components), 0, 0, 1, 1}, - {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, - {&__pyx_n_s_graph, __pyx_k_graph, sizeof(__pyx_k_graph), 0, 0, 1, 1}, - {&__pyx_n_s_graph_ptr, __pyx_k_graph_ptr, sizeof(__pyx_k_graph_ptr), 0, 0, 1, 1}, - {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, - {&__pyx_n_s_iter, __pyx_k_iter, sizeof(__pyx_k_iter), 0, 0, 1, 1}, - {&__pyx_n_s_khmer__khmer, __pyx_k_khmer__khmer, sizeof(__pyx_k_khmer__khmer), 0, 0, 1, 1}, - {&__pyx_n_s_khmer__oxli, __pyx_k_khmer__oxli, sizeof(__pyx_k_khmer__oxli), 0, 0, 1, 1}, - {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, - {&__pyx_n_s_other, __pyx_k_other, sizeof(__pyx_k_other), 0, 0, 1, 1}, - {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, - {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, - {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, - {&__pyx_n_s_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 0, 0, 1, 1}, - {&__pyx_n_s_tag_components, __pyx_k_tag_components, sizeof(__pyx_k_tag_components), 0, 0, 1, 1}, - {&__pyx_n_s_tag_counts, __pyx_k_tag_counts, sizeof(__pyx_k_tag_counts), 0, 0, 1, 1}, - {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, - {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, - {0, 0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 85, __pyx_L1_error) - __pyx_builtin_staticmethod = __Pyx_GetBuiltinName(__pyx_n_s_staticmethod); if (!__pyx_builtin_staticmethod) __PYX_ERR(1, 65, __pyx_L1_error) - __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(1, 46, __pyx_L1_error) - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 90, __pyx_L1_error) - __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s_IOError); if (!__pyx_builtin_IOError) __PYX_ERR(1, 149, __pyx_L1_error) - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) - return 0; - __pyx_L1_error:; - return -1; -} - -static int __Pyx_InitCachedConstants(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - - /* "khmer/_oxli.pyx":46 - * return x.component_id == y.component_id - * else: - * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Operator_not_available); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 46, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple_); - __Pyx_GIVEREF(__pyx_tuple_); - - /* "khmer/_oxli.pyx":90 - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< - * - * - */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 90, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__2); - __Pyx_GIVEREF(__pyx_tuple__2); - - /* "khmer/_oxli.pyx":103 - * - * def consume_sequence(self, sequence): - * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< - * self.n_consumed += 1 - * - */ - __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 103, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__3); - __Pyx_GIVEREF(__pyx_tuple__3); - - /* "khmer/_oxli.pyx":107 - * - * def consume_fasta(self, filename): - * return deref(self._this).consume_fasta(filename.encode('utf-8')) # <<<<<<<<<<<<<< - * - * def get_tag_component(self, kmer): - */ - __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 107, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__4); - __Pyx_GIVEREF(__pyx_tuple__4); - - /* "khmer/_oxli.pyx":111 - * def get_tag_component(self, kmer): - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if compptr == NULL: - * return None - */ - __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 111, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__5); - __Pyx_GIVEREF(__pyx_tuple__5); - - /* "khmer/_oxli.pyx":119 - * def get_nearest_component(self, kmer): - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if compptr == NULL: - * return None - */ - __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 119, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__6); - __Pyx_GIVEREF(__pyx_tuple__6); - - /* "khmer/_oxli.pyx":133 - * yield Component.create(cmpptr) - * else: - * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< - * - * def tag_components(self): - */ - __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 133, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__7); - __Pyx_GIVEREF(__pyx_tuple__7); - - /* "khmer/_oxli.pyx":143 - * yield cpair.first, Component.create(cpair.second) - * else: - * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< - * - * def write_components(self, filename): - */ - __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(1, 143, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__8); - __Pyx_GIVEREF(__pyx_tuple__8); - - /* "khmer/_oxli.pyx":147 - * def write_components(self, filename): - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') # <<<<<<<<<<<<<< - * if fp == NULL: - * raise IOError("Can't open file.") - */ - __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 147, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__9); - __Pyx_GIVEREF(__pyx_tuple__9); - - /* "khmer/_oxli.pyx":149 - * fp = fopen(filename.encode('utf-8'), 'wb') - * if fp == NULL: - * raise IOError("Can't open file.") # <<<<<<<<<<<<<< - * - * cdef ComponentPtr cmpptr - */ - __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_s_Can_t_open_file); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(1, 149, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - - /* "khmer/_oxli.pyx":66 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CyCpHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - __pyx_tuple__11 = PyTuple_Pack(3, __pyx_n_s_component, __pyx_n_s_graph, __pyx_n_s_graph_ptr); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(1, 66, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__11); - __Pyx_GIVEREF(__pyx_tuple__11); - __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_camille_w_khmer_khmer__ox, __pyx_n_s_tag_counts, 66, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(1, 66, __pyx_L1_error) - __Pyx_RefNannyFinishContext(); - return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(1, 1, __pyx_L1_error); - __pyx_int_2 = PyInt_FromLong(2); if (unlikely(!__pyx_int_2)) __PYX_ERR(1, 1, __pyx_L1_error) - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC init_oxli(void); /*proto*/ -PyMODINIT_FUNC init_oxli(void) -#else -PyMODINIT_FUNC PyInit__oxli(void); /*proto*/ -PyMODINIT_FUNC PyInit__oxli(void) -#endif -{ - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannyDeclarations - #if CYTHON_REFNANNY - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); - if (!__Pyx_RefNanny) { - PyErr_Clear(); - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); - if (!__Pyx_RefNanny) - Py_FatalError("failed to import 'refnanny' module"); - } - #endif - __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit__oxli(void)", 0); - if (__Pyx_check_binary_version() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(1, 1, __pyx_L1_error) - #ifdef __Pyx_CyFunction_USED - if (__pyx_CyFunction_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_FusedFunction_USED - if (__pyx_FusedFunction_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Coroutine_USED - if (__pyx_Coroutine_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Generator_USED - if (__pyx_Generator_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_StopAsyncIteration_USED - if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - #endif - /*--- Library function declarations ---*/ - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4("_oxli", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (unlikely(!__pyx_m)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(1, 1, __pyx_L1_error) - Py_INCREF(__pyx_d); - __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(1, 1, __pyx_L1_error) - #if CYTHON_COMPILING_IN_PYPY - Py_INCREF(__pyx_b); - #endif - if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(1, 1, __pyx_L1_error); - /*--- Initialize various global constants etc. ---*/ - if (__Pyx_InitGlobals() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - #endif - if (__pyx_module_is_main_khmer___oxli) { - if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(1, 1, __pyx_L1_error) - } - #if PY_MAJOR_VERSION >= 3 - { - PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(1, 1, __pyx_L1_error) - if (!PyDict_GetItemString(modules, "khmer._oxli")) { - if (unlikely(PyDict_SetItemString(modules, "khmer._oxli", __pyx_m) < 0)) __PYX_ERR(1, 1, __pyx_L1_error) - } - } - #endif - /*--- Builtin init code ---*/ - if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - /*--- Constants init code ---*/ - if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - /*--- Global init code ---*/ - /*--- Variable export code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - __pyx_vtabptr_5khmer_5_oxli_Component = &__pyx_vtable_5khmer_5_oxli_Component; - __pyx_vtable_5khmer_5_oxli_Component.create = (struct __pyx_obj_5khmer_5_oxli_Component *(*)(khmer::ComponentPtr))__pyx_f_5khmer_5_oxli_9Component_create; - __pyx_vtable_5khmer_5_oxli_Component._tag_counts = (std::vector (*)(khmer::ComponentPtr, khmer::Hashtable *))__pyx_f_5khmer_5_oxli_9Component__tag_counts; - __pyx_vtable_5khmer_5_oxli_Component._mean_tag_count = (float (*)(khmer::ComponentPtr, khmer::Hashtable *))__pyx_f_5khmer_5_oxli_9Component__mean_tag_count; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 11, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_Component.tp_print = 0; - if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_Component.tp_dict, __pyx_vtabptr_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 11, __pyx_L1_error) - if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_Component) < 0) __PYX_ERR(1, 11, __pyx_L1_error) - __pyx_ptype_5khmer_5_oxli_Component = &__pyx_type_5khmer_5_oxli_Component; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 80, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_StreamingPartitioner.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_StreamingPartitioner) < 0) __PYX_ERR(1, 80, __pyx_L1_error) - __pyx_ptype_5khmer_5_oxli_StreamingPartitioner = &__pyx_type_5khmer_5_oxli_StreamingPartitioner; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__) < 0) __PYX_ERR(1, 34, __pyx_L1_error) - __pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__.tp_print = 0; - __pyx_ptype_5khmer_5_oxli___pyx_scope_struct____iter__ = &__pyx_type_5khmer_5_oxli___pyx_scope_struct____iter__; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components) < 0) __PYX_ERR(1, 125, __pyx_L1_error) - __pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components.tp_print = 0; - __pyx_ptype_5khmer_5_oxli___pyx_scope_struct_1_components = &__pyx_type_5khmer_5_oxli___pyx_scope_struct_1_components; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components) < 0) __PYX_ERR(1, 135, __pyx_L1_error) - __pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components.tp_print = 0; - __pyx_ptype_5khmer_5_oxli___pyx_scope_struct_2_tag_components = &__pyx_type_5khmer_5_oxli___pyx_scope_struct_2_tag_components; - /*--- Type import code ---*/ - /*--- Variable import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - if (__Pyx_patch_abc() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - #endif - - /* "khmer/_oxli.pyx":8 - * from libc.stdio cimport FILE, fopen, fwrite, fclose, stdout, stderr, fprintf - * - * from khmer._khmer import Countgraph # <<<<<<<<<<<<<< - * from khmer._khmer import Nodegraph - * - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(__pyx_n_s_Countgraph); - __Pyx_GIVEREF(__pyx_n_s_Countgraph); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_Countgraph); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(1, 8, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "khmer/_oxli.pyx":9 - * - * from khmer._khmer import Countgraph - * from khmer._khmer import Nodegraph # <<<<<<<<<<<<<< - * - * cdef class Component: - */ - __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_INCREF(__pyx_n_s_Nodegraph); - __Pyx_GIVEREF(__pyx_n_s_Nodegraph); - PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Nodegraph); - __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer__khmer, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 9, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(1, 9, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "khmer/_oxli.pyx":66 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CyCpHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5khmer_5_oxli_9Component_12tag_counts, NULL, __pyx_n_s_khmer__oxli); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - - /* "khmer/_oxli.pyx":65 - * return counts - * - * @staticmethod # <<<<<<<<<<<<<< - * def tag_counts(Component component, graph): - * cdef CyCpHashtable_Object* graph_ptr = graph - */ - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_staticmethod, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5khmer_5_oxli_Component->tp_dict, __pyx_n_s_tag_counts, __pyx_t_1) < 0) __PYX_ERR(1, 66, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - PyType_Modified(__pyx_ptype_5khmer_5_oxli_Component); - - /* "khmer/_oxli.pyx":66 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CyCpHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - __pyx_t_1 = __Pyx_GetNameInClass((PyObject *)__pyx_ptype_5khmer_5_oxli_Component, __pyx_n_s_tag_counts); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - - /* "khmer/_oxli.pyx":65 - * return counts - * - * @staticmethod # <<<<<<<<<<<<<< - * def tag_counts(Component component, graph): - * cdef CyCpHashtable_Object* graph_ptr = graph - */ - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_staticmethod, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5khmer_5_oxli_Component->tp_dict, __pyx_n_s_tag_counts, __pyx_t_1) < 0) __PYX_ERR(1, 66, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - PyType_Modified(__pyx_ptype_5khmer_5_oxli_Component); - - /* "khmer/_oxli.pyx":1 - * import cython # <<<<<<<<<<<<<< - * from cython.operator cimport dereference as deref, preincrement as inc - * from libc.limits cimport UINT_MAX - */ - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(1, 1, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - - /*--- Wrapped vars code ---*/ - - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - if (__pyx_m) { - if (__pyx_d) { - __Pyx_AddTraceback("init khmer._oxli", __pyx_clineno, __pyx_lineno, __pyx_filename); - } - Py_DECREF(__pyx_m); __pyx_m = 0; - } else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ImportError, "init khmer._oxli"); - } - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -/* --- Runtime support code --- */ -/* Refnanny */ -#if CYTHON_REFNANNY -static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { - PyObject *m = NULL, *p = NULL; - void *r = NULL; - m = PyImport_ImportModule((char *)modname); - if (!m) goto end; - p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); - if (!p) goto end; - r = PyLong_AsVoidPtr(p); -end: - Py_XDECREF(p); - Py_XDECREF(m); - return (__Pyx_RefNannyAPIStruct *)r; -} -#endif - -/* GetBuiltinName */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); - if (unlikely(!result)) { - PyErr_Format(PyExc_NameError, -#if PY_MAJOR_VERSION >= 3 - "name '%U' is not defined", name); -#else - "name '%.200s' is not defined", PyString_AS_STRING(name)); -#endif - } - return result; -} - -/* RaiseDoubleKeywords */ -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AsString(kw_name)); - #endif -} - -/* ParseKeywords */ -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - continue; - } - name = first_kw_arg; - #if PY_MAJOR_VERSION < 3 - if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { - while (*name) { - if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) - && _PyString_Eq(**name, key)) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - if ((**argname == key) || ( - (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) - && _PyString_Eq(**argname, key))) { - goto arg_passed_twice; - } - argname++; - } - } - } else - #endif - if (likely(PyUnicode_Check(key))) { - while (*name) { - int cmp = (**name == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**name, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - int cmp = (**argname == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**argname, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) goto arg_passed_twice; - argname++; - } - } - } else - goto invalid_keyword_type; - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -/* RaiseArgTupleInvalid */ -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *more_or_less; - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", - func_name, more_or_less, num_expected, - (num_expected == 1) ? "" : "s", num_found); -} - -/* ArgTypeTest */ -static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); -} -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - if (none_allowed && obj == Py_None) return 1; - else if (exact) { - if (likely(Py_TYPE(obj) == type)) return 1; - #if PY_MAJOR_VERSION == 2 - else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } - else { - if (likely(PyObject_TypeCheck(obj, type))) return 1; - } - __Pyx_RaiseArgumentTypeInvalid(name, obj, type); - return 0; -} - -/* PyIntBinop */ -#if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { - if (op1 == op2) { - Py_RETURN_TRUE; - } - #if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(op1))) { - const long b = intval; - long a = PyInt_AS_LONG(op1); - if (a == b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } - } - #endif - #if CYTHON_USE_PYLONG_INTERNALS - if (likely(PyLong_CheckExact(op1))) { - const long b = intval; - long a; - const digit* digits = ((PyLongObject*)op1)->ob_digit; - const Py_ssize_t size = Py_SIZE(op1); - if (likely(__Pyx_sst_abs(size) <= 1)) { - a = likely(size) ? digits[0] : 0; - if (size == -1) a = -a; - } else { - switch (size) { - case -2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case -3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case -4: - if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 4: - if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 - default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); - #else - default: Py_RETURN_FALSE; - #endif - } - } - if (a == b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } - } - #endif - if (PyFloat_CheckExact(op1)) { - const long b = intval; - double a = PyFloat_AS_DOUBLE(op1); - if ((double)a == (double)b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } - } - return PyObject_RichCompare(op1, op2, Py_EQ); -} -#endif - -/* PyObjectCall */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyObject *result; - ternaryfunc call = func->ob_type->tp_call; - if (unlikely(!call)) - return PyObject_Call(func, arg, kw); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = (*call)(func, arg, kw); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyErrFetchRestore */ -#if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -} -#endif - -/* RaiseException */ -#if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { - __Pyx_PyThreadState_declare - Py_XINCREF(type); - if (!value || value == Py_None) - value = NULL; - else - Py_INCREF(value); - if (!tb || tb == Py_None) - tb = NULL; - else { - Py_INCREF(tb); - if (!PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - } - if (PyType_Check(type)) { -#if CYTHON_COMPILING_IN_PYPY - if (!value) { - Py_INCREF(Py_None); - value = Py_None; - } -#endif - PyErr_NormalizeException(&type, &value, &tb); - } else { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - value = type; - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - } - __Pyx_PyThreadState_assign - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} -#else -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { - PyObject* owned_instance = NULL; - if (tb == Py_None) { - tb = 0; - } else if (tb && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto bad; - } - if (value == Py_None) - value = 0; - if (PyExceptionInstance_Check(type)) { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto bad; - } - value = type; - type = (PyObject*) Py_TYPE(value); - } else if (PyExceptionClass_Check(type)) { - PyObject *instance_class = NULL; - if (value && PyExceptionInstance_Check(value)) { - instance_class = (PyObject*) Py_TYPE(value); - if (instance_class != type) { - int is_subclass = PyObject_IsSubclass(instance_class, type); - if (!is_subclass) { - instance_class = NULL; - } else if (unlikely(is_subclass == -1)) { - goto bad; - } else { - type = instance_class; - } - } - } - if (!instance_class) { - PyObject *args; - if (!value) - args = PyTuple_New(0); - else if (PyTuple_Check(value)) { - Py_INCREF(value); - args = value; - } else - args = PyTuple_Pack(1, value); - if (!args) - goto bad; - owned_instance = PyObject_Call(type, args, NULL); - Py_DECREF(args); - if (!owned_instance) - goto bad; - value = owned_instance; - if (!PyExceptionInstance_Check(value)) { - PyErr_Format(PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto bad; - } - } - } else { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto bad; - } -#if PY_VERSION_HEX >= 0x03030000 - if (cause) { -#else - if (cause && cause != Py_None) { -#endif - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; - } - PyException_SetCause(value, fixed_cause); - } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = PyThreadState_GET(); - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); - } -#endif - } -bad: - Py_XDECREF(owned_instance); - return; -} -#endif - -/* RaiseTooManyValuesToUnpack */ - static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { - PyErr_Format(PyExc_ValueError, - "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); -} - -/* RaiseNeedMoreValuesToUnpack */ - static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { - PyErr_Format(PyExc_ValueError, - "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", - index, (index == 1) ? "" : "s"); -} - -/* IterFinish */ - static CYTHON_INLINE int __Pyx_IterFinish(void) { -#if CYTHON_FAST_THREAD_STATE - PyThreadState *tstate = PyThreadState_GET(); - PyObject* exc_type = tstate->curexc_type; - if (unlikely(exc_type)) { - if (likely(exc_type == PyExc_StopIteration) || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) { - PyObject *exc_value, *exc_tb; - exc_value = tstate->curexc_value; - exc_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; - Py_DECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_tb); - return 0; - } else { - return -1; - } - } - return 0; -#else - if (unlikely(PyErr_Occurred())) { - if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) { - PyErr_Clear(); - return 0; - } else { - return -1; - } - } - return 0; -#endif -} - -/* UnpackItemEndCheck */ - static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { - if (unlikely(retval)) { - Py_DECREF(retval); - __Pyx_RaiseTooManyValuesError(expected); - return -1; - } else { - return __Pyx_IterFinish(); - } - return 0; -} - -/* WriteUnraisableException */ - static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, - CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, - int full_traceback, CYTHON_UNUSED int nogil) { - PyObject *old_exc, *old_val, *old_tb; - PyObject *ctx; - __Pyx_PyThreadState_declare -#ifdef WITH_THREAD - PyGILState_STATE state; - if (nogil) - state = PyGILState_Ensure(); -#ifdef _MSC_VER - else state = (PyGILState_STATE)-1; -#endif -#endif - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); - if (full_traceback) { - Py_XINCREF(old_exc); - Py_XINCREF(old_val); - Py_XINCREF(old_tb); - __Pyx_ErrRestore(old_exc, old_val, old_tb); - PyErr_PrintEx(1); - } - #if PY_MAJOR_VERSION < 3 - ctx = PyString_FromString(name); - #else - ctx = PyUnicode_FromString(name); - #endif - __Pyx_ErrRestore(old_exc, old_val, old_tb); - if (!ctx) { - PyErr_WriteUnraisable(Py_None); - } else { - PyErr_WriteUnraisable(ctx); - Py_DECREF(ctx); - } -#ifdef WITH_THREAD - if (nogil) - PyGILState_Release(state); -#endif -} - -/* GetModuleGlobalName */ - static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS - result = PyDict_GetItem(__pyx_d, name); - if (likely(result)) { - Py_INCREF(result); - } else { -#else - result = PyObject_GetItem(__pyx_d, name); - if (!result) { - PyErr_Clear(); -#endif - result = __Pyx_GetBuiltinName(name); - } - return result; -} - -/* SetVTable */ - static int __Pyx_SetVtable(PyObject *dict, void *vtable) { -#if PY_VERSION_HEX >= 0x02070000 - PyObject *ob = PyCapsule_New(vtable, 0, 0); -#else - PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); -#endif - if (!ob) - goto bad; - if (PyDict_SetItem(dict, __pyx_n_s_pyx_vtable, ob) < 0) - goto bad; - Py_DECREF(ob); - return 0; -bad: - Py_XDECREF(ob); - return -1; -} - -/* Import */ - static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_import; - py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); - if (!py_import) - goto bad; - #endif - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - { - #if PY_MAJOR_VERSION >= 3 - if (level == -1) { - if (strchr(__Pyx_MODULE_NAME, '.')) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(1); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - #endif - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - goto bad; - PyErr_Clear(); - } - } - level = 0; - } - #endif - if (!module) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(level); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); - #endif - } - } -bad: - #if PY_VERSION_HEX < 0x03030000 - Py_XDECREF(py_import); - #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); - return module; -} - -/* ImportFrom */ - static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { - PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); - if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Format(PyExc_ImportError, - #if PY_MAJOR_VERSION < 3 - "cannot import name %.230s", PyString_AS_STRING(name)); - #else - "cannot import name %S", name); - #endif - } - return value; -} - -/* GetNameInClass */ - static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name) { - PyObject *result; - result = __Pyx_PyObject_GetAttrStr(nmspace, name); - if (!result) - result = __Pyx_GetModuleGlobalName(name); - return result; -} - -/* CodeObjectCache */ - static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { - int start = 0, mid = 0, end = count - 1; - if (end >= 0 && code_line > entries[end].code_line) { - return count; - } - while (start < end) { - mid = start + (end - start) / 2; - if (code_line < entries[mid].code_line) { - end = mid; - } else if (code_line > entries[mid].code_line) { - start = mid + 1; - } else { - return mid; - } - } - if (code_line <= entries[mid].code_line) { - return mid; - } else { - return mid + 1; - } -} -static PyCodeObject *__pyx_find_code_object(int code_line) { - PyCodeObject* code_object; - int pos; - if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { - return NULL; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { - return NULL; - } - code_object = __pyx_code_cache.entries[pos].code_object; - Py_INCREF(code_object); - return code_object; -} -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { - int pos, i; - __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; - if (unlikely(!code_line)) { - return; - } - if (unlikely(!entries)) { - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); - if (likely(entries)) { - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = 64; - __pyx_code_cache.count = 1; - entries[0].code_line = code_line; - entries[0].code_object = code_object; - Py_INCREF(code_object); - } - return; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { - PyCodeObject* tmp = entries[pos].code_object; - entries[pos].code_object = code_object; - Py_DECREF(tmp); - return; - } - if (__pyx_code_cache.count == __pyx_code_cache.max_count) { - int new_max = __pyx_code_cache.max_count + 64; - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( - __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); - if (unlikely(!entries)) { - return; - } - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = new_max; - } - for (i=__pyx_code_cache.count; i>pos; i--) { - entries[i] = entries[i-1]; - } - entries[pos].code_line = code_line; - entries[pos].code_object = code_object; - __pyx_code_cache.count++; - Py_INCREF(code_object); -} - -/* AddTraceback */ - #include "compile.h" -#include "frameobject.h" -#include "traceback.h" -static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( - const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(filename); - #else - py_srcfile = PyUnicode_FromString(filename); - #endif - if (!py_srcfile) goto bad; - if (c_line) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_code = __Pyx_PyCode_New( - 0, - 0, - 0, - 0, - 0, - __pyx_empty_bytes, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - py_line, - __pyx_empty_bytes /*PyObject *lnotab*/ - ); - Py_DECREF(py_srcfile); - Py_DECREF(py_funcname); - return py_code; -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - return NULL; -} -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - py_code = __pyx_find_code_object(c_line ? c_line : py_line); - if (!py_code) { - py_code = __Pyx_CreateCodeObjectForTraceback( - funcname, c_line, py_line, filename); - if (!py_code) goto bad; - __pyx_insert_code_object(c_line ? c_line : py_line, py_code); - } - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - __pyx_d, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - __Pyx_PyFrame_SetLineNumber(py_frame, py_line); - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { - const int neg_one = (int) -1, const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(int) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(int) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(int) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(int), - little, !is_unsigned); - } -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value) { - const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(uint64_t) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(uint64_t) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(uint64_t) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(uint64_t), - little, !is_unsigned); - } -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value) { - const khmer::HashIntoType neg_one = (khmer::HashIntoType) -1, const_zero = (khmer::HashIntoType) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(khmer::HashIntoType) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(khmer::HashIntoType) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::HashIntoType) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(khmer::HashIntoType), - little, !is_unsigned); - } -} - -/* CIntFromPyVerify */ - #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) -#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) -#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ - {\ - func_type value = func_value;\ - if (sizeof(target_type) < sizeof(func_type)) {\ - if (unlikely(value != (func_type) (target_type) value)) {\ - func_type zero = 0;\ - if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ - return (target_type) -1;\ - if (is_unsigned && unlikely(value < zero))\ - goto raise_neg_overflow;\ - else\ - goto raise_overflow;\ - }\ - }\ - return (target_type) value;\ - } - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType(khmer::BoundedCounterType value) { - const khmer::BoundedCounterType neg_one = (khmer::BoundedCounterType) -1, const_zero = (khmer::BoundedCounterType) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(khmer::BoundedCounterType) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(khmer::BoundedCounterType) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::BoundedCounterType) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(khmer::BoundedCounterType) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::BoundedCounterType) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(khmer::BoundedCounterType), - little, !is_unsigned); - } -} - -/* CIntFromPy */ - static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) -1, const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (int) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { - return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { - return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { - return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (int) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) - case -2: - if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - } -#endif - if (sizeof(int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - int val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (int) -1; - } - } else { - int val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (int) -1; - val = __Pyx_PyInt_As_int(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to int"); - return (int) -1; -} - -/* CIntFromPy */ - static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *x) { - const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(uint64_t) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(uint64_t, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (uint64_t) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (uint64_t) 0; - case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, digits[0]) - case 2: - if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) >= 2 * PyLong_SHIFT) { - return (uint64_t) (((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) >= 3 * PyLong_SHIFT) { - return (uint64_t) (((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) >= 4 * PyLong_SHIFT) { - return (uint64_t) (((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (uint64_t) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(uint64_t) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (uint64_t) 0; - case -1: __PYX_VERIFY_RETURN_INT(uint64_t, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, +digits[0]) - case -2: - if (8 * sizeof(uint64_t) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { - return (uint64_t) (((uint64_t)-1)*(((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { - return (uint64_t) ((((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { - return (uint64_t) (((uint64_t)-1)*(((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { - return (uint64_t) ((((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { - return (uint64_t) (((uint64_t)-1)*(((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { - return (uint64_t) ((((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - } -#endif - if (sizeof(uint64_t) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(uint64_t, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(uint64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - uint64_t val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (uint64_t) -1; - } - } else { - uint64_t val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (uint64_t) -1; - val = __Pyx_PyInt_As_uint64_t(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to uint64_t"); - return (uint64_t) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to uint64_t"); - return (uint64_t) -1; -} - -/* CIntFromPy */ - static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { - const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(size_t) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (size_t) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (size_t) 0; - case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) - case 2: - if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { - return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { - return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { - return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (size_t) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(size_t) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (size_t) 0; - case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) - case -2: - if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { - return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - } -#endif - if (sizeof(size_t) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - size_t val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (size_t) -1; - } - } else { - size_t val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (size_t) -1; - val = __Pyx_PyInt_As_size_t(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to size_t"); - return (size_t) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to size_t"); - return (size_t) -1; -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - -/* CIntFromPy */ - static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(long) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (long) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { - return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { - return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { - return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (long) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(long) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) - case -2: - if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - } -#endif - if (sizeof(long) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - long val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (long) -1; - } - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (long) -1; - val = __Pyx_PyInt_As_long(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to long"); - return (long) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to long"); - return (long) -1; -} - -/* FetchCommonType */ - static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } -done: - Py_DECREF(fake_module); - return cached_type; -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; -} - -/* SwapException */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = *type; - tstate->exc_value = *value; - tstate->exc_traceback = *tb; - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} -#else -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); - PyErr_SetExcInfo(*type, *value, *tb); - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} -#endif - -/* PyCFunctionFastCall */ - #if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - PyObject *result; - int flags; - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); -} -#endif // CYTHON_FAST_PYCCALL - -/* PyFunctionFastCall */ - #if CYTHON_FAST_PYCALL -#include "frameobject.h" -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = PyThreadState_GET(); - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = f->f_localsplus; - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; -} -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && -#endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); -#endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); - } - else { - d = NULL; - nd = 0; - } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); -#else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); -#endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; -} -#endif // CPython < 3.6 -#endif // CYTHON_FAST_PYCALL - -/* PyObjectCallMethO */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyObjectCallOneArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif - -/* PyObjectCallMethod1 */ - static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { - PyObject *method, *result = NULL; - method = __Pyx_PyObject_GetAttrStr(obj, method_name); - if (unlikely(!method)) goto done; -#if CYTHON_UNPACK_METHODS - if (likely(PyMethod_Check(method))) { - PyObject *self = PyMethod_GET_SELF(method); - if (likely(self)) { - PyObject *args; - PyObject *function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyFunction_FastCall(function, args, 2); - goto done; - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyCFunction_FastCall(function, args, 2); - goto done; - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; - } - } -#endif - result = __Pyx_PyObject_CallOneArg(method, arg); -done: - Py_XDECREF(method); - return result; -} - -/* CoroutineBase */ - #include -#include -static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); -static PyObject *__Pyx_Coroutine_Close(PyObject *self); -static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); -#define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { - PyObject *et, *ev, *tb; - PyObject *value = NULL; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&et, &ev, &tb); - if (!et) { - Py_XDECREF(tb); - Py_XDECREF(ev); - Py_INCREF(Py_None); - *pvalue = Py_None; - return 0; - } - if (likely(et == PyExc_StopIteration)) { - if (!ev) { - Py_INCREF(Py_None); - value = Py_None; - } -#if PY_VERSION_HEX >= 0x030300A0 - else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { - value = ((PyStopIterationObject *)ev)->value; - Py_INCREF(value); - Py_DECREF(ev); - } -#endif - else if (unlikely(PyTuple_Check(ev))) { - if (PyTuple_GET_SIZE(ev) >= 1) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - value = PyTuple_GET_ITEM(ev, 0); - Py_INCREF(value); -#else - value = PySequence_ITEM(ev, 0); -#endif - } else { - Py_INCREF(Py_None); - value = Py_None; - } - Py_DECREF(ev); - } - else if (!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { - value = ev; - } - if (likely(value)) { - Py_XDECREF(tb); - Py_DECREF(et); - *pvalue = value; - return 0; - } - } else if (!PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - PyErr_NormalizeException(&et, &ev, &tb); - if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - Py_XDECREF(tb); - Py_DECREF(et); -#if PY_VERSION_HEX >= 0x030300A0 - value = ((PyStopIterationObject *)ev)->value; - Py_INCREF(value); - Py_DECREF(ev); -#else - { - PyObject* args = __Pyx_PyObject_GetAttrStr(ev, __pyx_n_s_args); - Py_DECREF(ev); - if (likely(args)) { - value = PySequence_GetItem(args, 0); - Py_DECREF(args); - } - if (unlikely(!value)) { - __Pyx_ErrRestore(NULL, NULL, NULL); - Py_INCREF(Py_None); - value = Py_None; - } - } -#endif - *pvalue = value; - return 0; -} -#endif -static CYTHON_INLINE -void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { - PyObject *exc_type = self->exc_type; - PyObject *exc_value = self->exc_value; - PyObject *exc_traceback = self->exc_traceback; - self->exc_type = NULL; - self->exc_value = NULL; - self->exc_traceback = NULL; - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_traceback); -} -static CYTHON_INLINE -int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) { - if (unlikely(gen->is_running)) { - PyErr_SetString(PyExc_ValueError, - "generator already executing"); - return 1; - } - return 0; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) { - PyObject *retval; - __Pyx_PyThreadState_declare - assert(!self->is_running); - if (unlikely(self->resume_label == 0)) { - if (unlikely(value && value != Py_None)) { - PyErr_SetString(PyExc_TypeError, - "can't send non-None value to a " - "just-started generator"); - return NULL; - } - } - if (unlikely(self->resume_label == -1)) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - __Pyx_PyThreadState_assign - if (value) { -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON -#else - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_XINCREF(__pyx_tstate->frame); - assert(f->f_back == NULL); - f->f_back = __pyx_tstate->frame; - } -#endif - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); - } else { - __Pyx_Coroutine_ExceptionClear(self); - } - self->is_running = 1; - retval = self->body((PyObject *) self, value); - self->is_running = 0; - if (retval) { - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON -#else - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_CLEAR(f->f_back); - } -#endif - } else { - __Pyx_Coroutine_ExceptionClear(self); - } - return retval; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) { - if (unlikely(!retval && !PyErr_Occurred())) { - PyErr_SetNone(PyExc_StopIteration); - } - return retval; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { - PyObject *ret; - PyObject *val = NULL; - __Pyx_Coroutine_Undelegate(gen); - __Pyx_PyGen_FetchStopIterationValue(&val); - ret = __Pyx_Coroutine_SendEx(gen, val); - Py_XDECREF(val); - return ret; -} -static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { - PyObject *retval; - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Coroutine_Send(yf, value); - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - ret = __Pyx_Coroutine_Send(yf, value); - } else - #endif - { - if (value == Py_None) - ret = Py_TYPE(yf)->tp_iternext(yf); - else - ret = __Pyx_PyObject_CallMethod1(yf, __pyx_n_s_send, value); - } - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - retval = __Pyx_Coroutine_FinishDelegation(gen); - } else { - retval = __Pyx_Coroutine_SendEx(gen, value); - } - return __Pyx_Coroutine_MethodReturn(retval); -} -static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { - PyObject *retval = NULL; - int err = 0; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - retval = __Pyx_Coroutine_Close(yf); - if (!retval) - return -1; - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - retval = __Pyx_Coroutine_Close(yf); - if (!retval) - return -1; - } else - #endif - { - PyObject *meth; - gen->is_running = 1; - meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_close); - if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_WriteUnraisable(yf); - } - PyErr_Clear(); - } else { - retval = PyObject_CallFunction(meth, NULL); - Py_DECREF(meth); - if (!retval) - err = -1; - } - gen->is_running = 0; - } - Py_XDECREF(retval); - return err; -} -static PyObject *__Pyx_Generator_Next(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Generator_Next(yf); - } else - #endif - ret = Py_TYPE(yf)->tp_iternext(yf); - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - return __Pyx_Coroutine_FinishDelegation(gen); - } - return __Pyx_Coroutine_SendEx(gen, Py_None); -} -static PyObject *__Pyx_Coroutine_Close(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject *retval, *raised_exception; - PyObject *yf = gen->yieldfrom; - int err = 0; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - Py_INCREF(yf); - err = __Pyx_Coroutine_CloseIter(gen, yf); - __Pyx_Coroutine_Undelegate(gen); - Py_DECREF(yf); - } - if (err == 0) - PyErr_SetNone(PyExc_GeneratorExit); - retval = __Pyx_Coroutine_SendEx(gen, NULL); - if (retval) { - Py_DECREF(retval); - PyErr_SetString(PyExc_RuntimeError, - "generator ignored GeneratorExit"); - return NULL; - } - raised_exception = PyErr_Occurred(); - if (!raised_exception - || raised_exception == PyExc_StopIteration - || raised_exception == PyExc_GeneratorExit - || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) - || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) - { - if (raised_exception) PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - return NULL; -} -static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; - PyObject *yf = gen->yieldfrom; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) - return NULL; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - Py_INCREF(yf); - if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { - int err = __Pyx_Coroutine_CloseIter(gen, yf); - Py_DECREF(yf); - __Pyx_Coroutine_Undelegate(gen); - if (err < 0) - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); - goto throw_here; - } - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else - #endif - { - PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_throw); - if (unlikely(!meth)) { - Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - gen->is_running = 0; - return NULL; - } - PyErr_Clear(); - __Pyx_Coroutine_Undelegate(gen); - gen->is_running = 0; - goto throw_here; - } - ret = PyObject_CallObject(meth, args); - Py_DECREF(meth); - } - gen->is_running = 0; - Py_DECREF(yf); - if (!ret) { - ret = __Pyx_Coroutine_FinishDelegation(gen); - } - return __Pyx_Coroutine_MethodReturn(ret); - } -throw_here: - __Pyx_Raise(typ, val, tb, NULL); - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); -} -static int __Pyx_Coroutine_traverse(PyObject *self, visitproc visit, void *arg) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - Py_VISIT(gen->closure); - Py_VISIT(gen->classobj); - Py_VISIT(gen->yieldfrom); - Py_VISIT(gen->exc_type); - Py_VISIT(gen->exc_value); - Py_VISIT(gen->exc_traceback); - return 0; -} -static int __Pyx_Coroutine_clear(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - Py_CLEAR(gen->closure); - Py_CLEAR(gen->classobj); - Py_CLEAR(gen->yieldfrom); - Py_CLEAR(gen->exc_type); - Py_CLEAR(gen->exc_value); - Py_CLEAR(gen->exc_traceback); - Py_CLEAR(gen->gi_name); - Py_CLEAR(gen->gi_qualname); - return 0; -} -static void __Pyx_Coroutine_dealloc(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject_GC_UnTrack(gen); - if (gen->gi_weakreflist != NULL) - PyObject_ClearWeakRefs(self); - if (gen->resume_label > 0) { - PyObject_GC_Track(self); -#if PY_VERSION_HEX >= 0x030400a1 - if (PyObject_CallFinalizerFromDealloc(self)) -#else - Py_TYPE(gen)->tp_del(self); - if (self->ob_refcnt > 0) -#endif - { - return; - } - PyObject_GC_UnTrack(self); - } - __Pyx_Coroutine_clear(self); - PyObject_GC_Del(gen); -} -static void __Pyx_Coroutine_del(PyObject *self) { - PyObject *res; - PyObject *error_type, *error_value, *error_traceback; - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - __Pyx_PyThreadState_declare - if (gen->resume_label <= 0) - return ; -#if PY_VERSION_HEX < 0x030400a1 - assert(self->ob_refcnt == 0); - self->ob_refcnt = 1; -#endif - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); - res = __Pyx_Coroutine_Close(self); - if (res == NULL) - PyErr_WriteUnraisable(self); - else - Py_DECREF(res); - __Pyx_ErrRestore(error_type, error_value, error_traceback); -#if PY_VERSION_HEX < 0x030400a1 - assert(self->ob_refcnt > 0); - if (--self->ob_refcnt == 0) { - return; - } - { - Py_ssize_t refcnt = self->ob_refcnt; - _Py_NewReference(self); - self->ob_refcnt = refcnt; - } -#if CYTHON_COMPILING_IN_CPYTHON - assert(PyType_IS_GC(self->ob_type) && - _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); - _Py_DEC_REFTOTAL; -#endif -#ifdef COUNT_ALLOCS - --Py_TYPE(self)->tp_frees; - --Py_TYPE(self)->tp_allocs; -#endif -#endif -} -static PyObject * -__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self) -{ - PyObject *name = self->gi_name; - if (unlikely(!name)) name = Py_None; - Py_INCREF(name); - return name; -} -static int -__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { -#else - if (unlikely(value == NULL || !PyString_Check(value))) { -#endif - PyErr_SetString(PyExc_TypeError, - "__name__ must be set to a string object"); - return -1; - } - tmp = self->gi_name; - Py_INCREF(value); - self->gi_name = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self) -{ - PyObject *name = self->gi_qualname; - if (unlikely(!name)) name = Py_None; - Py_INCREF(name); - return name; -} -static int -__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { -#else - if (unlikely(value == NULL || !PyString_Check(value))) { -#endif - PyErr_SetString(PyExc_TypeError, - "__qualname__ must be set to a string object"); - return -1; - } - tmp = self->gi_qualname; - Py_INCREF(value); - self->gi_qualname = value; - Py_XDECREF(tmp); - return 0; -} -static __pyx_CoroutineObject *__Pyx__Coroutine_New( - PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *closure, - PyObject *name, PyObject *qualname, PyObject *module_name) { - __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); - if (gen == NULL) - return NULL; - gen->body = body; - gen->closure = closure; - Py_XINCREF(closure); - gen->is_running = 0; - gen->resume_label = 0; - gen->classobj = NULL; - gen->yieldfrom = NULL; - gen->exc_type = NULL; - gen->exc_value = NULL; - gen->exc_traceback = NULL; - gen->gi_weakreflist = NULL; - Py_XINCREF(qualname); - gen->gi_qualname = qualname; - Py_XINCREF(name); - gen->gi_name = name; - Py_XINCREF(module_name); - gen->gi_modulename = module_name; - PyObject_GC_Track(gen); - return gen; -} - -/* PatchModuleWithCoroutine */ - static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code) { -#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - int result; - PyObject *globals, *result_obj; - globals = PyDict_New(); if (unlikely(!globals)) goto ignore; - result = PyDict_SetItemString(globals, "_cython_coroutine_type", - #ifdef __Pyx_Coroutine_USED - (PyObject*)__pyx_CoroutineType); - #else - Py_None); - #endif - if (unlikely(result < 0)) goto ignore; - result = PyDict_SetItemString(globals, "_cython_generator_type", - #ifdef __Pyx_Generator_USED - (PyObject*)__pyx_GeneratorType); - #else - Py_None); - #endif - if (unlikely(result < 0)) goto ignore; - if (unlikely(PyDict_SetItemString(globals, "_module", module) < 0)) goto ignore; - if (unlikely(PyDict_SetItemString(globals, "__builtins__", __pyx_b) < 0)) goto ignore; - result_obj = PyRun_String(py_code, Py_file_input, globals, globals); - if (unlikely(!result_obj)) goto ignore; - Py_DECREF(result_obj); - Py_DECREF(globals); - return module; -ignore: - Py_XDECREF(globals); - PyErr_WriteUnraisable(module); - if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, "Cython module failed to patch module with custom type", 1) < 0)) { - Py_DECREF(module); - module = NULL; - } -#else - py_code++; -#endif - return module; -} - -/* PatchGeneratorABC */ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) -static PyObject* __Pyx_patch_abc_module(PyObject *module); -static PyObject* __Pyx_patch_abc_module(PyObject *module) { - module = __Pyx_Coroutine_patch_module( - module, "" -"if _cython_generator_type is not None:\n" -" try: Generator = _module.Generator\n" -" except AttributeError: pass\n" -" else: Generator.register(_cython_generator_type)\n" -"if _cython_coroutine_type is not None:\n" -" try: Coroutine = _module.Coroutine\n" -" except AttributeError: pass\n" -" else: Coroutine.register(_cython_coroutine_type)\n" - ); - return module; -} -#endif -static int __Pyx_patch_abc(void) { -#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - static int abc_patched = 0; - if (!abc_patched) { - PyObject *module; - module = PyImport_ImportModule((PY_VERSION_HEX >= 0x03030000) ? "collections.abc" : "collections"); - if (!module) { - PyErr_WriteUnraisable(NULL); - if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, - ((PY_VERSION_HEX >= 0x03030000) ? - "Cython module failed to register with collections.abc module" : - "Cython module failed to register with collections module"), 1) < 0)) { - return -1; - } - } else { - module = __Pyx_patch_abc_module(module); - abc_patched = 1; - if (unlikely(!module)) - return -1; - Py_DECREF(module); - } - module = PyImport_ImportModule("backports_abc"); - if (module) { - module = __Pyx_patch_abc_module(module); - Py_XDECREF(module); - } - if (!module) { - PyErr_Clear(); - } - } -#else - if (0) __Pyx_Coroutine_patch_module(NULL, NULL); -#endif - return 0; -} - -/* Generator */ - static PyMethodDef __pyx_Generator_methods[] = { - {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, - (char*) PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, - {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, - (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, - {"close", (PyCFunction) __Pyx_Coroutine_Close, METH_NOARGS, - (char*) PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, - {0, 0, 0, 0} -}; -static PyMemberDef __pyx_Generator_memberlist[] = { - {(char *) "gi_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, - {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, - (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, - {0, 0, 0, 0, 0} -}; -static PyGetSetDef __pyx_Generator_getsets[] = { - {(char *) "__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, - (char*) PyDoc_STR("name of the generator"), 0}, - {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, - (char*) PyDoc_STR("qualified name of the generator"), 0}, - {0, 0, 0, 0, 0} -}; -static PyTypeObject __pyx_GeneratorType_type = { - PyVarObject_HEAD_INIT(0, 0) - "generator", - sizeof(__pyx_CoroutineObject), - 0, - (destructor) __Pyx_Coroutine_dealloc, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, - 0, - (traverseproc) __Pyx_Coroutine_traverse, - 0, - 0, - offsetof(__pyx_CoroutineObject, gi_weakreflist), - 0, - (iternextfunc) __Pyx_Generator_Next, - __pyx_Generator_methods, - __pyx_Generator_memberlist, - __pyx_Generator_getsets, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, -#if PY_VERSION_HEX >= 0x030400a1 - 0, -#else - __Pyx_Coroutine_del, -#endif - 0, -#if PY_VERSION_HEX >= 0x030400a1 - __Pyx_Coroutine_del, -#endif -}; -static int __pyx_Generator_init(void) { - __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; - __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; - __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); - if (unlikely(!__pyx_GeneratorType)) { - return -1; - } - return 0; -} - -/* CheckBinaryVersion */ - static int __Pyx_check_binary_version(void) { - char ctversion[4], rtversion[4]; - PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); - PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); - if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { - char message[200]; - PyOS_snprintf(message, sizeof(message), - "compiletime version %s of module '%.100s' " - "does not match runtime version %s", - ctversion, __Pyx_MODULE_NAME, rtversion); - return PyErr_WarnEx(NULL, message, 1); - } - return 0; -} - -/* InitStrings */ - static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { - return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { - Py_ssize_t ignore; - return __Pyx_PyObject_AsStringAndSize(o, &ignore); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { -#if PY_VERSION_HEX < 0x03030000 - char* defenc_c; - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - PyUnicode_AsASCIIString(o); - return NULL; - } - } - } -#endif - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; -#else - if (__Pyx_PyUnicode_READY(o) == -1) return NULL; -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (PyUnicode_IS_ASCII(o)) { - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - PyUnicode_AsASCIIString(o); - return NULL; - } -#else - return PyUnicode_AsUTF8AndSize(o, length); -#endif -#endif - } else -#endif -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) - if (PyByteArray_Check(o)) { - *length = PyByteArray_GET_SIZE(o); - return PyByteArray_AS_STRING(o); - } else -#endif - { - char* result; - int r = PyBytes_AsStringAndSize(o, &result, length); - if (unlikely(r < 0)) { - return NULL; - } else { - return result; - } - } -} -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - int is_true = x == Py_True; - if (is_true | (x == Py_False) | (x == Py_None)) return is_true; - else return PyObject_IsTrue(x); -} -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { -#if CYTHON_USE_TYPE_SLOTS - PyNumberMethods *m; -#endif - const char *name = NULL; - PyObject *res = NULL; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return __Pyx_NewRef(x); -#if CYTHON_USE_TYPE_SLOTS - m = Py_TYPE(x)->tp_as_number; - #if PY_MAJOR_VERSION < 3 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } - #else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } - #endif -#else - res = PyNumber_Int(x); -#endif - if (res) { -#if PY_MAJOR_VERSION < 3 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject *x; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(b))) { - if (sizeof(Py_ssize_t) >= sizeof(long)) - return PyInt_AS_LONG(b); - else - return PyInt_AsSsize_t(x); - } -#endif - if (likely(PyLong_CheckExact(b))) { - #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)b)->ob_digit; - const Py_ssize_t size = Py_SIZE(b); - if (likely(__Pyx_sst_abs(size) <= 1)) { - ival = likely(size) ? digits[0] : 0; - if (size == -1) ival = -ival; - return ival; - } else { - switch (size) { - case 2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - } - } - #endif - return PyLong_AsSsize_t(b); - } - x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { - return PyInt_FromSize_t(ival); -} - - -#endif /* Py_PYTHON_H */ +#error Do not use this file, it is the result of a failed Cython compilation. diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index 983c9033ff..e9a41c6ff6 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -17,20 +17,8 @@ cdef extern from "khmer.hh" namespace "khmer": ctypedef bool (*KmerFilter) (Kmer kmer) -cdef extern from "hashtable.hh" namespace "khmer": - cdef cppclass CyHashtable "khmer::Hashtable": - CyHashtable(WordLength) - const BoundedCounterType get_count(const char *) const - const BoundedCounterType get_count(HashIntoType) const - - -cdef extern from "_khmer.hh": - ctypedef struct CyCpHashtable_Object "khmer::khmer_KHashtable_Object": - CyHashtable* hashtable - - cdef extern from "kmer_hash.hh" namespace "khmer": - cdef cppclass Kmer: + cdef cppclass Kmer "khmer::Kmer": HashIntoType kmer_f HashIntoType kmer_r HashIntoType kmer_u @@ -40,61 +28,25 @@ cdef extern from "kmer_hash.hh" namespace "khmer": Kmer() bool is_forward() const + void set_from_unique_hash(HashIntoType, WordLength) + HashIntoType _hash(const string, const WordLength) + string _revhash(HashIntoType, WordLength) -cdef extern from "traversal.hh": - cdef cppclass CyTraverser "khmer::Traverser": - CyLeftNodeGatherer(CyHashtable *) - - void push_filter(KmerFilter) - KmerFilter pop_filter() - - uint32_t traverse(const Kmer&, KmerQueue&) const - uint32_t traverse_left(const Kmer&, KmerQueue&) const - uint32_t traverse_right(const Kmer&, KmerQueue&) const - - uint32_t degree(const Kmer&) const - uint32_t degree_left(const Kmer&) const - uint32_t degree_right(const Kmer&) const +# Definitions for interacting with the CPython interface. +# All we really need are the PyObject struct definitions +# for our extension objects. +cdef extern from "_khmer.hh": + ctypedef struct CpHashtable_Object "khmer::khmer_KHashtable_Object": + Hashtable* hashtable -cdef extern from "partitioning.hh" namespace "khmer": - - cdef cppclass CyComponent "khmer::Component": - const uint64_t component_id - set[HashIntoType] tags - - void merge(set[shared_ptr[CyComponent]]) - void add_tag(HashIntoType) - void add_tag(set[HashIntoType]) - uint64_t get_n_tags() const - uint64_t get_n_created() const - uint64_t get_n_destroyed() const - - ctypedef shared_ptr[CyComponent] ComponentPtr - ctypedef set[ComponentPtr] ComponentPtrSet - - cdef cppclass GuardedKmerCompMap "khmer::GuardedKmerCompMap": - map[HashIntoType, ComponentPtr] data - - ComponentPtr get(HashIntoType) - void set(HashIntoType, ComponentPtr) - bool contains(HashIntoType) - - cdef cppclass CyStreamingPartitioner "khmer::StreamingPartitioner": - CyStreamingPartitioner(CyHashtable * ) except +MemoryError - void consume_sequence(string&) except +MemoryError - uint64_t consume_fasta(string&) except +MemoryError - void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) - void find_connected_tags(queue[Kmer]&, - set[HashIntoType]&, - set[HashIntoType]&) except +MemoryError - uint64_t get_n_components() const - uint64_t get_n_tags() const +cdef extern from "hashtable.hh" namespace "khmer": + cdef cppclass Hashtable "khmer::Hashtable": + Hashtable(WordLength) + const BoundedCounterType get_count(const char *) const + const BoundedCounterType get_count(HashIntoType) const - ComponentPtr get_tag_component(string&) const - ComponentPtr get_nearest_component(string&) const +#### - weak_ptr[ComponentPtrSet] get_component_set() - weak_ptr[GuardedKmerCompMap] get_tag_component_map() diff --git a/khmer/_oxli/_oxli.pyx b/khmer/_oxli/_oxli.pyx index 95bb8aae74..462314d9d9 100644 --- a/khmer/_oxli/_oxli.pyx +++ b/khmer/_oxli/_oxli.pyx @@ -1,169 +1,41 @@ import cython -from cython.operator cimport dereference as deref, preincrement as inc -from libc.limits cimport UINT_MAX -from libcpp.memory cimport unique_ptr, weak_ptr -from libc.stdint cimport uintptr_t -from libc.stdio cimport FILE, fopen, fwrite, fclose, stdout, stderr, fprintf +from libcpp.string cimport string +from libcpp.memory cimport shared_ptr +from libc.stdint cimport uint64_t -from khmer._khmer import Countgraph -from khmer._khmer import Nodegraph +from cython.operator cimport dereference as deref +cdef class Kmer: -cdef class Component: + cdef shared_ptr[Kmer] _this + cdef readonly str kmer - cdef ComponentPtr _this - - def __cinit__(self, Component other=None): - if other is not None: - self._this.reset(other._this.get()) - - property component_id: - def __get__(self): - return deref(self._this).component_id - - property _n_created: - def __get__(self): - return deref(self._this).get_n_created() - - property _n_destroyed: - def __get__(self): - return deref(self._this).get_n_destroyed() - - def __len__(self): - return deref(self._this).get_n_tags() - - def __iter__(self): - cdef HashIntoType tag - for tag in deref(self._this).tags: - yield tag - - def __hash__(self): - return self._this.get() - - def __richcmp__(x, y, op): - if op == 2: - return x.component_id == y.component_id + def __cinit__(self, kmer=None): + self.kmer = kmer + if self.kmer is not None: + self._this.reset(new Kmer(kmer, len(kmer))) else: - raise NotImplementedError('Operator not available.') + self._this.reset(new Kmer()) - @staticmethod - cdef Component create(ComponentPtr ptr): - cdef Component comp = Component() - comp._this = ptr - return comp + @property + def kmer_f(self): + return deref(self._this).kmer_f - @staticmethod - cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CyHashtable * graph): - cdef uint64_t n_tags = deref(comp).get_n_tags() - cdef vector[BoundedCounterType] counts - counts = vector[BoundedCounterType](n_tags) - cdef int idx - cdef uint64_t tag - for idx, tag in deref(comp).tags: - counts[idx] = deref(graph).get_count(tag) - return counts + @property + def kmer_r(self): + return deref(self._this).kmer_r - @staticmethod - def tag_counts(Component component, graph): - cdef CyCpHashtable_Object* graph_ptr = graph - return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + @property + def kmer_u(self): + return deref(self._this).kmer_u @staticmethod - cdef float _mean_tag_count(ComponentPtr comp, CyHashtable * graph): - cdef uint64_t n_tags = deref(comp).get_n_tags() - cdef float acc = 0 - cdef uint64_t tag - for tag in deref(comp).tags: - acc += deref(graph).get_count(tag) - return acc / n_tags - - -cdef class StreamingPartitioner: - - cdef unique_ptr[CyStreamingPartitioner] _this - cdef weak_ptr[ComponentPtrSet] _components - cdef weak_ptr[GuardedKmerCompMap] _tag_component_map - cdef CyHashtable * _graph_ptr - cdef readonly uint64_t n_consumed - - def __cinit__(self, graph): - if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - raise ValueError('Must take an object with Hashtable *') - - - cdef CyCpHashtable_Object* ptr = graph - self._graph_ptr = deref(ptr).hashtable - - self._this.reset(new CyStreamingPartitioner(self._graph_ptr)) + cdef Kmer create(Kmer& cpkmer): + cdef Kmer kmer = Kmer() + kmer._this.reset(&cpkmer) + return kmer - self._tag_component_map = deref(self._this).get_tag_component_map() - self._components = deref(self._this).get_component_set() - self.n_consumed = 0 - - def consume_sequence(self, sequence): - deref(self._this).consume_sequence(sequence.encode('utf-8')) - self.n_consumed += 1 - - def consume_fasta(self, filename): - return deref(self._this).consume_fasta(filename.encode('utf-8')) - - def get_tag_component(self, kmer): - cdef ComponentPtr compptr - compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - if compptr == NULL: - return None - else: - return Component.create(compptr) - - def get_nearest_component(self, kmer): - cdef ComponentPtr compptr - compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - if compptr == NULL: - return None - else: - return Component.create(compptr) - - def components(self): - cdef shared_ptr[ComponentPtrSet] locked - cdef ComponentPtr cmpptr - lockedptr = self._components.lock() - if lockedptr: - for cmpptr in deref(lockedptr): - yield Component.create(cmpptr) - else: - raise MemoryError("Can't locked underlying Component set") - - def tag_components(self): - cdef shared_ptr[GuardedKmerCompMap] locked - cdef pair[HashIntoType,ComponentPtr] cpair - lockedptr = self._tag_component_map.lock() - if lockedptr: - for cpair in deref(lockedptr).data: - yield cpair.first, Component.create(cpair.second) - else: - raise MemoryError("Can't locked underlying Component set") - - def write_components(self, filename): - cdef FILE* fp - fp = fopen(filename.encode('utf-8'), 'wb') - if fp == NULL: - raise IOError("Can't open file.") - - cdef ComponentPtr cmpptr - cdef shared_ptr[ComponentPtrSet] lockedptr - lockedptr = self._components.lock() - - if lockedptr: - for cmpptr in deref(lockedptr): - fprintf(fp, "%llu,%llu,%f\n", - deref(cmpptr).component_id, - deref(cmpptr).get_n_tags(), - Component._mean_tag_count(cmpptr, self._graph_ptr)) - fclose(fp) - - property n_components: - def __get__(self): - return deref(self._this).get_n_components() - - property n_tags: - def __get__(self): - return deref(self._this).get_n_tags() + @staticmethod + cdef Kmer create(uint64_t tag, WordLength K): + cdef Kmer kmer = Kmer() + deref(kmer._this).set_from_unique_hash(tag, K) + return kmer diff --git a/khmer/_oxli/hashing.pyx b/khmer/_oxli/hashing.pyx new file mode 100644 index 0000000000..f0e66a25c1 --- /dev/null +++ b/khmer/_oxli/hashing.pyx @@ -0,0 +1,42 @@ +from libcpp.string cimport string +from libcpp.memory cimport shared_ptr +from libc.stdint cimport uint64_t +from cython.operator cimport dereference as deref + +cimport ._oxli as _ox + +cdef class Kmer: + + cdef shared_ptr[_ox.Kmer] _this + cdef readonly str kmer + + def __cinit__(self, kmer=None): + self.kmer = kmer + if self.kmer is not None: + self._this.reset(new _ox.Kmer(kmer, len(kmer))) + else: + self._this.reset(new _ox.Kmer()) + + @property + def kmer_f(self): + return deref(self._this).kmer_f + + @property + def kmer_r(self): + return deref(self._this).kmer_r + + @property + def kmer_u(self): + return deref(self._this).kmer_u + + @staticmethod + cdef Kmer create(_ox.Kmer& cpkmer): + cdef Kmer kmer = Kmer() + kmer._this.reset(&cpkmer) + return kmer + + @staticmethod + cdef Kmer create(_ox.HashIntoType tag, _ox.WordLength K): + cdef Kmer kmer = Kmer() + deref(kmer._this).set_from_unique_hash(tag, K) + return kmer diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd new file mode 100644 index 0000000000..16986d3ac7 --- /dev/null +++ b/khmer/_oxli/partitioning.pxd @@ -0,0 +1,12 @@ +from ._oxli cimport * + +from libcpp cimport bool +from libcpp.string cimport string +from libcpp.vector cimport vector +from libcpp.map cimport map +from libcpp.set cimport set +from libcpp.queue cimport queue +from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr +from libcpp.utility cimport pair +from libc.stdint cimport uint32_t, uint8_t, uint64_t + diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx new file mode 100644 index 0000000000..81bad97000 --- /dev/null +++ b/khmer/_oxli/partitioning.pyx @@ -0,0 +1,181 @@ +import cython +from cython.operator cimport dereference as deref, preincrement as inc + +from libcpp cimport bool +from libcpp.string cimport string +from libcpp.vector cimport vector +from libcpp.map cimport map +from libcpp.set cimport set +from libcpp.queue cimport queue +from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr +from libcpp.utility cimport pair + +from libc.stdint cimport uint32_t, uint8_t, uint64_t +from libc.limits cimport UINT_MAX + +from libc.stdint cimport uintptr_t +from libc.stdio cimport FILE, fopen, fwrite, fclose, stdout, stderr, fprintf + +cimport ._oxli as ox +from .._khmer import Countgraph +from .._khmer import Nodegraph + +cdef class Component: + + cdef ox.ComponentPtr _this + + def __cinit__(self, Component other=None): + if other is not None: + self._this.reset(other._this.get()) + + property component_id: + def __get__(self): + return deref(self._this).component_id + + property _n_created: + def __get__(self): + return deref(self._this).get_n_created() + + property _n_destroyed: + def __get__(self): + return deref(self._this).get_n_destroyed() + + def __len__(self): + return deref(self._this).get_n_tags() + + def __iter__(self): + cdef ox.HashIntoType tag + for tag in deref(self._this).tags: + yield tag + + def __hash__(self): + return self._this.get() + + def __richcmp__(x, y, op): + if op == 2: + return x.component_id == y.component_id + else: + raise NotImplementedError('Operator not available.') + + @staticmethod + cdef Component create(ox.ComponentPtr ptr): + cdef Component comp = Component() + comp._this = ptr + return comp + + @staticmethod + cdef vector[ox.BoundedCounterType] _tag_counts(ox.ComponentPtr comp, ox.Hashtable * graph): + cdef uint64_t n_tags = deref(comp).get_n_tags() + cdef vector[ox.BoundedCounterType] counts + counts = vector[ox.BoundedCounterType](n_tags) + cdef int idx + cdef uint64_t tag + for idx, tag in deref(comp).tags: + counts[idx] = deref(graph).get_count(tag) + return counts + + @staticmethod + def tag_counts(Component component, graph): + cdef ox.CpHashtable_Object* graph_ptr = graph + return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + + @staticmethod + cdef float _mean_tag_count(ox.ComponentPtr comp, ox.Hashtable * graph): + cdef uint64_t n_tags = deref(comp).get_n_tags() + cdef float acc = 0 + cdef uint64_t tag + for tag in deref(comp).tags: + acc += deref(graph).get_count(tag) + return acc / n_tags + + +cdef class StreamingPartitioner: + + cdef unique_ptr[ox.StreamingPartitioner] _this + cdef weak_ptr[ox.ComponentPtrSet] _components + cdef weak_ptr[ox.GuardedKmerCompMap] _tag_component_map + cdef ox.Hashtable * _graph_ptr + cdef readonly uint64_t n_consumed + + def __cinit__(self, graph): + if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + raise ValueError('Must take an object with Hashtable *') + + + cdef ox.CpHashtable_Object* ptr = graph + self._graph_ptr = deref(ptr).hashtable + + self._this.reset(new ox.StreamingPartitioner(self._graph_ptr)) + + self._tag_component_map = deref(self._this).get_tag_component_map() + self._components = deref(self._this).get_component_set() + self.n_consumed = 0 + + def consume_sequence(self, sequence): + deref(self._this).consume_sequence(sequence.encode('utf-8')) + self.n_consumed += 1 + + def consume_fasta(self, filename): + return deref(self._this).consume_fasta(filename.encode('utf-8')) + + def get_tag_component(self, kmer): + cdef ox.ComponentPtr compptr + compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + if compptr == NULL: + return None + else: + return Component.create(compptr) + + def get_nearest_component(self, kmer): + cdef ox.ComponentPtr compptr + compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + if compptr == NULL: + return None + else: + return Component.create(compptr) + + def components(self): + cdef shared_ptr[ox.ComponentPtrSet] locked + cdef ox.ComponentPtr cmpptr + lockedptr = self._components.lock() + if lockedptr: + for cmpptr in deref(lockedptr): + yield Component.create(cmpptr) + else: + raise MemoryError("Can't locked underlying Component set") + + def tag_components(self): + cdef shared_ptr[ox.GuardedKmerCompMap] locked + cdef pair[ox.HashIntoType,ox.ComponentPtr] cpair + lockedptr = self._tag_component_map.lock() + if lockedptr: + for cpair in deref(lockedptr).data: + yield cpair.first, Component.create(cpair.second) + else: + raise MemoryError("Can't locked underlying Component set") + + def write_components(self, filename): + cdef FILE* fp + fp = fopen(filename.encode('utf-8'), 'wb') + if fp == NULL: + raise IOError("Can't open file.") + + cdef ox.ComponentPtr cmpptr + cdef shared_ptr[ox.ComponentPtrSet] lockedptr + lockedptr = self._components.lock() + + if lockedptr: + for cmpptr in deref(lockedptr): + fprintf(fp, "%llu,%llu,%f\n", + deref(cmpptr).component_id, + deref(cmpptr).get_n_tags(), + Component._mean_tag_count(cmpptr, self._graph_ptr)) + fclose(fp) + + property n_components: + def __get__(self): + return deref(self._this).get_n_components() + + property n_tags: + def __get__(self): + return deref(self._this).get_n_tags() diff --git a/khmer/_oxli/traversal.pxd b/khmer/_oxli/traversal.pxd new file mode 100644 index 0000000000..f83259e9e0 --- /dev/null +++ b/khmer/_oxli/traversal.pxd @@ -0,0 +1,19 @@ +from ._oxli cimport * + +cdef extern from "traversal.hh": + cdef cppclass Traverser "khmer::Traverser": + Traverser(Hashtable *) + + void push_filter(KmerFilter) + KmerFilter pop_filter() + + uint32_t traverse(const Kmer&, KmerQueue&) const + uint32_t traverse_left(const Kmer&, KmerQueue&) const + uint32_t traverse_right(const Kmer&, KmerQueue&) const + + uint32_t degree(const Kmer&) const + uint32_t degree_left(const Kmer&) const + uint32_t degree_right(const Kmer&) const + + + diff --git a/khmer/_oxli/traversal.pyx b/khmer/_oxli/traversal.pyx new file mode 100644 index 0000000000..e69de29bb2 From 77d0f29cfa2af690c096b2dc5b8fa0768af28df1 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 14 Nov 2016 02:18:22 -0800 Subject: [PATCH 033/185] Move all defs to _oxli.pxd --- khmer/_oxli/_oxli.pxd | 61 ++++++++++++++++++++++++++++++++++++ khmer/_oxli/_oxli.pyx | 41 ------------------------ khmer/_oxli/partitioning.pxd | 12 ------- khmer/_oxli/traversal.pxd | 17 ---------- 4 files changed, 61 insertions(+), 70 deletions(-) delete mode 100644 khmer/_oxli/_oxli.pyx delete mode 100644 khmer/_oxli/partitioning.pxd diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index e9a41c6ff6..2ed99e74ae 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -48,5 +48,66 @@ cdef extern from "hashtable.hh" namespace "khmer": const BoundedCounterType get_count(const char *) const const BoundedCounterType get_count(HashIntoType) const + +cdef extern from "traversal.hh": + cdef cppclass Traverser "khmer::Traverser": + Traverser(Hashtable *) + + void push_filter(KmerFilter) + KmerFilter pop_filter() + + uint32_t traverse(const Kmer&, KmerQueue&) const + uint32_t traverse_left(const Kmer&, KmerQueue&) const + uint32_t traverse_right(const Kmer&, KmerQueue&) const + + uint32_t degree(const Kmer&) const + uint32_t degree_left(const Kmer&) const + uint32_t degree_right(const Kmer&) const + + + + + +cdef extern from "partitioning.hh" namespace "khmer": + + cdef cppclass Component "khmer::Component": + const uint64_t component_id + set[HashIntoType] tags + + void merge(set[shared_ptr[Component]]) + void add_tag(HashIntoType) + void add_tag(set[HashIntoType]) + uint64_t get_n_tags() const + uint64_t get_n_created() const + uint64_t get_n_destroyed() const + + ctypedef shared_ptr[Component] ComponentPtr + ctypedef set[ComponentPtr] ComponentPtrSet + + cdef cppclass GuardedKmerCompMap "khmer::GuardedKmerCompMap": + map[HashIntoType, ComponentPtr] data + + ComponentPtr get(HashIntoType) + void set(HashIntoType, ComponentPtr) + bool contains(HashIntoType) + + cdef cppclass StreamingPartitioner "khmer::StreamingPartitioner": + StreamingPartitioner(Hashtable * ) except +MemoryError + + void consume_sequence(string&) except +MemoryError + uint64_t consume_fasta(string&) except +MemoryError + void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) + void find_connected_tags(queue[Kmer]&, + set[HashIntoType]&, + set[HashIntoType]&) except +MemoryError + uint64_t get_n_components() const + uint64_t get_n_tags() const + + ComponentPtr get_tag_component(string&) const + ComponentPtr get_nearest_component(string&) const + + weak_ptr[ComponentPtrSet] get_component_set() + weak_ptr[GuardedKmerCompMap] get_tag_component_map() + #### diff --git a/khmer/_oxli/_oxli.pyx b/khmer/_oxli/_oxli.pyx deleted file mode 100644 index 462314d9d9..0000000000 --- a/khmer/_oxli/_oxli.pyx +++ /dev/null @@ -1,41 +0,0 @@ -import cython -from libcpp.string cimport string -from libcpp.memory cimport shared_ptr -from libc.stdint cimport uint64_t - -from cython.operator cimport dereference as deref -cdef class Kmer: - - cdef shared_ptr[Kmer] _this - cdef readonly str kmer - - def __cinit__(self, kmer=None): - self.kmer = kmer - if self.kmer is not None: - self._this.reset(new Kmer(kmer, len(kmer))) - else: - self._this.reset(new Kmer()) - - @property - def kmer_f(self): - return deref(self._this).kmer_f - - @property - def kmer_r(self): - return deref(self._this).kmer_r - - @property - def kmer_u(self): - return deref(self._this).kmer_u - - @staticmethod - cdef Kmer create(Kmer& cpkmer): - cdef Kmer kmer = Kmer() - kmer._this.reset(&cpkmer) - return kmer - - @staticmethod - cdef Kmer create(uint64_t tag, WordLength K): - cdef Kmer kmer = Kmer() - deref(kmer._this).set_from_unique_hash(tag, K) - return kmer diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd deleted file mode 100644 index 16986d3ac7..0000000000 --- a/khmer/_oxli/partitioning.pxd +++ /dev/null @@ -1,12 +0,0 @@ -from ._oxli cimport * - -from libcpp cimport bool -from libcpp.string cimport string -from libcpp.vector cimport vector -from libcpp.map cimport map -from libcpp.set cimport set -from libcpp.queue cimport queue -from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr -from libcpp.utility cimport pair -from libc.stdint cimport uint32_t, uint8_t, uint64_t - diff --git a/khmer/_oxli/traversal.pxd b/khmer/_oxli/traversal.pxd index f83259e9e0..9d03fdf20c 100644 --- a/khmer/_oxli/traversal.pxd +++ b/khmer/_oxli/traversal.pxd @@ -1,19 +1,2 @@ from ._oxli cimport * -cdef extern from "traversal.hh": - cdef cppclass Traverser "khmer::Traverser": - Traverser(Hashtable *) - - void push_filter(KmerFilter) - KmerFilter pop_filter() - - uint32_t traverse(const Kmer&, KmerQueue&) const - uint32_t traverse_left(const Kmer&, KmerQueue&) const - uint32_t traverse_right(const Kmer&, KmerQueue&) const - - uint32_t degree(const Kmer&) const - uint32_t degree_left(const Kmer&) const - uint32_t degree_right(const Kmer&) const - - - From e3cad33665066debb286e49de59ddd3b508805c0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 14 Nov 2016 05:01:24 -0800 Subject: [PATCH 034/185] Update setup and Makefile for Cython reorg --- Makefile | 9 +++++---- khmer/_oxli/traversal.pxd | 2 -- setup.py | 2 -- 3 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 khmer/_oxli/traversal.pxd diff --git a/Makefile b/Makefile index 0fe56127cb..463e479fce 100644 --- a/Makefile +++ b/Makefile @@ -38,9 +38,10 @@ # `SHELL=bash` Will break Titus's laptop, so don't use BASH-isms like # `[[` conditional expressions. CPPSOURCES=$(wildcard lib/*.cc lib/*.hh khmer/_khmer.cc) setup.py +CYSOURCES=$(wildcard khmer/_oxli/*.pyx khmer/_oxli/*.pxd) PYSOURCES=$(filter-out khmer/_version.py, \ $(wildcard khmer/*.py scripts/*.py oxli/*.py) ) -SOURCES=$(PYSOURCES) $(CPPSOURCES) setup.py +SOURCES=$(PYSOURCES) $(CPPSOURCES) $(CYSOURCES) setup.py DEVPKGS=pep8==1.6.2 diff_cover autopep8 pylint coverage gcovr pytest \ pydocstyle screed pyenchant cython @@ -83,7 +84,7 @@ endif MODEXT=$(shell python -c \ "import sysconfig;print(sysconfig.get_config_var('SO'))") EXTENSION_MODULE = khmer/_khmer$(MODEXT) -CYTHON_MODULE = khmer/_oxli$(MODEXT) +CYTHON_MODULE = khmer/_oxli/_oxli$(MODEXT) PYLINT_TEMPLATE="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" @@ -103,12 +104,12 @@ install-dependencies: pip install --upgrade --requirement doc/requirements.txt ## sharedobj : build khmer shared object file -sharedobj: $(EXTENSION_MODULE) +sharedobj: $(EXTENSION_MODULE) $(CYTHON_MODULE) $(EXTENSION_MODULE): $(CPPSOURCES) ./setup.py build_ext --inplace -$(CYTHON_MODULE): $(CPPSOURCES) +$(CYTHON_MODULE): $(CPPSOURCES) $(CYSOURCES) ./setup.py build_ext --inplace coverage-debug: $(CPPSOURCES) diff --git a/khmer/_oxli/traversal.pxd b/khmer/_oxli/traversal.pxd deleted file mode 100644 index 9d03fdf20c..0000000000 --- a/khmer/_oxli/traversal.pxd +++ /dev/null @@ -1,2 +0,0 @@ -from ._oxli cimport * - diff --git a/setup.py b/setup.py index f7b25764f1..54e93a7a8a 100755 --- a/setup.py +++ b/setup.py @@ -188,7 +188,6 @@ def build_dir(): EXTENSION_MODS = [Extension("khmer._khmer", ** CP_EXTENSION_MOD_DICT)] for cython_ext in glob.glob(os.path.join("khmer", "_oxli", "*.pyx")): - print('ext for', cython_ext) CY_EXTENSION_MOD_DICT = \ { "sources": [cython_ext], @@ -202,7 +201,6 @@ def build_dir(): } ext_name = "khmer._oxli.{0}".format(splitext(os.path.basename(cython_ext))[0]) - print('Extension name:', ext_name) CY_EXTENSION_MOD = Extension(ext_name, ** CY_EXTENSION_MOD_DICT) EXTENSION_MODS.append(CY_EXTENSION_MOD) From d86dd80aeaa37ec0b6f7cec81dd2c91e90b6247b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 14 Nov 2016 05:02:12 -0800 Subject: [PATCH 035/185] Finish reorganization of cython files and add Kmer and Traverser --- khmer/_oxli/_oxli.pxd | 88 ++++++++++++++++++++++++------------ khmer/_oxli/hashing.pxd | 14 ++++++ khmer/_oxli/hashing.pyx | 30 +++++++----- khmer/_oxli/partitioning.pxd | 28 ++++++++++++ khmer/_oxli/partitioning.pyx | 45 ++++++++---------- khmer/_oxli/traversal.pxd | 10 ++++ khmer/_oxli/traversal.pyx | 39 ++++++++++++++++ 7 files changed, 187 insertions(+), 67 deletions(-) create mode 100644 khmer/_oxli/hashing.pxd create mode 100644 khmer/_oxli/partitioning.pxd create mode 100644 khmer/_oxli/traversal.pxd diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index 2ed99e74ae..d34b96e4c1 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -9,23 +9,36 @@ from libcpp.utility cimport pair from libc.stdint cimport uint32_t, uint8_t, uint64_t +######################################################################## +# +# Core: typedefs from khmer.hh. +# +######################################################################## + cdef extern from "khmer.hh" namespace "khmer": ctypedef unsigned long long int HashIntoType ctypedef unsigned char WordLength ctypedef unsigned short int BoundedCounterType - ctypedef queue[Kmer] KmerQueue - ctypedef bool (*KmerFilter) (Kmer kmer) + ctypedef queue[CpKmer] KmerQueue + ctypedef bool (*KmerFilter) (CpKmer kmer) + +######################################################################## +# +# Hashing: Definitions from kmer_hash.hh. +# +######################################################################## cdef extern from "kmer_hash.hh" namespace "khmer": - cdef cppclass Kmer "khmer::Kmer": + cdef cppclass CpKmer "khmer::Kmer": HashIntoType kmer_f HashIntoType kmer_r HashIntoType kmer_u - Kmer(HashIntoType, HashIntoType, HashIntoType) - Kmer(string, WordLength) - Kmer() + CpKmer(HashIntoType, HashIntoType, HashIntoType) + CpKmer(string, WordLength) + CpKmer(const CpKmer&) + CpKmer() bool is_forward() const void set_from_unique_hash(HashIntoType, WordLength) @@ -34,70 +47,88 @@ cdef extern from "kmer_hash.hh" namespace "khmer": string _revhash(HashIntoType, WordLength) -# Definitions for interacting with the CPython interface. + +######################################################################## +# +# Hashtable: Bindings for the existing CPython Hashtable wrapper. +# +######################################################################## + # All we really need are the PyObject struct definitions # for our extension objects. cdef extern from "_khmer.hh": - ctypedef struct CpHashtable_Object "khmer::khmer_KHashtable_Object": - Hashtable* hashtable + ctypedef struct CPyHashtable_Object "khmer::khmer_KHashtable_Object": + CpHashtable* hashtable cdef extern from "hashtable.hh" namespace "khmer": - cdef cppclass Hashtable "khmer::Hashtable": - Hashtable(WordLength) + cdef cppclass CpHashtable "khmer::Hashtable": + CpHashtable(WordLength) const BoundedCounterType get_count(const char *) const const BoundedCounterType get_count(HashIntoType) const + const WordLength ksize() const + + +######################################################################## +# +# Traversal: wrapper for traversal.hh. +# +######################################################################## cdef extern from "traversal.hh": - cdef cppclass Traverser "khmer::Traverser": - Traverser(Hashtable *) + cdef cppclass CpTraverser "khmer::Traverser": + CpTraverser(CpHashtable *) void push_filter(KmerFilter) KmerFilter pop_filter() - uint32_t traverse(const Kmer&, KmerQueue&) const - uint32_t traverse_left(const Kmer&, KmerQueue&) const - uint32_t traverse_right(const Kmer&, KmerQueue&) const - - uint32_t degree(const Kmer&) const - uint32_t degree_left(const Kmer&) const - uint32_t degree_right(const Kmer&) const + uint32_t traverse(const CpKmer&, KmerQueue&) const + uint32_t traverse_left(const CpKmer&, KmerQueue&) const + uint32_t traverse_right(const CpKmer&, KmerQueue&) const + uint32_t degree(const CpKmer&) const + uint32_t degree_left(const CpKmer&) const + uint32_t degree_right(const CpKmer&) const +######################################################################## +# +# Partitioning: Component and StreamingPartitioner +# +######################################################################## cdef extern from "partitioning.hh" namespace "khmer": - cdef cppclass Component "khmer::Component": + cdef cppclass CpComponent "khmer::Component": const uint64_t component_id set[HashIntoType] tags - void merge(set[shared_ptr[Component]]) + void merge(set[shared_ptr[CpComponent]]) void add_tag(HashIntoType) void add_tag(set[HashIntoType]) uint64_t get_n_tags() const uint64_t get_n_created() const uint64_t get_n_destroyed() const - ctypedef shared_ptr[Component] ComponentPtr + ctypedef shared_ptr[CpComponent] ComponentPtr ctypedef set[ComponentPtr] ComponentPtrSet - cdef cppclass GuardedKmerCompMap "khmer::GuardedKmerCompMap": + cdef cppclass CpGuardedKmerCompMap "khmer::GuardedKmerCompMap": map[HashIntoType, ComponentPtr] data ComponentPtr get(HashIntoType) void set(HashIntoType, ComponentPtr) bool contains(HashIntoType) - cdef cppclass StreamingPartitioner "khmer::StreamingPartitioner": - StreamingPartitioner(Hashtable * ) except +MemoryError + cdef cppclass CpStreamingPartitioner "khmer::StreamingPartitioner": + CpStreamingPartitioner(CpHashtable * ) except +MemoryError void consume_sequence(string&) except +MemoryError uint64_t consume_fasta(string&) except +MemoryError void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) - void find_connected_tags(queue[Kmer]&, + void find_connected_tags(queue[CpKmer]&, set[HashIntoType]&, set[HashIntoType]&) except +MemoryError uint64_t get_n_components() const @@ -107,7 +138,6 @@ cdef extern from "partitioning.hh" namespace "khmer": ComponentPtr get_nearest_component(string&) const weak_ptr[ComponentPtrSet] get_component_set() - weak_ptr[GuardedKmerCompMap] get_tag_component_map() + weak_ptr[CpGuardedKmerCompMap] get_tag_component_map() -#### diff --git a/khmer/_oxli/hashing.pxd b/khmer/_oxli/hashing.pxd new file mode 100644 index 0000000000..c7ebe7d695 --- /dev/null +++ b/khmer/_oxli/hashing.pxd @@ -0,0 +1,14 @@ +from libcpp.memory cimport shared_ptr +from _oxli cimport CpKmer, HashIntoType, WordLength + +cdef class Kmer: + cdef shared_ptr[CpKmer] _this + cdef readonly str kmer + + @staticmethod + cdef Kmer wrap(CpKmer * cpkmer, WordLength K) + + @staticmethod + cdef Kmer create(HashIntoType tag, WordLength K) + + diff --git a/khmer/_oxli/hashing.pyx b/khmer/_oxli/hashing.pyx index f0e66a25c1..0a48ea64d3 100644 --- a/khmer/_oxli/hashing.pyx +++ b/khmer/_oxli/hashing.pyx @@ -1,21 +1,27 @@ from libcpp.string cimport string -from libcpp.memory cimport shared_ptr +from libcpp.memory cimport make_shared from libc.stdint cimport uint64_t from cython.operator cimport dereference as deref -cimport ._oxli as _ox +from _oxli cimport _revhash cdef class Kmer: - cdef shared_ptr[_ox.Kmer] _this - cdef readonly str kmer - - def __cinit__(self, kmer=None): + def __cinit__(self, str kmer=None): self.kmer = kmer if self.kmer is not None: - self._this.reset(new _ox.Kmer(kmer, len(kmer))) + self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) else: - self._this.reset(new _ox.Kmer()) + self._this.reset(new CpKmer()) + + def __len__(self): + return len(self.kmer) + + def __str__(self): + return self.kmer + + def __hash__(self): + return self.kmer_u @property def kmer_f(self): @@ -30,13 +36,15 @@ cdef class Kmer: return deref(self._this).kmer_u @staticmethod - cdef Kmer create(_ox.Kmer& cpkmer): + cdef Kmer wrap(CpKmer * cpkmer, WordLength K): cdef Kmer kmer = Kmer() - kmer._this.reset(&cpkmer) + kmer._this.reset(cpkmer) + kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') return kmer @staticmethod - cdef Kmer create(_ox.HashIntoType tag, _ox.WordLength K): + cdef Kmer create(HashIntoType tag, WordLength K): cdef Kmer kmer = Kmer() deref(kmer._this).set_from_unique_hash(tag, K) + kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') return kmer diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd new file mode 100644 index 0000000000..ca0f67230f --- /dev/null +++ b/khmer/_oxli/partitioning.pxd @@ -0,0 +1,28 @@ +from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr +from libcpp.vector cimport vector +from libc.stdint cimport uint32_t, uint8_t, uint64_t + +from _oxli cimport ComponentPtr, ComponentPtrSet, CpGuardedKmerCompMap +from _oxli cimport CpHashtable, CpStreamingPartitioner, BoundedCounterType + +cdef class Component: + cdef ComponentPtr _this + + @staticmethod + cdef Component create(ComponentPtr ptr) + + @staticmethod + cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph) + + @staticmethod + cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph) + + +cdef class StreamingPartitioner: + cdef unique_ptr[CpStreamingPartitioner] _this + cdef weak_ptr[ComponentPtrSet] _components + cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map + cdef CpHashtable * _graph_ptr + cdef readonly uint64_t n_consumed + + diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 81bad97000..78a43b06da 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -16,14 +16,12 @@ from libc.limits cimport UINT_MAX from libc.stdint cimport uintptr_t from libc.stdio cimport FILE, fopen, fwrite, fclose, stdout, stderr, fprintf -cimport ._oxli as ox +from _oxli cimport * from .._khmer import Countgraph from .._khmer import Nodegraph cdef class Component: - cdef ox.ComponentPtr _this - def __cinit__(self, Component other=None): if other is not None: self._this.reset(other._this.get()) @@ -44,7 +42,7 @@ cdef class Component: return deref(self._this).get_n_tags() def __iter__(self): - cdef ox.HashIntoType tag + cdef HashIntoType tag for tag in deref(self._this).tags: yield tag @@ -58,16 +56,16 @@ cdef class Component: raise NotImplementedError('Operator not available.') @staticmethod - cdef Component create(ox.ComponentPtr ptr): + cdef Component create(ComponentPtr ptr): cdef Component comp = Component() comp._this = ptr return comp @staticmethod - cdef vector[ox.BoundedCounterType] _tag_counts(ox.ComponentPtr comp, ox.Hashtable * graph): + cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph): cdef uint64_t n_tags = deref(comp).get_n_tags() - cdef vector[ox.BoundedCounterType] counts - counts = vector[ox.BoundedCounterType](n_tags) + cdef vector[BoundedCounterType] counts + counts = vector[BoundedCounterType](n_tags) cdef int idx cdef uint64_t tag for idx, tag in deref(comp).tags: @@ -76,11 +74,11 @@ cdef class Component: @staticmethod def tag_counts(Component component, graph): - cdef ox.CpHashtable_Object* graph_ptr = graph + cdef CPyHashtable_Object* graph_ptr = graph return Component._tag_counts(component._this, deref(graph_ptr).hashtable) @staticmethod - cdef float _mean_tag_count(ox.ComponentPtr comp, ox.Hashtable * graph): + cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): cdef uint64_t n_tags = deref(comp).get_n_tags() cdef float acc = 0 cdef uint64_t tag @@ -91,21 +89,14 @@ cdef class Component: cdef class StreamingPartitioner: - cdef unique_ptr[ox.StreamingPartitioner] _this - cdef weak_ptr[ox.ComponentPtrSet] _components - cdef weak_ptr[ox.GuardedKmerCompMap] _tag_component_map - cdef ox.Hashtable * _graph_ptr - cdef readonly uint64_t n_consumed - def __cinit__(self, graph): if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): raise ValueError('Must take an object with Hashtable *') - - cdef ox.CpHashtable_Object* ptr = graph + cdef CPyHashtable_Object* ptr = graph self._graph_ptr = deref(ptr).hashtable - self._this.reset(new ox.StreamingPartitioner(self._graph_ptr)) + self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) self._tag_component_map = deref(self._this).get_tag_component_map() self._components = deref(self._this).get_component_set() @@ -119,7 +110,7 @@ cdef class StreamingPartitioner: return deref(self._this).consume_fasta(filename.encode('utf-8')) def get_tag_component(self, kmer): - cdef ox.ComponentPtr compptr + cdef ComponentPtr compptr compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) if compptr == NULL: return None @@ -127,7 +118,7 @@ cdef class StreamingPartitioner: return Component.create(compptr) def get_nearest_component(self, kmer): - cdef ox.ComponentPtr compptr + cdef ComponentPtr compptr compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) if compptr == NULL: return None @@ -135,8 +126,8 @@ cdef class StreamingPartitioner: return Component.create(compptr) def components(self): - cdef shared_ptr[ox.ComponentPtrSet] locked - cdef ox.ComponentPtr cmpptr + cdef shared_ptr[ComponentPtrSet] locked + cdef ComponentPtr cmpptr lockedptr = self._components.lock() if lockedptr: for cmpptr in deref(lockedptr): @@ -145,8 +136,8 @@ cdef class StreamingPartitioner: raise MemoryError("Can't locked underlying Component set") def tag_components(self): - cdef shared_ptr[ox.GuardedKmerCompMap] locked - cdef pair[ox.HashIntoType,ox.ComponentPtr] cpair + cdef shared_ptr[CpGuardedKmerCompMap] locked + cdef pair[HashIntoType,ComponentPtr] cpair lockedptr = self._tag_component_map.lock() if lockedptr: for cpair in deref(lockedptr).data: @@ -160,8 +151,8 @@ cdef class StreamingPartitioner: if fp == NULL: raise IOError("Can't open file.") - cdef ox.ComponentPtr cmpptr - cdef shared_ptr[ox.ComponentPtrSet] lockedptr + cdef ComponentPtr cmpptr + cdef shared_ptr[ComponentPtrSet] lockedptr lockedptr = self._components.lock() if lockedptr: diff --git a/khmer/_oxli/traversal.pxd b/khmer/_oxli/traversal.pxd new file mode 100644 index 0000000000..934f514aa1 --- /dev/null +++ b/khmer/_oxli/traversal.pxd @@ -0,0 +1,10 @@ +from libcpp.memory cimport unique_ptr + +from _oxli cimport CpTraverser, CpHashtable + +cdef class Traverser: + cdef unique_ptr[CpTraverser] _this + cdef CpHashtable * _graph_ptr + + + diff --git a/khmer/_oxli/traversal.pyx b/khmer/_oxli/traversal.pyx index e69de29bb2..9dca040012 100644 --- a/khmer/_oxli/traversal.pyx +++ b/khmer/_oxli/traversal.pyx @@ -0,0 +1,39 @@ +from libcpp.memory cimport unique_ptr +from cython.operator cimport dereference as deref + +from .._khmer import Countgraph +from .._khmer import Nodegraph + +from _oxli cimport * +cimport hashing +import hashing + +cdef class Traverser: + + def __cinit__(self, graph): + if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + raise ValueError('Must take an object with Hashtable *') + + cdef CPyHashtable_Object* ptr = graph + self._graph_ptr = deref(ptr).hashtable + + self._this.reset(new CpTraverser(self._graph_ptr)) + ''' + def left_neighbors(self, Kmer kmer): + cdef KmerQueue kmer_q + cdef Kmer neighbor + deref(self._this).traverse_left(kmer, kmer_q) + for neighbor in kmer_q: + yield neighbor + ''' + + def right_neighbors(self, hashing.Kmer kmer): + cdef KmerQueue kmer_q + cdef CpKmer neighbor + cdef CpKmer* ptr + deref(self._this).traverse_right(deref(kmer._this), kmer_q) + while(kmer_q.empty() == 0): + neighbor = kmer_q.back() + ptr = new CpKmer(neighbor) + yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) + kmer_q.pop() From eeed8b80fe05cb275cf0445fc378b62192aaaffd Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 14 Nov 2016 05:02:36 -0800 Subject: [PATCH 036/185] Update cpp files --- khmer/_oxli/_oxli.cpp | 1 - khmer/_oxli/hashing.cpp | 3569 +++++++++++++++ khmer/_oxli/partitioning.cpp | 7995 ++++++++++++++++++++++++++++++++++ khmer/_oxli/traversal.cpp | 4360 ++++++++++++++++++ 4 files changed, 15924 insertions(+), 1 deletion(-) delete mode 100644 khmer/_oxli/_oxli.cpp create mode 100644 khmer/_oxli/hashing.cpp create mode 100644 khmer/_oxli/partitioning.cpp create mode 100644 khmer/_oxli/traversal.cpp diff --git a/khmer/_oxli/_oxli.cpp b/khmer/_oxli/_oxli.cpp deleted file mode 100644 index 06f2230dd7..0000000000 --- a/khmer/_oxli/_oxli.cpp +++ /dev/null @@ -1 +0,0 @@ -#error Do not use this file, it is the result of a failed Cython compilation. diff --git a/khmer/_oxli/hashing.cpp b/khmer/_oxli/hashing.cpp new file mode 100644 index 0000000000..2a5066a163 --- /dev/null +++ b/khmer/_oxli/hashing.cpp @@ -0,0 +1,3569 @@ +/* Generated by Cython 0.25.1 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) + #error Cython requires Python 2.6+ or Python 3.2+. +#else +#define CYTHON_ABI "0_25_1" +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#ifndef HAVE_LONG_LONG + #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) + #define HAVE_LONG_LONG + #endif +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#ifdef PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#elif defined(PYSTON_VERSION) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #if PY_MAJOR_VERSION < 3 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #ifndef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if PY_VERSION_HEX < 0x030300F0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif +#endif +#if !defined(CYTHON_FAST_PYCCALL) +#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #include "longintrepr.h" + #undef SHIFT + #undef BASE + #undef MASK +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) + #define Py_OptimizeFlag 0 +#endif +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyClass_Type +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #define __Pyx_PyCFunctionFast _PyCFunctionFast +#endif +#if CYTHON_FAST_PYCCALL +#define __Pyx_PyFastCFunction_Check(func)\ + ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) +#else +#define __Pyx_PyFastCFunction_Check(func) 0 +#endif +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) +#else + #define CYTHON_PEP393_ENABLED 0 + #define PyUnicode_1BYTE_KIND 1 + #define PyUnicode_2BYTE_KIND 2 + #define PyUnicode_4BYTE_KIND 4 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) + #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_PYSTON + #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) +#else + #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) +#endif +#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) + #define PyObject_ASCII(o) PyObject_Repr(o) +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyStringObject PyUnicodeObject + #define PyString_Type PyUnicode_Type + #define PyString_Check PyUnicode_Check + #define PyString_CheckExact PyUnicode_CheckExact +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) + #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) +#else + #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) + #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#endif +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#if PY_MAJOR_VERSION >= 3 + #define PyIntObject PyLongObject + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define PyNumber_Int PyNumber_Long +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBoolObject PyLongObject +#endif +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY + #ifndef PyUnicode_InternFromString + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) + #endif +#endif +#if PY_VERSION_HEX < 0x030200A4 + typedef long Py_hash_t; + #define __Pyx_PyInt_FromHash_t PyInt_FromLong + #define __Pyx_PyInt_AsHash_t PyInt_AsLong +#else + #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t + #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) +#else + #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) +#endif +#if CYTHON_USE_ASYNC_SLOTS + #if PY_VERSION_HEX >= 0x030500B1 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods + #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) + #else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) + +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #define CYTHON_INLINE inline +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(U other) { return *ptr == other; }; + template bool operator !=(U other) { return *ptr != other; }; + private: + T *ptr; +}; + +#if defined(WIN32) || defined(MS_WINDOWS) + #define _USE_MATH_DEFINES +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + + +#define __PYX_ERR(f_index, lineno, Ln_error) \ +{ \ + __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ +} + +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#define __PYX_HAVE__khmer___oxli__hashing +#define __PYX_HAVE_API__khmer___oxli__hashing +#include +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include +#include +#include +#include +#include +#include +#include +#include "khmer.hh" +#include "kmer_hash.hh" +#include "_khmer.hh" +#include "hashtable.hh" +#include "traversal.hh" +#include "partitioning.hh" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#ifdef PYREX_WITHOUT_ASSERTIONS +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; + const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) && defined (_M_X64) + #define __Pyx_sst_abs(value) _abs64(value) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#else + #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize +#endif +#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#if PY_MAJOR_VERSION < 3 +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ + const Py_UNICODE *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen +#endif +#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) +#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) +#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) +#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +#if CYTHON_ASSUME_SAFE_MACROS +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#else +#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#endif +#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#else +#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) +#endif +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII +static int __Pyx_sys_getdefaultencoding_not_ascii; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + PyObject* ascii_chars_u = NULL; + PyObject* ascii_chars_b = NULL; + const char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + if (strcmp(default_encoding_c, "ascii") == 0) { + __Pyx_sys_getdefaultencoding_not_ascii = 0; + } else { + char ascii_chars[128]; + int c; + for (c = 0; c < 128; c++) { + ascii_chars[c] = c; + } + __Pyx_sys_getdefaultencoding_not_ascii = 1; + ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); + if (!ascii_chars_u) goto bad; + ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); + if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { + PyErr_Format( + PyExc_ValueError, + "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", + default_encoding_c); + goto bad; + } + Py_DECREF(ascii_chars_u); + Py_DECREF(ascii_chars_b); + } + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return -1; +} +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#else +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +static char* __PYX_DEFAULT_STRING_ENCODING; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; + strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + return -1; +} +#endif +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_d; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static PyObject *__pyx_empty_bytes; +static PyObject *__pyx_empty_unicode; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; + + +static const char *__pyx_f[] = { + "khmer/_oxli/hashing.pyx", + "stringsource", +}; + +/*--- Type declarations ---*/ +struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer; + +/* "khmer/_oxli/hashing.pxd":4 + * from _oxli cimport CpKmer, HashIntoType, WordLength + * + * cdef class Kmer: # <<<<<<<<<<<<<< + * cdef shared_ptr[CpKmer] _this + * cdef readonly str kmer + */ +struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer { + PyObject_HEAD + struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer *__pyx_vtab; + std::shared_ptr _this; + PyObject *kmer; +}; + + + +/* "khmer/_oxli/hashing.pyx":8 + * from _oxli cimport _revhash + * + * cdef class Kmer: # <<<<<<<<<<<<<< + * + * def __cinit__(self, str kmer=None): + */ + +struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer { + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*wrap)(khmer::Kmer *, khmer::WordLength); + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*create)(khmer::HashIntoType, khmer::WordLength); +}; +static struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer *__pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer; + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*SetupContext)(const char*, int, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; +#ifdef WITH_THREAD + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + } +#else + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) +#endif + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ + const char* function_name); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTest.proto */ +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact); + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* IncludeCppStringH.proto */ +#include + +/* decode_c_bytes.proto */ +static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( + const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)); + +/* decode_cpp_string.proto */ +static CYTHON_INLINE PyObject* __Pyx_decode_cpp_string( + std::string cppstring, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { + return __Pyx_decode_c_bytes( + cppstring.data(), cppstring.size(), start, stop, encoding, errors, decode_func); +} + +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyObject *dict, void *vtable); + +/* CodeObjectCache.proto */ +typedef struct { + PyCodeObject* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; +}; +static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static PyCodeObject *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* None.proto */ +#include + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE khmer::HashIntoType __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(void); + +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); + +static struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_f_5khmer_5_oxli_7hashing_4Kmer_wrap(khmer::Kmer *__pyx_v_cpkmer, khmer::WordLength __pyx_v_K); /* proto*/ +static struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_f_5khmer_5_oxli_7hashing_4Kmer_create(khmer::HashIntoType __pyx_v_tag, khmer::WordLength __pyx_v_K); /* proto*/ + +/* Module declarations from 'libcpp' */ + +/* Module declarations from 'libcpp.memory' */ + +/* Module declarations from 'libc.string' */ + +/* Module declarations from 'libcpp.string' */ + +/* Module declarations from 'libcpp.vector' */ + +/* Module declarations from 'libcpp.utility' */ + +/* Module declarations from 'libcpp.map' */ + +/* Module declarations from 'libcpp.set' */ + +/* Module declarations from 'libcpp.queue' */ + +/* Module declarations from 'libc.stdint' */ + +/* Module declarations from 'khmer._oxli._oxli' */ + +/* Module declarations from 'khmer._oxli.hashing' */ +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_7hashing_Kmer = 0; +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_string(std::string const &); /*proto*/ +#define __Pyx_MODULE_NAME "khmer._oxli.hashing" +int __pyx_module_is_main_khmer___oxli__hashing = 0; + +/* Implementation of 'khmer._oxli.hashing' */ +static const char __pyx_k_kmer[] = "kmer"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_utf_8[] = "utf-8"; +static const char __pyx_k_encode[] = "encode"; +static const char __pyx_k_kmer_u[] = "kmer_u"; +static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; +static PyObject *__pyx_n_s_encode; +static PyObject *__pyx_n_s_kmer; +static PyObject *__pyx_n_s_kmer_u; +static PyObject *__pyx_n_s_main; +static PyObject *__pyx_n_s_pyx_vtable; +static PyObject *__pyx_n_s_test; +static PyObject *__pyx_kp_s_utf_8; +static int __pyx_pf_5khmer_5_oxli_7hashing_4Kmer___cinit__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ +static Py_ssize_t __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_2__len__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4__str__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ +static Py_hash_t __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6__hash__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_f___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_r___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_u___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4kmer___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ +static PyObject *__pyx_tp_new_5khmer_5_oxli_7hashing_Kmer(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tuple_; + +/* "khmer/_oxli/hashing.pyx":10 + * cdef class Kmer: + * + * def __cinit__(self, str kmer=None): # <<<<<<<<<<<<<< + * self.kmer = kmer + * if self.kmer is not None: + */ + +/* Python wrapper */ +static int __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_kmer = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_kmer,0}; + PyObject* values[1] = {0}; + values[0] = ((PyObject*)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_kmer); + if (value) { values[0] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 10, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_kmer = ((PyObject*)values[0]); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 10, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_kmer), (&PyString_Type), 1, "kmer", 1))) __PYX_ERR(0, 10, __pyx_L1_error) + __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer___cinit__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self), __pyx_v_kmer); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5khmer_5_oxli_7hashing_4Kmer___cinit__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self, PyObject *__pyx_v_kmer) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + std::string __pyx_t_5; + Py_ssize_t __pyx_t_6; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "khmer/_oxli/hashing.pyx":11 + * + * def __cinit__(self, str kmer=None): + * self.kmer = kmer # <<<<<<<<<<<<<< + * if self.kmer is not None: + * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) + */ + __Pyx_INCREF(__pyx_v_kmer); + __Pyx_GIVEREF(__pyx_v_kmer); + __Pyx_GOTREF(__pyx_v_self->kmer); + __Pyx_DECREF(__pyx_v_self->kmer); + __pyx_v_self->kmer = __pyx_v_kmer; + + /* "khmer/_oxli/hashing.pyx":12 + * def __cinit__(self, str kmer=None): + * self.kmer = kmer + * if self.kmer is not None: # <<<<<<<<<<<<<< + * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) + * else: + */ + __pyx_t_1 = (__pyx_v_self->kmer != ((PyObject*)Py_None)); + __pyx_t_2 = (__pyx_t_1 != 0); + if (__pyx_t_2) { + + /* "khmer/_oxli/hashing.pyx":13 + * self.kmer = kmer + * if self.kmer is not None: + * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) # <<<<<<<<<<<<<< + * else: + * self._this.reset(new CpKmer()) + */ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_5 = __pyx_convert_string_from_py_std__in_string(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 13, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = PyObject_Length(__pyx_v_kmer); if (unlikely(__pyx_t_6 == -1)) __PYX_ERR(0, 13, __pyx_L1_error) + __pyx_v_self->_this.reset(new khmer::Kmer(__pyx_t_5, __pyx_t_6)); + + /* "khmer/_oxli/hashing.pyx":12 + * def __cinit__(self, str kmer=None): + * self.kmer = kmer + * if self.kmer is not None: # <<<<<<<<<<<<<< + * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) + * else: + */ + goto __pyx_L3; + } + + /* "khmer/_oxli/hashing.pyx":15 + * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) + * else: + * self._this.reset(new CpKmer()) # <<<<<<<<<<<<<< + * + * def __len__(self): + */ + /*else*/ { + __pyx_v_self->_this.reset(new khmer::Kmer()); + } + __pyx_L3:; + + /* "khmer/_oxli/hashing.pyx":10 + * cdef class Kmer: + * + * def __cinit__(self, str kmer=None): # <<<<<<<<<<<<<< + * self.kmer = kmer + * if self.kmer is not None: + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pyx":17 + * self._this.reset(new CpKmer()) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return len(self.kmer) + * + */ + +/* Python wrapper */ +static Py_ssize_t __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_3__len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_3__len__(PyObject *__pyx_v_self) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_2__len__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_2__len__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + Py_ssize_t __pyx_t_2; + __Pyx_RefNannySetupContext("__len__", 0); + + /* "khmer/_oxli/hashing.pyx":18 + * + * def __len__(self): + * return len(self.kmer) # <<<<<<<<<<<<<< + * + * def __str__(self): + */ + __pyx_t_1 = __pyx_v_self->kmer; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "khmer/_oxli/hashing.pyx":17 + * self._this.reset(new CpKmer()) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return len(self.kmer) + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.__len__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pyx":20 + * return len(self.kmer) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return self.kmer + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_5__str__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_5__str__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4__str__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4__str__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__", 0); + + /* "khmer/_oxli/hashing.pyx":21 + * + * def __str__(self): + * return self.kmer # <<<<<<<<<<<<<< + * + * def __hash__(self): + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->kmer); + __pyx_r = __pyx_v_self->kmer; + goto __pyx_L0; + + /* "khmer/_oxli/hashing.pyx":20 + * return len(self.kmer) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return self.kmer + * + */ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pyx":23 + * return self.kmer + * + * def __hash__(self): # <<<<<<<<<<<<<< + * return self.kmer_u + * + */ + +/* Python wrapper */ +static Py_hash_t __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_7__hash__(PyObject *__pyx_v_self); /*proto*/ +static Py_hash_t __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_7__hash__(PyObject *__pyx_v_self) { + Py_hash_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6__hash__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_hash_t __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6__hash__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { + Py_hash_t __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + Py_hash_t __pyx_t_2; + __Pyx_RefNannySetupContext("__hash__", 0); + + /* "khmer/_oxli/hashing.pyx":24 + * + * def __hash__(self): + * return self.kmer_u # <<<<<<<<<<<<<< + * + * @property + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_kmer_u); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 24, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_AsHash_t(__pyx_t_1); if (unlikely((__pyx_t_2 == (Py_hash_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 24, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "khmer/_oxli/hashing.pyx":23 + * return self.kmer + * + * def __hash__(self): # <<<<<<<<<<<<<< + * return self.kmer_u + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.__hash__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pyx":27 + * + * @property + * def kmer_f(self): # <<<<<<<<<<<<<< + * return deref(self._this).kmer_f + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_f_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_f_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_f___get__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_f___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli/hashing.pyx":28 + * @property + * def kmer_f(self): + * return deref(self._this).kmer_f # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_v_self->_this).kmer_f); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/hashing.pyx":27 + * + * @property + * def kmer_f(self): # <<<<<<<<<<<<<< + * return deref(self._this).kmer_f + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.kmer_f.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pyx":31 + * + * @property + * def kmer_r(self): # <<<<<<<<<<<<<< + * return deref(self._this).kmer_r + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_r_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_r_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_r___get__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_r___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli/hashing.pyx":32 + * @property + * def kmer_r(self): + * return deref(self._this).kmer_r # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_v_self->_this).kmer_r); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 32, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/hashing.pyx":31 + * + * @property + * def kmer_r(self): # <<<<<<<<<<<<<< + * return deref(self._this).kmer_r + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.kmer_r.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pyx":35 + * + * @property + * def kmer_u(self): # <<<<<<<<<<<<<< + * return deref(self._this).kmer_u + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_u_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_u_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_u___get__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_u___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli/hashing.pyx":36 + * @property + * def kmer_u(self): + * return deref(self._this).kmer_u # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_v_self->_this).kmer_u); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/hashing.pyx":35 + * + * @property + * def kmer_u(self): # <<<<<<<<<<<<<< + * return deref(self._this).kmer_u + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.kmer_u.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pyx":39 + * + * @staticmethod + * cdef Kmer wrap(CpKmer * cpkmer, WordLength K): # <<<<<<<<<<<<<< + * cdef Kmer kmer = Kmer() + * kmer._this.reset(cpkmer) + */ + +static struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_f_5khmer_5_oxli_7hashing_4Kmer_wrap(khmer::Kmer *__pyx_v_cpkmer, khmer::WordLength __pyx_v_K) { + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer = 0; + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + khmer::HashIntoType __pyx_t_2; + __Pyx_RefNannySetupContext("wrap", 0); + + /* "khmer/_oxli/hashing.pyx":40 + * @staticmethod + * cdef Kmer wrap(CpKmer * cpkmer, WordLength K): + * cdef Kmer kmer = Kmer() # <<<<<<<<<<<<<< + * kmer._this.reset(cpkmer) + * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') + */ + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_7hashing_Kmer), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_kmer = ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "khmer/_oxli/hashing.pyx":41 + * cdef Kmer wrap(CpKmer * cpkmer, WordLength K): + * cdef Kmer kmer = Kmer() + * kmer._this.reset(cpkmer) # <<<<<<<<<<<<<< + * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') + * return kmer + */ + __pyx_v_kmer->_this.reset(__pyx_v_cpkmer); + + /* "khmer/_oxli/hashing.pyx":42 + * cdef Kmer kmer = Kmer() + * kmer._this.reset(cpkmer) + * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') # <<<<<<<<<<<<<< + * return kmer + * + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_kmer), __pyx_n_s_kmer_u); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(__pyx_t_1); if (unlikely((__pyx_t_2 == ((khmer::HashIntoType)-1)) && PyErr_Occurred())) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_decode_cpp_string(khmer::_revhash(__pyx_t_2, __pyx_v_K), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(PyString_CheckExact(__pyx_t_1))||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_kmer->kmer); + __Pyx_DECREF(__pyx_v_kmer->kmer); + __pyx_v_kmer->kmer = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "khmer/_oxli/hashing.pyx":43 + * kmer._this.reset(cpkmer) + * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') + * return kmer # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __Pyx_XDECREF(((PyObject *)__pyx_r)); + __Pyx_INCREF(((PyObject *)__pyx_v_kmer)); + __pyx_r = __pyx_v_kmer; + goto __pyx_L0; + + /* "khmer/_oxli/hashing.pyx":39 + * + * @staticmethod + * cdef Kmer wrap(CpKmer * cpkmer, WordLength K): # <<<<<<<<<<<<<< + * cdef Kmer kmer = Kmer() + * kmer._this.reset(cpkmer) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.wrap", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_kmer); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pyx":46 + * + * @staticmethod + * cdef Kmer create(HashIntoType tag, WordLength K): # <<<<<<<<<<<<<< + * cdef Kmer kmer = Kmer() + * deref(kmer._this).set_from_unique_hash(tag, K) + */ + +static struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_f_5khmer_5_oxli_7hashing_4Kmer_create(khmer::HashIntoType __pyx_v_tag, khmer::WordLength __pyx_v_K) { + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer = 0; + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + khmer::HashIntoType __pyx_t_2; + __Pyx_RefNannySetupContext("create", 0); + + /* "khmer/_oxli/hashing.pyx":47 + * @staticmethod + * cdef Kmer create(HashIntoType tag, WordLength K): + * cdef Kmer kmer = Kmer() # <<<<<<<<<<<<<< + * deref(kmer._this).set_from_unique_hash(tag, K) + * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') + */ + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_7hashing_Kmer), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_kmer = ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "khmer/_oxli/hashing.pyx":48 + * cdef Kmer create(HashIntoType tag, WordLength K): + * cdef Kmer kmer = Kmer() + * deref(kmer._this).set_from_unique_hash(tag, K) # <<<<<<<<<<<<<< + * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') + * return kmer + */ + (*__pyx_v_kmer->_this).set_from_unique_hash(__pyx_v_tag, __pyx_v_K); + + /* "khmer/_oxli/hashing.pyx":49 + * cdef Kmer kmer = Kmer() + * deref(kmer._this).set_from_unique_hash(tag, K) + * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') # <<<<<<<<<<<<<< + * return kmer + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_kmer), __pyx_n_s_kmer_u); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(__pyx_t_1); if (unlikely((__pyx_t_2 == ((khmer::HashIntoType)-1)) && PyErr_Occurred())) __PYX_ERR(0, 49, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_decode_cpp_string(khmer::_revhash(__pyx_t_2, __pyx_v_K), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(PyString_CheckExact(__pyx_t_1))||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 49, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_kmer->kmer); + __Pyx_DECREF(__pyx_v_kmer->kmer); + __pyx_v_kmer->kmer = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "khmer/_oxli/hashing.pyx":50 + * deref(kmer._this).set_from_unique_hash(tag, K) + * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') + * return kmer # <<<<<<<<<<<<<< + */ + __Pyx_XDECREF(((PyObject *)__pyx_r)); + __Pyx_INCREF(((PyObject *)__pyx_v_kmer)); + __pyx_r = __pyx_v_kmer; + goto __pyx_L0; + + /* "khmer/_oxli/hashing.pyx":46 + * + * @staticmethod + * cdef Kmer create(HashIntoType tag, WordLength K): # <<<<<<<<<<<<<< + * cdef Kmer kmer = Kmer() + * deref(kmer._this).set_from_unique_hash(tag, K) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.create", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_kmer); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/hashing.pxd":6 + * cdef class Kmer: + * cdef shared_ptr[CpKmer] _this + * cdef readonly str kmer # <<<<<<<<<<<<<< + * + * @staticmethod + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_4kmer_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_4kmer_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4kmer___get__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4kmer___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->kmer); + __pyx_r = __pyx_v_self->kmer; + goto __pyx_L0; + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { + Py_ssize_t __pyx_v_length; + char *__pyx_v_data; + std::string __pyx_r; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); + + /* "string.from_py":15 + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< + * return string(data, length) + * + */ + __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(1, 15, __pyx_L1_error) + __pyx_v_data = __pyx_t_1; + + /* "string.from_py":16 + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * return string(data, length) # <<<<<<<<<<<<<< + * + * + */ + __pyx_r = std::string(__pyx_v_data, __pyx_v_length); + goto __pyx_L0; + + /* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":31 + * + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyObject_string_to_py_std__in_string", 0); + + /* "string.to_py":32 + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyUnicode_FromStringAndSize(char*, size_t) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 32, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":31 + * + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyObject_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":37 + * + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyUnicode_string_to_py_std__in_string", 0); + + /* "string.to_py":38 + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyStr_FromStringAndSize(char*, size_t) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyUnicode_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":37 + * + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyUnicode_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":43 + * + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyStr_string_to_py_std__in_string", 0); + + /* "string.to_py":44 + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyBytes_FromStringAndSize(char*, size_t) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyStr_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":43 + * + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyStr_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":49 + * + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyBytes_string_to_py_std__in_string", 0); + + /* "string.to_py":50 + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyByteArray_FromStringAndSize(char*, size_t) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":49 + * + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyBytes_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyByteArray_string_to_py_std__in_string", 0); + + /* "string.to_py":56 + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyByteArray_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyByteArray_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer __pyx_vtable_5khmer_5_oxli_7hashing_Kmer; + +static PyObject *__pyx_tp_new_5khmer_5_oxli_7hashing_Kmer(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *p; + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)o); + p->__pyx_vtab = __pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer; + new((void*)&(p->_this)) std::shared_ptr (); + p->kmer = ((PyObject*)Py_None); Py_INCREF(Py_None); + if (unlikely(__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_7hashing_Kmer(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *p = (struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)o; + #if PY_VERSION_HEX >= 0x030400a1 + if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + __Pyx_call_destructor(p->_this); + Py_CLEAR(p->kmer); + (*Py_TYPE(o)->tp_free)(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_f(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_f_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_r(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_r_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_u(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_u_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_4kmer_1__get__(o); +} + +static PyMethodDef __pyx_methods_5khmer_5_oxli_7hashing_Kmer[] = { + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_7hashing_Kmer[] = { + {(char *)"kmer_f", __pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_f, 0, (char *)0, 0}, + {(char *)"kmer_r", __pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_r, 0, (char *)0, 0}, + {(char *)"kmer_u", __pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_u, 0, (char *)0, 0}, + {(char *)"kmer", __pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer, 0, (char *)0, 0}, + {0, 0, 0, 0, 0} +}; + +static PySequenceMethods __pyx_tp_as_sequence_Kmer = { + __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_3__len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Kmer = { + __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_3__len__, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyTypeObject __pyx_type_5khmer_5_oxli_7hashing_Kmer = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.hashing.Kmer", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_7hashing_Kmer, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_Kmer, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Kmer, /*tp_as_mapping*/ + __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_7__hash__, /*tp_hash*/ + 0, /*tp_call*/ + __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_5__str__, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5khmer_5_oxli_7hashing_Kmer, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5khmer_5_oxli_7hashing_Kmer, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_7hashing_Kmer, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + #if PY_VERSION_HEX < 0x03020000 + { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, + #else + PyModuleDef_HEAD_INIT, + #endif + "hashing", + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, + {&__pyx_n_s_kmer, __pyx_k_kmer, sizeof(__pyx_k_kmer), 0, 0, 1, 1}, + {&__pyx_n_s_kmer_u, __pyx_k_kmer_u, sizeof(__pyx_k_kmer_u), 0, 0, 1, 1}, + {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, + {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, + {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, + {0, 0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + return 0; +} + +static int __Pyx_InitCachedConstants(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "khmer/_oxli/hashing.pyx":13 + * self.kmer = kmer + * if self.kmer is not None: + * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) # <<<<<<<<<<<<<< + * else: + * self._this.reset(new CpKmer()) + */ + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(0, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple_); + __Pyx_GIVEREF(__pyx_tuple_); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC inithashing(void); /*proto*/ +PyMODINIT_FUNC inithashing(void) +#else +PyMODINIT_FUNC PyInit_hashing(void); /*proto*/ +PyMODINIT_FUNC PyInit_hashing(void) +#endif +{ + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannyDeclarations + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_hashing(void)", 0); + if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_StopAsyncIteration_USED + if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4("hashing", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_d); + __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + #if CYTHON_COMPILING_IN_PYPY + Py_INCREF(__pyx_b); + #endif + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + if (__pyx_module_is_main_khmer___oxli__hashing) { + if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + } + #if PY_MAJOR_VERSION >= 3 + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "khmer._oxli.hashing")) { + if (unlikely(PyDict_SetItemString(modules, "khmer._oxli.hashing", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #endif + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global init code ---*/ + /*--- Variable export code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + __pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer = &__pyx_vtable_5khmer_5_oxli_7hashing_Kmer; + __pyx_vtable_5khmer_5_oxli_7hashing_Kmer.wrap = (struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*)(khmer::Kmer *, khmer::WordLength))__pyx_f_5khmer_5_oxli_7hashing_4Kmer_wrap; + __pyx_vtable_5khmer_5_oxli_7hashing_Kmer.create = (struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*)(khmer::HashIntoType, khmer::WordLength))__pyx_f_5khmer_5_oxli_7hashing_4Kmer_create; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_7hashing_Kmer) < 0) __PYX_ERR(0, 8, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_7hashing_Kmer.tp_print = 0; + if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_7hashing_Kmer.tp_dict, __pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer) < 0) __PYX_ERR(0, 8, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Kmer", (PyObject *)&__pyx_type_5khmer_5_oxli_7hashing_Kmer) < 0) __PYX_ERR(0, 8, __pyx_L1_error) + __pyx_ptype_5khmer_5_oxli_7hashing_Kmer = &__pyx_type_5khmer_5_oxli_7hashing_Kmer; + /*--- Type import code ---*/ + /*--- Variable import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + + /* "khmer/_oxli/hashing.pyx":1 + * from libcpp.string cimport string # <<<<<<<<<<<<<< + * from libcpp.memory cimport make_shared + * from libc.stdint cimport uint64_t + */ + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + if (__pyx_m) { + if (__pyx_d) { + __Pyx_AddTraceback("init khmer._oxli.hashing", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + Py_DECREF(__pyx_m); __pyx_m = 0; + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init khmer._oxli.hashing"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* RaiseDoubleKeywords */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AsString(kw_name)); + #endif +} + +/* ParseKeywords */ +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + continue; + } + name = first_kw_arg; + #if PY_MAJOR_VERSION < 3 + if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + while (*name) { + if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) + && _PyString_Eq(**name, key)) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + if ((**argname == key) || ( + (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) + && _PyString_Eq(**argname, key))) { + goto arg_passed_twice; + } + argname++; + } + } + } else + #endif + if (likely(PyUnicode_Check(key))) { + while (*name) { + int cmp = (**name == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**name, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + int cmp = (**argname == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**argname, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + argname++; + } + } + } else + goto invalid_keyword_type; + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTest */ +static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); +} +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact) +{ + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (none_allowed && obj == Py_None) return 1; + else if (exact) { + if (likely(Py_TYPE(obj) == type)) return 1; + #if PY_MAJOR_VERSION == 2 + else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + #endif + } + else { + if (likely(PyObject_TypeCheck(obj, type))) return 1; + } + __Pyx_RaiseArgumentTypeInvalid(name, obj, type); + return 0; +} + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* decode_c_bytes */ +static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( + const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { + if (unlikely((start < 0) | (stop < 0))) { + if (start < 0) { + start += length; + if (start < 0) + start = 0; + } + if (stop < 0) + stop += length; + } + if (stop > length) + stop = length; + length = stop - start; + if (unlikely(length <= 0)) + return PyUnicode_FromUnicode(NULL, 0); + cstring += start; + if (decode_func) { + return decode_func(cstring, length, errors); + } else { + return PyUnicode_Decode(cstring, length, encoding, errors); + } +} + +/* SetVTable */ +static int __Pyx_SetVtable(PyObject *dict, void *vtable) { +#if PY_VERSION_HEX >= 0x02070000 + PyObject *ob = PyCapsule_New(vtable, 0, 0); +#else + PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); +#endif + if (!ob) + goto bad; + if (PyDict_SetItem(dict, __pyx_n_s_pyx_vtable, ob) < 0) + goto bad; + Py_DECREF(ob); + return 0; +bad: + Py_XDECREF(ob); + return -1; +} + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, + 0, + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + py_code = __pyx_find_code_object(c_line ? c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + } + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value) { + const khmer::HashIntoType neg_one = (khmer::HashIntoType) -1, const_zero = (khmer::HashIntoType) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(khmer::HashIntoType) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(khmer::HashIntoType) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::HashIntoType) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(khmer::HashIntoType), + little, !is_unsigned); + } +} + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE khmer::HashIntoType __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(PyObject *x) { + const khmer::HashIntoType neg_one = (khmer::HashIntoType) -1, const_zero = (khmer::HashIntoType) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(khmer::HashIntoType) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (khmer::HashIntoType) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (khmer::HashIntoType) 0; + case 1: __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, digit, digits[0]) + case 2: + if (8 * sizeof(khmer::HashIntoType) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) >= 2 * PyLong_SHIFT) { + return (khmer::HashIntoType) (((((khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(khmer::HashIntoType) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) >= 3 * PyLong_SHIFT) { + return (khmer::HashIntoType) (((((((khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(khmer::HashIntoType) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) >= 4 * PyLong_SHIFT) { + return (khmer::HashIntoType) (((((((((khmer::HashIntoType)digits[3]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (khmer::HashIntoType) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(khmer::HashIntoType) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(khmer::HashIntoType, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(khmer::HashIntoType, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (khmer::HashIntoType) 0; + case -1: __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, digit, +digits[0]) + case -2: + if (8 * sizeof(khmer::HashIntoType) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) - 1 > 2 * PyLong_SHIFT) { + return (khmer::HashIntoType) (((khmer::HashIntoType)-1)*(((((khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(khmer::HashIntoType) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) - 1 > 2 * PyLong_SHIFT) { + return (khmer::HashIntoType) ((((((khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(khmer::HashIntoType) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) - 1 > 3 * PyLong_SHIFT) { + return (khmer::HashIntoType) (((khmer::HashIntoType)-1)*(((((((khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(khmer::HashIntoType) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) - 1 > 3 * PyLong_SHIFT) { + return (khmer::HashIntoType) ((((((((khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(khmer::HashIntoType) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) - 1 > 4 * PyLong_SHIFT) { + return (khmer::HashIntoType) (((khmer::HashIntoType)-1)*(((((((((khmer::HashIntoType)digits[3]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(khmer::HashIntoType) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(khmer::HashIntoType) - 1 > 4 * PyLong_SHIFT) { + return (khmer::HashIntoType) ((((((((((khmer::HashIntoType)digits[3]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); + } + } + break; + } +#endif + if (sizeof(khmer::HashIntoType) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(khmer::HashIntoType, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::HashIntoType) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(khmer::HashIntoType, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + khmer::HashIntoType val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (khmer::HashIntoType) -1; + } + } else { + khmer::HashIntoType val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (khmer::HashIntoType) -1; + val = __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to khmer::HashIntoType"); + return (khmer::HashIntoType) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to khmer::HashIntoType"); + return (khmer::HashIntoType) -1; +} + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case -2: + if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (long) -1; + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (long) -1; + val = __Pyx_PyInt_As_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case -2: + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int) -1; + } + } else { + int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(void) { + char ctversion[4], rtversion[4]; + PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); + PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); + if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compiletime version %s of module '%.100s' " + "does not match runtime version %s", + ctversion, __Pyx_MODULE_NAME, rtversion); + return PyErr_WarnEx(NULL, message, 1); + } + return 0; +} + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else + if (t->is_unicode | t->is_str) { + if (t->intern) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); + } else { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { +#if PY_VERSION_HEX < 0x03030000 + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; + } + } + } +#endif + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +#else + if (__Pyx_PyUnicode_READY(o) == -1) return NULL; +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (PyUnicode_IS_ASCII(o)) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif + } else +#endif +#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) + if (PyByteArray_Check(o)) { + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); + } else +#endif + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + const char *name = NULL; + PyObject *res = NULL; +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + #if PY_MAJOR_VERSION < 3 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } + #else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } + #endif +#else + res = PyNumber_Int(x); +#endif + if (res) { +#if PY_MAJOR_VERSION < 3 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(b))) { + if (sizeof(Py_ssize_t) >= sizeof(long)) + return PyInt_AS_LONG(b); + else + return PyInt_AsSsize_t(x); + } +#endif + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)b)->ob_digit; + const Py_ssize_t size = Py_SIZE(b); + if (likely(__Pyx_sst_abs(size) <= 1)) { + ival = likely(size) ? digits[0] : 0; + if (size == -1) ival = -ival; + return ival; + } else { + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { + return PyInt_FromSize_t(ival); +} + + +#endif /* Py_PYTHON_H */ diff --git a/khmer/_oxli/partitioning.cpp b/khmer/_oxli/partitioning.cpp new file mode 100644 index 0000000000..7de9d62c58 --- /dev/null +++ b/khmer/_oxli/partitioning.cpp @@ -0,0 +1,7995 @@ +/* Generated by Cython 0.25.1 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) + #error Cython requires Python 2.6+ or Python 3.2+. +#else +#define CYTHON_ABI "0_25_1" +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#ifndef HAVE_LONG_LONG + #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) + #define HAVE_LONG_LONG + #endif +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#ifdef PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#elif defined(PYSTON_VERSION) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #if PY_MAJOR_VERSION < 3 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #ifndef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if PY_VERSION_HEX < 0x030300F0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif +#endif +#if !defined(CYTHON_FAST_PYCCALL) +#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #include "longintrepr.h" + #undef SHIFT + #undef BASE + #undef MASK +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) + #define Py_OptimizeFlag 0 +#endif +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyClass_Type +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #define __Pyx_PyCFunctionFast _PyCFunctionFast +#endif +#if CYTHON_FAST_PYCCALL +#define __Pyx_PyFastCFunction_Check(func)\ + ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) +#else +#define __Pyx_PyFastCFunction_Check(func) 0 +#endif +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) +#else + #define CYTHON_PEP393_ENABLED 0 + #define PyUnicode_1BYTE_KIND 1 + #define PyUnicode_2BYTE_KIND 2 + #define PyUnicode_4BYTE_KIND 4 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) + #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_PYSTON + #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) +#else + #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) +#endif +#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) + #define PyObject_ASCII(o) PyObject_Repr(o) +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyStringObject PyUnicodeObject + #define PyString_Type PyUnicode_Type + #define PyString_Check PyUnicode_Check + #define PyString_CheckExact PyUnicode_CheckExact +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) + #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) +#else + #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) + #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#endif +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#if PY_MAJOR_VERSION >= 3 + #define PyIntObject PyLongObject + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define PyNumber_Int PyNumber_Long +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBoolObject PyLongObject +#endif +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY + #ifndef PyUnicode_InternFromString + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) + #endif +#endif +#if PY_VERSION_HEX < 0x030200A4 + typedef long Py_hash_t; + #define __Pyx_PyInt_FromHash_t PyInt_FromLong + #define __Pyx_PyInt_AsHash_t PyInt_AsLong +#else + #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t + #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) +#else + #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) +#endif +#if CYTHON_USE_ASYNC_SLOTS + #if PY_VERSION_HEX >= 0x030500B1 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods + #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) + #else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) + +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #define CYTHON_INLINE inline +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(U other) { return *ptr == other; }; + template bool operator !=(U other) { return *ptr != other; }; + private: + T *ptr; +}; + +#if defined(WIN32) || defined(MS_WINDOWS) + #define _USE_MATH_DEFINES +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + + +#define __PYX_ERR(f_index, lineno, Ln_error) \ +{ \ + __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ +} + +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#define __PYX_HAVE__khmer___oxli__partitioning +#define __PYX_HAVE_API__khmer___oxli__partitioning +#include +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include +#include +#include +#include +#include +#include +#include +#include "khmer.hh" +#include "kmer_hash.hh" +#include "_khmer.hh" +#include "hashtable.hh" +#include "traversal.hh" +#include "partitioning.hh" +#include +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#ifdef PYREX_WITHOUT_ASSERTIONS +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; + const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) && defined (_M_X64) + #define __Pyx_sst_abs(value) _abs64(value) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#else + #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize +#endif +#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#if PY_MAJOR_VERSION < 3 +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ + const Py_UNICODE *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen +#endif +#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) +#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) +#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) +#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +#if CYTHON_ASSUME_SAFE_MACROS +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#else +#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#endif +#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#else +#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) +#endif +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII +static int __Pyx_sys_getdefaultencoding_not_ascii; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + PyObject* ascii_chars_u = NULL; + PyObject* ascii_chars_b = NULL; + const char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + if (strcmp(default_encoding_c, "ascii") == 0) { + __Pyx_sys_getdefaultencoding_not_ascii = 0; + } else { + char ascii_chars[128]; + int c; + for (c = 0; c < 128; c++) { + ascii_chars[c] = c; + } + __Pyx_sys_getdefaultencoding_not_ascii = 1; + ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); + if (!ascii_chars_u) goto bad; + ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); + if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { + PyErr_Format( + PyExc_ValueError, + "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", + default_encoding_c); + goto bad; + } + Py_DECREF(ascii_chars_u); + Py_DECREF(ascii_chars_b); + } + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return -1; +} +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#else +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +static char* __PYX_DEFAULT_STRING_ENCODING; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; + strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + return -1; +} +#endif +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_d; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static PyObject *__pyx_empty_bytes; +static PyObject *__pyx_empty_unicode; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; + + +static const char *__pyx_f[] = { + "khmer/_oxli/partitioning.pyx", + "khmer/_oxli/partitioning.pxd", + "stringsource", +}; + +/*--- Type declarations ---*/ +struct __pyx_obj_5khmer_5_oxli_12partitioning_Component; +struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner; +struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__; +struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components; +struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components; + +/* "khmer/_oxli/partitioning.pxd":8 + * from _oxli cimport CpHashtable, CpStreamingPartitioner, BoundedCounterType + * + * cdef class Component: # <<<<<<<<<<<<<< + * cdef ComponentPtr _this + * + */ +struct __pyx_obj_5khmer_5_oxli_12partitioning_Component { + PyObject_HEAD + struct __pyx_vtabstruct_5khmer_5_oxli_12partitioning_Component *__pyx_vtab; + khmer::ComponentPtr _this; +}; + + +/* "khmer/_oxli/partitioning.pxd":21 + * + * + * cdef class StreamingPartitioner: # <<<<<<<<<<<<<< + * cdef unique_ptr[CpStreamingPartitioner] _this + * cdef weak_ptr[ComponentPtrSet] _components + */ +struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner { + PyObject_HEAD + std::unique_ptr _this; + std::weak_ptr _components; + std::weak_ptr _tag_component_map; + khmer::Hashtable *_graph_ptr; + uint64_t n_consumed; +}; + + +/* "khmer/_oxli/partitioning.pyx":44 + * return deref(self._this).get_n_tags() + * + * def __iter__(self): # <<<<<<<<<<<<<< + * cdef HashIntoType tag + * for tag in deref(self._this).tags: + */ +struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ { + PyObject_HEAD + struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self; + khmer::HashIntoType __pyx_v_tag; + std::set ::iterator __pyx_t_0; + std::set *__pyx_t_1; +}; + + +/* "khmer/_oxli/partitioning.pyx":128 + * return Component.create(compptr) + * + * def components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[ComponentPtrSet] locked + * cdef ComponentPtr cmpptr + */ +struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components { + PyObject_HEAD + khmer::ComponentPtr __pyx_v_cmpptr; + std::shared_ptr __pyx_v_locked; + std::shared_ptr __pyx_v_lockedptr; + struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self; + std::set ::iterator __pyx_t_0; + khmer::ComponentPtrSet *__pyx_t_1; +}; + + +/* "khmer/_oxli/partitioning.pyx":138 + * raise MemoryError("Can't locked underlying Component set") + * + * def tag_components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[CpGuardedKmerCompMap] locked + * cdef pair[HashIntoType,ComponentPtr] cpair + */ +struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components { + PyObject_HEAD + std::pair __pyx_v_cpair; + std::shared_ptr __pyx_v_locked; + std::shared_ptr __pyx_v_lockedptr; + struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self; + std::map ::iterator __pyx_t_0; + std::map *__pyx_t_1; +}; + + + +/* "khmer/_oxli/partitioning.pyx":23 + * from .._khmer import Nodegraph + * + * cdef class Component: # <<<<<<<<<<<<<< + * + * def __cinit__(self, Component other=None): + */ + +struct __pyx_vtabstruct_5khmer_5_oxli_12partitioning_Component { + struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *(*create)(khmer::ComponentPtr); + std::vector (*_tag_counts)(khmer::ComponentPtr, khmer::Hashtable *); + float (*_mean_tag_count)(khmer::ComponentPtr, khmer::Hashtable *); +}; +static struct __pyx_vtabstruct_5khmer_5_oxli_12partitioning_Component *__pyx_vtabptr_5khmer_5_oxli_12partitioning_Component; + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*SetupContext)(const char*, int, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; +#ifdef WITH_THREAD + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + } +#else + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) +#endif + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ + const char* function_name); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTest.proto */ +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact); + +/* PyIntBinop.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); +#else +#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ + PyObject_RichCompare(op1, op2, Py_EQ) + #endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* RaiseTooManyValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* RaiseNeedMoreValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* IterFinish.proto */ +static CYTHON_INLINE int __Pyx_IterFinish(void); + +/* UnpackItemEndCheck.proto */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); + +/* WriteUnraisableException.proto */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil); + +/* GetModuleGlobalName.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); + +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + PyList_SET_ITEM(list, len, x); + Py_SIZE(list) = len+1; + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +/* IncludeStringH.proto */ +#include + +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyObject *dict, void *vtable); + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* GetNameInClass.proto */ +static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name); + +/* CodeObjectCache.proto */ +typedef struct { + PyCodeObject* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; +}; +static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static PyCodeObject *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* None.proto */ +#include + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value); + +/* CppExceptionConversion.proto */ +#ifndef __Pyx_CppExn2PyErr +#include +#include +#include +#include +static void __Pyx_CppExn2PyErr() { + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one + else + throw; + } catch (const std::bad_alloc& exn) { + PyErr_SetString(PyExc_MemoryError, exn.what()); + } catch (const std::bad_cast& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::bad_typeid& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::domain_error& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::invalid_argument& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::ios_base::failure& exn) { + PyErr_SetString(PyExc_IOError, exn.what()); + } catch (const std::out_of_range& exn) { + PyErr_SetString(PyExc_IndexError, exn.what()); + } catch (const std::overflow_error& exn) { + PyErr_SetString(PyExc_OverflowError, exn.what()); + } catch (const std::range_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::underflow_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } + catch (...) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType(khmer::BoundedCounterType value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectCallMethod1.proto */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* CoroutineBase.proto */ +typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); +typedef struct { + PyObject_HEAD + __pyx_coroutine_body_t body; + PyObject *closure; + PyObject *exc_type; + PyObject *exc_value; + PyObject *exc_traceback; + PyObject *gi_weakreflist; + PyObject *classobj; + PyObject *yieldfrom; + PyObject *gi_name; + PyObject *gi_qualname; + PyObject *gi_modulename; + int resume_label; + char is_running; +} __pyx_CoroutineObject; +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); +static int __Pyx_Coroutine_clear(PyObject *self); +#if 1 || PY_VERSION_HEX < 0x030300B0 +static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); +#else +#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) +#endif + +/* PatchModuleWithCoroutine.proto */ +static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code); + +/* PatchGeneratorABC.proto */ +static int __Pyx_patch_abc(void); + +/* Generator.proto */ +#define __Pyx_Generator_USED +static PyTypeObject *__pyx_GeneratorType = 0; +#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) +#define __Pyx_Generator_New(body, closure, name, qualname, module_name)\ + __Pyx__Coroutine_New(__pyx_GeneratorType, body, closure, name, qualname, module_name) +static PyObject *__Pyx_Generator_Next(PyObject *self); +static int __pyx_Generator_init(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(void); + +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); + +static struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(khmer::ComponentPtr __pyx_v_ptr); /* proto*/ +static std::vector __pyx_f_5khmer_5_oxli_12partitioning_9Component__tag_counts(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph); /* proto*/ +static float __pyx_f_5khmer_5_oxli_12partitioning_9Component__mean_tag_count(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph); /* proto*/ + +/* Module declarations from 'libcpp' */ + +/* Module declarations from 'libcpp.memory' */ + +/* Module declarations from 'libcpp.vector' */ + +/* Module declarations from 'libc.stdint' */ + +/* Module declarations from 'libc.string' */ + +/* Module declarations from 'libcpp.string' */ + +/* Module declarations from 'libcpp.utility' */ + +/* Module declarations from 'libcpp.map' */ + +/* Module declarations from 'libcpp.set' */ + +/* Module declarations from 'libcpp.queue' */ + +/* Module declarations from 'khmer._oxli._oxli' */ + +/* Module declarations from 'cython' */ + +/* Module declarations from 'libc.limits' */ + +/* Module declarations from 'libc.stdio' */ + +/* Module declarations from 'khmer._oxli.partitioning' */ +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning_Component = 0; +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning_StreamingPartitioner = 0; +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ = 0; +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components = 0; +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components = 0; +static PyObject *__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(const std::vector &); /*proto*/ +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ +#define __Pyx_MODULE_NAME "khmer._oxli.partitioning" +int __pyx_module_is_main_khmer___oxli__partitioning = 0; + +/* Implementation of 'khmer._oxli.partitioning' */ +static PyObject *__pyx_builtin_staticmethod; +static PyObject *__pyx_builtin_NotImplementedError; +static PyObject *__pyx_builtin_ValueError; +static PyObject *__pyx_builtin_MemoryError; +static PyObject *__pyx_builtin_IOError; +static PyObject *__pyx_builtin_range; +static const char __pyx_k_args[] = "args"; +static const char __pyx_k_iter[] = "__iter__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_send[] = "send"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_close[] = "close"; +static const char __pyx_k_graph[] = "graph"; +static const char __pyx_k_khmer[] = "_khmer"; +static const char __pyx_k_other[] = "other"; +static const char __pyx_k_range[] = "range"; +static const char __pyx_k_throw[] = "throw"; +static const char __pyx_k_utf_8[] = "utf-8"; +static const char __pyx_k_encode[] = "encode"; +static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_IOError[] = "IOError"; +static const char __pyx_k_Nodegraph[] = "Nodegraph"; +static const char __pyx_k_component[] = "component"; +static const char __pyx_k_graph_ptr[] = "graph_ptr"; +static const char __pyx_k_Countgraph[] = "Countgraph"; +static const char __pyx_k_ValueError[] = "ValueError"; +static const char __pyx_k_components[] = "components"; +static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; +static const char __pyx_k_tag_counts[] = "tag_counts"; +static const char __pyx_k_MemoryError[] = "MemoryError"; +static const char __pyx_k_component_id[] = "component_id"; +static const char __pyx_k_staticmethod[] = "staticmethod"; +static const char __pyx_k_tag_components[] = "tag_components"; +static const char __pyx_k_Can_t_open_file[] = "Can't open file."; +static const char __pyx_k_Component___iter[] = "Component.__iter__"; +static const char __pyx_k_NotImplementedError[] = "NotImplementedError"; +static const char __pyx_k_Operator_not_available[] = "Operator not available."; +static const char __pyx_k_khmer__oxli_partitioning[] = "khmer._oxli.partitioning"; +static const char __pyx_k_StreamingPartitioner_components[] = "StreamingPartitioner.components"; +static const char __pyx_k_Users_camille_w_khmer_khmer__ox[] = "/Users/camille/w/khmer/khmer/_oxli/partitioning.pyx"; +static const char __pyx_k_Can_t_locked_underlying_Componen[] = "Can't locked underlying Component set"; +static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; +static const char __pyx_k_StreamingPartitioner_tag_compone[] = "StreamingPartitioner.tag_components"; +static PyObject *__pyx_kp_s_Can_t_locked_underlying_Componen; +static PyObject *__pyx_kp_s_Can_t_open_file; +static PyObject *__pyx_n_s_Component___iter; +static PyObject *__pyx_n_s_Countgraph; +static PyObject *__pyx_n_s_IOError; +static PyObject *__pyx_n_s_MemoryError; +static PyObject *__pyx_kp_s_Must_take_an_object_with_Hashtab; +static PyObject *__pyx_n_s_Nodegraph; +static PyObject *__pyx_n_s_NotImplementedError; +static PyObject *__pyx_kp_s_Operator_not_available; +static PyObject *__pyx_n_s_StreamingPartitioner_components; +static PyObject *__pyx_n_s_StreamingPartitioner_tag_compone; +static PyObject *__pyx_kp_s_Users_camille_w_khmer_khmer__ox; +static PyObject *__pyx_n_s_ValueError; +static PyObject *__pyx_n_s_args; +static PyObject *__pyx_n_s_close; +static PyObject *__pyx_n_s_component; +static PyObject *__pyx_n_s_component_id; +static PyObject *__pyx_n_s_components; +static PyObject *__pyx_n_s_encode; +static PyObject *__pyx_n_s_graph; +static PyObject *__pyx_n_s_graph_ptr; +static PyObject *__pyx_n_s_import; +static PyObject *__pyx_n_s_iter; +static PyObject *__pyx_n_s_khmer; +static PyObject *__pyx_n_s_khmer__oxli_partitioning; +static PyObject *__pyx_n_s_main; +static PyObject *__pyx_n_s_other; +static PyObject *__pyx_n_s_pyx_vtable; +static PyObject *__pyx_n_s_range; +static PyObject *__pyx_n_s_send; +static PyObject *__pyx_n_s_staticmethod; +static PyObject *__pyx_n_s_tag_components; +static PyObject *__pyx_n_s_tag_counts; +static PyObject *__pyx_n_s_test; +static PyObject *__pyx_n_s_throw; +static PyObject *__pyx_kp_s_utf_8; +static int __pyx_pf_5khmer_5_oxli_12partitioning_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_other); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_pf_5khmer_5_oxli_12partitioning_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ +static Py_hash_t __pyx_pf_5khmer_5_oxli_12partitioning_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_11tag_counts(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_component, PyObject *__pyx_v_graph); /* proto */ +static int __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_4consume_fasta(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6get_tag_component(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_8get_nearest_component(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_13tag_components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_16write_components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning_Component(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_int_2; +static PyObject *__pyx_tuple_; +static PyObject *__pyx_tuple__2; +static PyObject *__pyx_tuple__3; +static PyObject *__pyx_tuple__4; +static PyObject *__pyx_tuple__5; +static PyObject *__pyx_tuple__6; +static PyObject *__pyx_tuple__7; +static PyObject *__pyx_tuple__8; +static PyObject *__pyx_tuple__9; +static PyObject *__pyx_tuple__10; +static PyObject *__pyx_tuple__11; +static PyObject *__pyx_codeobj__12; + +/* "khmer/_oxli/partitioning.pyx":25 + * cdef class Component: + * + * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< + * if other is not None: + * self._this.reset(other._this.get()) + */ + +/* Python wrapper */ +static int __pyx_pw_5khmer_5_oxli_12partitioning_9Component_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5khmer_5_oxli_12partitioning_9Component_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_other = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_other,0}; + PyObject* values[1] = {0}; + values[0] = (PyObject *)((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_other); + if (value) { values[0] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 25, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_other = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)values[0]); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 25, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.partitioning.Component.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5khmer_5_oxli_12partitioning_Component, 1, "other", 0))) __PYX_ERR(0, 25, __pyx_L1_error) + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component___cinit__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self), __pyx_v_other); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5khmer_5_oxli_12partitioning_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_other) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "khmer/_oxli/partitioning.pyx":26 + * + * def __cinit__(self, Component other=None): + * if other is not None: # <<<<<<<<<<<<<< + * self._this.reset(other._this.get()) + * + */ + __pyx_t_1 = (((PyObject *)__pyx_v_other) != Py_None); + __pyx_t_2 = (__pyx_t_1 != 0); + if (__pyx_t_2) { + + /* "khmer/_oxli/partitioning.pyx":27 + * def __cinit__(self, Component other=None): + * if other is not None: + * self._this.reset(other._this.get()) # <<<<<<<<<<<<<< + * + * property component_id: + */ + __pyx_v_self->_this.reset(__pyx_v_other->_this.get()); + + /* "khmer/_oxli/partitioning.pyx":26 + * + * def __cinit__(self, Component other=None): + * if other is not None: # <<<<<<<<<<<<<< + * self._this.reset(other._this.get()) + * + */ + } + + /* "khmer/_oxli/partitioning.pyx":25 + * cdef class Component: + * + * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< + * if other is not None: + * self._this.reset(other._this.get()) + */ + + /* function exit code */ + __pyx_r = 0; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":30 + * + * property component_id: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).component_id + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12component_id_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12component_id_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_12component_id___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli/partitioning.pyx":31 + * property component_id: + * def __get__(self): + * return deref(self._this).component_id # <<<<<<<<<<<<<< + * + * property _n_created: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 31, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":30 + * + * property component_id: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).component_id + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.partitioning.Component.component_id.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":34 + * + * property _n_created: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_created() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_10_n_created_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_10_n_created_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_10_n_created___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli/partitioning.pyx":35 + * property _n_created: + * def __get__(self): + * return deref(self._this).get_n_created() # <<<<<<<<<<<<<< + * + * property _n_destroyed: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_created()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 35, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":34 + * + * property _n_created: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_created() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.partitioning.Component._n_created.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":38 + * + * property _n_destroyed: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_destroyed() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli/partitioning.pyx":39 + * property _n_destroyed: + * def __get__(self): + * return deref(self._this).get_n_destroyed() # <<<<<<<<<<<<<< + * + * def __len__(self): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_destroyed()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 39, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":38 + * + * property _n_destroyed: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_destroyed() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.partitioning.Component._n_destroyed.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":41 + * return deref(self._this).get_n_destroyed() + * + * def __len__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_tags() + * + */ + +/* Python wrapper */ +static Py_ssize_t __pyx_pw_5khmer_5_oxli_12partitioning_9Component_3__len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_pw_5khmer_5_oxli_12partitioning_9Component_3__len__(PyObject *__pyx_v_self) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_2__len__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_pf_5khmer_5_oxli_12partitioning_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__", 0); + + /* "khmer/_oxli/partitioning.pyx":42 + * + * def __len__(self): + * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< + * + * def __iter__(self): + */ + __pyx_r = (*__pyx_v_self->_this).get_n_tags(); + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":41 + * return deref(self._this).get_n_destroyed() + * + * def __len__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_tags() + * + */ + + /* function exit code */ + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ + +/* "khmer/_oxli/partitioning.pyx":44 + * return deref(self._this).get_n_tags() + * + * def __iter__(self): # <<<<<<<<<<<<<< + * cdef HashIntoType tag + * for tag in deref(self._this).tags: + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_5__iter__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_5__iter__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_4__iter__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__iter__", 0); + __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 44, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_12partitioning_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli_partitioning); if (unlikely(!gen)) __PYX_ERR(0, 44, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("khmer._oxli.partitioning.Component.__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + std::set ::iterator __pyx_t_1; + std::set *__pyx_t_2; + khmer::HashIntoType __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("None", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L6_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 44, __pyx_L1_error) + + /* "khmer/_oxli/partitioning.pyx":46 + * def __iter__(self): + * cdef HashIntoType tag + * for tag in deref(self._this).tags: # <<<<<<<<<<<<<< + * yield tag + * + */ + __pyx_t_2 = &(*__pyx_cur_scope->__pyx_v_self->_this).tags; + __pyx_t_1 = __pyx_t_2->begin(); + for (;;) { + if (!(__pyx_t_1 != __pyx_t_2->end())) break; + __pyx_t_3 = *__pyx_t_1; + ++__pyx_t_1; + __pyx_cur_scope->__pyx_v_tag = __pyx_t_3; + + /* "khmer/_oxli/partitioning.pyx":47 + * cdef HashIntoType tag + * for tag in deref(self._this).tags: + * yield tag # <<<<<<<<<<<<<< + * + * def __hash__(self): + */ + __pyx_t_4 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_cur_scope->__pyx_v_tag); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_2; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L6_resume_from_yield:; + __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; + __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 47, __pyx_L1_error) + + /* "khmer/_oxli/partitioning.pyx":46 + * def __iter__(self): + * cdef HashIntoType tag + * for tag in deref(self._this).tags: # <<<<<<<<<<<<<< + * yield tag + * + */ + } + if (1); else __pyx_cur_scope = __pyx_cur_scope; + + /* "khmer/_oxli/partitioning.pyx":44 + * return deref(self._this).get_n_tags() + * + * def __iter__(self): # <<<<<<<<<<<<<< + * cdef HashIntoType tag + * for tag in deref(self._this).tags: + */ + + /* function exit code */ + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); __pyx_r = 0; + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":49 + * yield tag + * + * def __hash__(self): # <<<<<<<<<<<<<< + * return self._this.get() + * + */ + +/* Python wrapper */ +static Py_hash_t __pyx_pw_5khmer_5_oxli_12partitioning_9Component_8__hash__(PyObject *__pyx_v_self); /*proto*/ +static Py_hash_t __pyx_pw_5khmer_5_oxli_12partitioning_9Component_8__hash__(PyObject *__pyx_v_self) { + Py_hash_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_7__hash__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_hash_t __pyx_pf_5khmer_5_oxli_12partitioning_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { + Py_hash_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__hash__", 0); + + /* "khmer/_oxli/partitioning.pyx":50 + * + * def __hash__(self): + * return self._this.get() # <<<<<<<<<<<<<< + * + * def __richcmp__(x, y, op): + */ + __pyx_r = ((uintptr_t)__pyx_v_self->_this.get()); + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":49 + * yield tag + * + * def __hash__(self): # <<<<<<<<<<<<<< + * return self._this.get() + * + */ + + /* function exit code */ + __pyx_L0:; + if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":52 + * return self._this.get() + * + * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< + * if op == 2: + * return x.component_id == y.component_id + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_10__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_arg_op); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_10__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_arg_op) { + PyObject *__pyx_v_op = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); + __pyx_v_op = __Pyx_PyInt_From_int(__pyx_arg_op); if (unlikely(!__pyx_v_op)) __PYX_ERR(0, 52, __pyx_L3_error) + __Pyx_GOTREF(__pyx_v_op); + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.partitioning.Component.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_9__richcmp__(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), ((PyObject *)__pyx_v_op)); + + /* function exit code */ + __Pyx_XDECREF(__pyx_v_op); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannySetupContext("__richcmp__", 0); + + /* "khmer/_oxli/partitioning.pyx":53 + * + * def __richcmp__(x, y, op): + * if op == 2: # <<<<<<<<<<<<<< + * return x.component_id == y.component_id + * else: + */ + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_v_op, __pyx_int_2, 2, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "khmer/_oxli/partitioning.pyx":54 + * def __richcmp__(x, y, op): + * if op == 2: + * return x.component_id == y.component_id # <<<<<<<<<<<<<< + * else: + * raise NotImplementedError('Operator not available.') + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_x, __pyx_n_s_component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 54, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_y, __pyx_n_s_component_id); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 54, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyObject_RichCompare(__pyx_t_1, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 54, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":53 + * + * def __richcmp__(x, y, op): + * if op == 2: # <<<<<<<<<<<<<< + * return x.component_id == y.component_id + * else: + */ + } + + /* "khmer/_oxli/partitioning.pyx":56 + * return x.component_id == y.component_id + * else: + * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< + * + * @staticmethod + */ + /*else*/ { + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_NotImplementedError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 56, __pyx_L1_error) + } + + /* "khmer/_oxli/partitioning.pyx":52 + * return self._this.get() + * + * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< + * if op == 2: + * return x.component_id == y.component_id + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("khmer._oxli.partitioning.Component.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":59 + * + * @staticmethod + * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< + * cdef Component comp = Component() + * comp._this = ptr + */ + +static struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(khmer::ComponentPtr __pyx_v_ptr) { + struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_comp = 0; + struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("create", 0); + + /* "khmer/_oxli/partitioning.pyx":60 + * @staticmethod + * cdef Component create(ComponentPtr ptr): + * cdef Component comp = Component() # <<<<<<<<<<<<<< + * comp._this = ptr + * return comp + */ + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_12partitioning_Component), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 60, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_comp = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "khmer/_oxli/partitioning.pyx":61 + * cdef Component create(ComponentPtr ptr): + * cdef Component comp = Component() + * comp._this = ptr # <<<<<<<<<<<<<< + * return comp + * + */ + __pyx_v_comp->_this = __pyx_v_ptr; + + /* "khmer/_oxli/partitioning.pyx":62 + * cdef Component comp = Component() + * comp._this = ptr + * return comp # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __Pyx_XDECREF(((PyObject *)__pyx_r)); + __Pyx_INCREF(((PyObject *)__pyx_v_comp)); + __pyx_r = __pyx_v_comp; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":59 + * + * @staticmethod + * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< + * cdef Component comp = Component() + * comp._this = ptr + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.partitioning.Component.create", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_comp); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":65 + * + * @staticmethod + * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph): # <<<<<<<<<<<<<< + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef vector[BoundedCounterType] counts + */ + +static std::vector __pyx_f_5khmer_5_oxli_12partitioning_9Component__tag_counts(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph) { + uint64_t __pyx_v_n_tags; + std::vector __pyx_v_counts; + int __pyx_v_idx; + uint64_t __pyx_v_tag; + std::vector __pyx_r; + __Pyx_RefNannyDeclarations + std::vector __pyx_t_1; + std::set ::iterator __pyx_t_2; + std::set *__pyx_t_3; + khmer::HashIntoType __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *(*__pyx_t_9)(PyObject *); + int __pyx_t_10; + uint64_t __pyx_t_11; + __Pyx_RefNannySetupContext("_tag_counts", 0); + + /* "khmer/_oxli/partitioning.pyx":66 + * @staticmethod + * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph): + * cdef uint64_t n_tags = deref(comp).get_n_tags() # <<<<<<<<<<<<<< + * cdef vector[BoundedCounterType] counts + * counts = vector[BoundedCounterType](n_tags) + */ + __pyx_v_n_tags = (*__pyx_v_comp).get_n_tags(); + + /* "khmer/_oxli/partitioning.pyx":68 + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef vector[BoundedCounterType] counts + * counts = vector[BoundedCounterType](n_tags) # <<<<<<<<<<<<<< + * cdef int idx + * cdef uint64_t tag + */ + try { + __pyx_t_1 = std::vector (__pyx_v_n_tags); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 68, __pyx_L1_error) + } + __pyx_v_counts = __pyx_t_1; + + /* "khmer/_oxli/partitioning.pyx":71 + * cdef int idx + * cdef uint64_t tag + * for idx, tag in deref(comp).tags: # <<<<<<<<<<<<<< + * counts[idx] = deref(graph).get_count(tag) + * return counts + */ + __pyx_t_3 = &(*__pyx_v_comp).tags; + __pyx_t_2 = __pyx_t_3->begin(); + for (;;) { + if (!(__pyx_t_2 != __pyx_t_3->end())) break; + __pyx_t_4 = *__pyx_t_2; + ++__pyx_t_2; + __pyx_t_5 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { + PyObject* sequence = __pyx_t_5; + #if !CYTHON_COMPILING_IN_PYPY + Py_ssize_t size = Py_SIZE(sequence); + #else + Py_ssize_t size = PySequence_Size(sequence); + #endif + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 71, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); + __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); + } else { + __pyx_t_6 = PyList_GET_ITEM(sequence, 0); + __pyx_t_7 = PyList_GET_ITEM(sequence, 1); + } + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(__pyx_t_7); + #else + __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + #endif + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_8 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_9 = Py_TYPE(__pyx_t_8)->tp_iternext; + index = 0; __pyx_t_6 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_6)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_6); + index = 1; __pyx_t_7 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_7)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < 0) __PYX_ERR(0, 71, __pyx_L1_error) + __pyx_t_9 = NULL; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L6_unpacking_done; + __pyx_L5_unpacking_failed:; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 71, __pyx_L1_error) + __pyx_L6_unpacking_done:; + } + __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_6); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_11 = __Pyx_PyInt_As_uint64_t(__pyx_t_7); if (unlikely((__pyx_t_11 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_v_idx = __pyx_t_10; + __pyx_v_tag = __pyx_t_11; + + /* "khmer/_oxli/partitioning.pyx":72 + * cdef uint64_t tag + * for idx, tag in deref(comp).tags: + * counts[idx] = deref(graph).get_count(tag) # <<<<<<<<<<<<<< + * return counts + * + */ + (__pyx_v_counts[__pyx_v_idx]) = (*__pyx_v_graph).get_count(__pyx_v_tag); + + /* "khmer/_oxli/partitioning.pyx":71 + * cdef int idx + * cdef uint64_t tag + * for idx, tag in deref(comp).tags: # <<<<<<<<<<<<<< + * counts[idx] = deref(graph).get_count(tag) + * return counts + */ + } + + /* "khmer/_oxli/partitioning.pyx":73 + * for idx, tag in deref(comp).tags: + * counts[idx] = deref(graph).get_count(tag) + * return counts # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __pyx_r = __pyx_v_counts; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":65 + * + * @staticmethod + * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph): # <<<<<<<<<<<<<< + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef vector[BoundedCounterType] counts + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_WriteUnraisable("khmer._oxli.partitioning.Component._tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":76 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CPyHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12tag_counts(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_5khmer_5_oxli_12partitioning_9Component_12tag_counts = {"tag_counts", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12tag_counts, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12tag_counts(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_component = 0; + PyObject *__pyx_v_graph = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("tag_counts (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_component,&__pyx_n_s_graph,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_component)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("tag_counts", 1, 2, 2, 1); __PYX_ERR(0, 76, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "tag_counts") < 0)) __PYX_ERR(0, 76, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + } + __pyx_v_component = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)values[0]); + __pyx_v_graph = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("tag_counts", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 76, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.partitioning.Component.tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_component), __pyx_ptype_5khmer_5_oxli_12partitioning_Component, 1, "component", 0))) __PYX_ERR(0, 76, __pyx_L1_error) + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_11tag_counts(__pyx_v_component, __pyx_v_graph); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_11tag_counts(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_component, PyObject *__pyx_v_graph) { + khmer::khmer_KHashtable_Object *__pyx_v_graph_ptr; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("tag_counts", 0); + + /* "khmer/_oxli/partitioning.pyx":77 + * @staticmethod + * def tag_counts(Component component, graph): + * cdef CPyHashtable_Object* graph_ptr = graph # <<<<<<<<<<<<<< + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + * + */ + __pyx_v_graph_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); + + /* "khmer/_oxli/partitioning.pyx":78 + * def tag_counts(Component component, graph): + * cdef CPyHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(__pyx_f_5khmer_5_oxli_12partitioning_9Component__tag_counts(__pyx_v_component->_this, (*__pyx_v_graph_ptr).hashtable)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":76 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CPyHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.partitioning.Component.tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":81 + * + * @staticmethod + * cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): # <<<<<<<<<<<<<< + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef float acc = 0 + */ + +static float __pyx_f_5khmer_5_oxli_12partitioning_9Component__mean_tag_count(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph) { + uint64_t __pyx_v_n_tags; + float __pyx_v_acc; + uint64_t __pyx_v_tag; + float __pyx_r; + __Pyx_RefNannyDeclarations + std::set ::iterator __pyx_t_1; + std::set *__pyx_t_2; + khmer::HashIntoType __pyx_t_3; + __Pyx_RefNannySetupContext("_mean_tag_count", 0); + + /* "khmer/_oxli/partitioning.pyx":82 + * @staticmethod + * cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): + * cdef uint64_t n_tags = deref(comp).get_n_tags() # <<<<<<<<<<<<<< + * cdef float acc = 0 + * cdef uint64_t tag + */ + __pyx_v_n_tags = (*__pyx_v_comp).get_n_tags(); + + /* "khmer/_oxli/partitioning.pyx":83 + * cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef float acc = 0 # <<<<<<<<<<<<<< + * cdef uint64_t tag + * for tag in deref(comp).tags: + */ + __pyx_v_acc = 0.0; + + /* "khmer/_oxli/partitioning.pyx":85 + * cdef float acc = 0 + * cdef uint64_t tag + * for tag in deref(comp).tags: # <<<<<<<<<<<<<< + * acc += deref(graph).get_count(tag) + * return acc / n_tags + */ + __pyx_t_2 = &(*__pyx_v_comp).tags; + __pyx_t_1 = __pyx_t_2->begin(); + for (;;) { + if (!(__pyx_t_1 != __pyx_t_2->end())) break; + __pyx_t_3 = *__pyx_t_1; + ++__pyx_t_1; + __pyx_v_tag = __pyx_t_3; + + /* "khmer/_oxli/partitioning.pyx":86 + * cdef uint64_t tag + * for tag in deref(comp).tags: + * acc += deref(graph).get_count(tag) # <<<<<<<<<<<<<< + * return acc / n_tags + * + */ + __pyx_v_acc = (__pyx_v_acc + ((float)(*__pyx_v_graph).get_count(__pyx_v_tag))); + + /* "khmer/_oxli/partitioning.pyx":85 + * cdef float acc = 0 + * cdef uint64_t tag + * for tag in deref(comp).tags: # <<<<<<<<<<<<<< + * acc += deref(graph).get_count(tag) + * return acc / n_tags + */ + } + + /* "khmer/_oxli/partitioning.pyx":87 + * for tag in deref(comp).tags: + * acc += deref(graph).get_count(tag) + * return acc / n_tags # <<<<<<<<<<<<<< + * + * + */ + if (unlikely(((float)__pyx_v_n_tags) == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "float division"); + __PYX_ERR(0, 87, __pyx_L1_error) + } + __pyx_r = (__pyx_v_acc / ((float)__pyx_v_n_tags)); + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":81 + * + * @staticmethod + * cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): # <<<<<<<<<<<<<< + * cdef uint64_t n_tags = deref(comp).get_n_tags() + * cdef float acc = 0 + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_WriteUnraisable("khmer._oxli.partitioning.Component._mean_tag_count", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); + __pyx_r = 0; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":92 + * cdef class StreamingPartitioner: + * + * def __cinit__(self, graph): # <<<<<<<<<<<<<< + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') + */ + +/* Python wrapper */ +static int __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_graph = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_graph,0}; + PyObject* values[1] = {0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 92, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + } + __pyx_v_graph = values[0]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 92, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner___cinit__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), __pyx_v_graph); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph) { + khmer::khmer_KHashtable_Object *__pyx_v_ptr; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + khmer::Hashtable *__pyx_t_5; + khmer::StreamingPartitioner *__pyx_t_6; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "khmer/_oxli/partitioning.pyx":93 + * + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< + * raise ValueError('Must take an object with Hashtable *') + * + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 93, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = (__pyx_t_3 != 0); + if (!__pyx_t_4) { + } else { + __pyx_t_1 = __pyx_t_4; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 93, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_3 = (__pyx_t_4 != 0); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + __pyx_t_3 = ((!__pyx_t_1) != 0); + if (__pyx_t_3) { + + /* "khmer/_oxli/partitioning.pyx":94 + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< + * + * cdef CPyHashtable_Object* ptr = graph + */ + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 94, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 94, __pyx_L1_error) + + /* "khmer/_oxli/partitioning.pyx":93 + * + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< + * raise ValueError('Must take an object with Hashtable *') + * + */ + } + + /* "khmer/_oxli/partitioning.pyx":96 + * raise ValueError('Must take an object with Hashtable *') + * + * cdef CPyHashtable_Object* ptr = graph # <<<<<<<<<<<<<< + * self._graph_ptr = deref(ptr).hashtable + * + */ + __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); + + /* "khmer/_oxli/partitioning.pyx":97 + * + * cdef CPyHashtable_Object* ptr = graph + * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< + * + * self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) + */ + __pyx_t_5 = (*__pyx_v_ptr).hashtable; + __pyx_v_self->_graph_ptr = __pyx_t_5; + + /* "khmer/_oxli/partitioning.pyx":99 + * self._graph_ptr = deref(ptr).hashtable + * + * self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) # <<<<<<<<<<<<<< + * + * self._tag_component_map = deref(self._this).get_tag_component_map() + */ + try { + __pyx_t_6 = new khmer::StreamingPartitioner(__pyx_v_self->_graph_ptr); + } catch(...) { + try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } + __PYX_ERR(0, 99, __pyx_L1_error) + } + __pyx_v_self->_this.reset(__pyx_t_6); + + /* "khmer/_oxli/partitioning.pyx":101 + * self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) + * + * self._tag_component_map = deref(self._this).get_tag_component_map() # <<<<<<<<<<<<<< + * self._components = deref(self._this).get_component_set() + * self.n_consumed = 0 + */ + __pyx_v_self->_tag_component_map = (*__pyx_v_self->_this).get_tag_component_map(); + + /* "khmer/_oxli/partitioning.pyx":102 + * + * self._tag_component_map = deref(self._this).get_tag_component_map() + * self._components = deref(self._this).get_component_set() # <<<<<<<<<<<<<< + * self.n_consumed = 0 + * + */ + __pyx_v_self->_components = (*__pyx_v_self->_this).get_component_set(); + + /* "khmer/_oxli/partitioning.pyx":103 + * self._tag_component_map = deref(self._this).get_tag_component_map() + * self._components = deref(self._this).get_component_set() + * self.n_consumed = 0 # <<<<<<<<<<<<<< + * + * def consume_sequence(self, sequence): + */ + __pyx_v_self->n_consumed = 0; + + /* "khmer/_oxli/partitioning.pyx":92 + * cdef class StreamingPartitioner: + * + * def __cinit__(self, graph): # <<<<<<<<<<<<<< + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":105 + * self.n_consumed = 0 + * + * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< + * deref(self._this).consume_sequence(sequence.encode('utf-8')) + * self.n_consumed += 1 + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_3consume_sequence(PyObject *__pyx_v_self, PyObject *__pyx_v_sequence); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_3consume_sequence(PyObject *__pyx_v_self, PyObject *__pyx_v_sequence) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("consume_sequence (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_2consume_sequence(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_sequence)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + std::string __pyx_t_3; + __Pyx_RefNannySetupContext("consume_sequence", 0); + + /* "khmer/_oxli/partitioning.pyx":106 + * + * def consume_sequence(self, sequence): + * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< + * self.n_consumed += 1 + * + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + try { + (*__pyx_v_self->_this).consume_sequence(__pyx_t_3); + } catch(...) { + try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } + __PYX_ERR(0, 106, __pyx_L1_error) + } + + /* "khmer/_oxli/partitioning.pyx":107 + * def consume_sequence(self, sequence): + * deref(self._this).consume_sequence(sequence.encode('utf-8')) + * self.n_consumed += 1 # <<<<<<<<<<<<<< + * + * def consume_fasta(self, filename): + */ + __pyx_v_self->n_consumed = (__pyx_v_self->n_consumed + 1); + + /* "khmer/_oxli/partitioning.pyx":105 + * self.n_consumed = 0 + * + * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< + * deref(self._this).consume_sequence(sequence.encode('utf-8')) + * self.n_consumed += 1 + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.consume_sequence", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":109 + * self.n_consumed += 1 + * + * def consume_fasta(self, filename): # <<<<<<<<<<<<<< + * return deref(self._this).consume_fasta(filename.encode('utf-8')) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_5consume_fasta(PyObject *__pyx_v_self, PyObject *__pyx_v_filename); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_5consume_fasta(PyObject *__pyx_v_self, PyObject *__pyx_v_filename) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("consume_fasta (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_4consume_fasta(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_filename)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_4consume_fasta(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + std::string __pyx_t_3; + uint64_t __pyx_t_4; + __Pyx_RefNannySetupContext("consume_fasta", 0); + + /* "khmer/_oxli/partitioning.pyx":110 + * + * def consume_fasta(self, filename): + * return deref(self._this).consume_fasta(filename.encode('utf-8')) # <<<<<<<<<<<<<< + * + * def get_tag_component(self, kmer): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_filename, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + try { + __pyx_t_4 = (*__pyx_v_self->_this).consume_fasta(__pyx_t_3); + } catch(...) { + try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } + __PYX_ERR(0, 110, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyInt_From_uint64_t(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":109 + * self.n_consumed += 1 + * + * def consume_fasta(self, filename): # <<<<<<<<<<<<<< + * return deref(self._this).consume_fasta(filename.encode('utf-8')) + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.consume_fasta", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":112 + * return deref(self._this).consume_fasta(filename.encode('utf-8')) + * + * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_7get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_7get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("get_tag_component (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6get_tag_component(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6get_tag_component(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { + khmer::ComponentPtr __pyx_v_compptr; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + std::string __pyx_t_3; + int __pyx_t_4; + __Pyx_RefNannySetupContext("get_tag_component", 0); + + /* "khmer/_oxli/partitioning.pyx":114 + * def get_tag_component(self, kmer): + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if compptr == NULL: + * return None + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_compptr = (*__pyx_v_self->_this).get_tag_component(__pyx_t_3); + + /* "khmer/_oxli/partitioning.pyx":115 + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if compptr == NULL: # <<<<<<<<<<<<<< + * return None + * else: + */ + __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); + if (__pyx_t_4) { + + /* "khmer/_oxli/partitioning.pyx":116 + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if compptr == NULL: + * return None # <<<<<<<<<<<<<< + * else: + * return Component.create(compptr) + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_None); + __pyx_r = Py_None; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":115 + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + * if compptr == NULL: # <<<<<<<<<<<<<< + * return None + * else: + */ + } + + /* "khmer/_oxli/partitioning.pyx":118 + * return None + * else: + * return Component.create(compptr) # <<<<<<<<<<<<<< + * + * def get_nearest_component(self, kmer): + */ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 118, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "khmer/_oxli/partitioning.pyx":112 + * return deref(self._this).consume_fasta(filename.encode('utf-8')) + * + * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.get_tag_component", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":120 + * return Component.create(compptr) + * + * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_9get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_9get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("get_nearest_component (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_8get_nearest_component(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_8get_nearest_component(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { + khmer::ComponentPtr __pyx_v_compptr; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + std::string __pyx_t_3; + int __pyx_t_4; + __Pyx_RefNannySetupContext("get_nearest_component", 0); + + /* "khmer/_oxli/partitioning.pyx":122 + * def get_nearest_component(self, kmer): + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if compptr == NULL: + * return None + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 122, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 122, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_compptr = (*__pyx_v_self->_this).get_nearest_component(__pyx_t_3); + + /* "khmer/_oxli/partitioning.pyx":123 + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if compptr == NULL: # <<<<<<<<<<<<<< + * return None + * else: + */ + __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); + if (__pyx_t_4) { + + /* "khmer/_oxli/partitioning.pyx":124 + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if compptr == NULL: + * return None # <<<<<<<<<<<<<< + * else: + * return Component.create(compptr) + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_None); + __pyx_r = Py_None; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":123 + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + * if compptr == NULL: # <<<<<<<<<<<<<< + * return None + * else: + */ + } + + /* "khmer/_oxli/partitioning.pyx":126 + * return None + * else: + * return Component.create(compptr) # <<<<<<<<<<<<<< + * + * def components(self): + */ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 126, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "khmer/_oxli/partitioning.pyx":120 + * return Component.create(compptr) + * + * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.get_nearest_component", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ + +/* "khmer/_oxli/partitioning.pyx":128 + * return Component.create(compptr) + * + * def components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[ComponentPtrSet] locked + * cdef ComponentPtr cmpptr + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_11components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_11components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("components (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10components(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("components", 0); + __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 128, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_components, __pyx_n_s_StreamingPartitioner_components, __pyx_n_s_khmer__oxli_partitioning); if (unlikely(!gen)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + bool __pyx_t_1; + std::set ::iterator __pyx_t_2; + khmer::ComponentPtrSet *__pyx_t_3; + khmer::ComponentPtr __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("None", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L7_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 128, __pyx_L1_error) + + /* "khmer/_oxli/partitioning.pyx":131 + * cdef shared_ptr[ComponentPtrSet] locked + * cdef ComponentPtr cmpptr + * lockedptr = self._components.lock() # <<<<<<<<<<<<<< + * if lockedptr: + * for cmpptr in deref(lockedptr): + */ + __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_components.lock(); + + /* "khmer/_oxli/partitioning.pyx":132 + * cdef ComponentPtr cmpptr + * lockedptr = self._components.lock() + * if lockedptr: # <<<<<<<<<<<<<< + * for cmpptr in deref(lockedptr): + * yield Component.create(cmpptr) + */ + __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); + if (__pyx_t_1) { + + /* "khmer/_oxli/partitioning.pyx":133 + * lockedptr = self._components.lock() + * if lockedptr: + * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< + * yield Component.create(cmpptr) + * else: + */ + __pyx_t_3 = &(*__pyx_cur_scope->__pyx_v_lockedptr); + __pyx_t_2 = __pyx_t_3->begin(); + for (;;) { + if (!(__pyx_t_2 != __pyx_t_3->end())) break; + __pyx_t_4 = *__pyx_t_2; + ++__pyx_t_2; + __pyx_cur_scope->__pyx_v_cmpptr = __pyx_t_4; + + /* "khmer/_oxli/partitioning.pyx":134 + * if lockedptr: + * for cmpptr in deref(lockedptr): + * yield Component.create(cmpptr) # <<<<<<<<<<<<<< + * else: + * raise MemoryError("Can't locked underlying Component set") + */ + __pyx_t_5 = ((PyObject *)__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(__pyx_cur_scope->__pyx_v_cmpptr)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L7_resume_from_yield:; + __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; + __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 134, __pyx_L1_error) + + /* "khmer/_oxli/partitioning.pyx":133 + * lockedptr = self._components.lock() + * if lockedptr: + * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< + * yield Component.create(cmpptr) + * else: + */ + } + + /* "khmer/_oxli/partitioning.pyx":132 + * cdef ComponentPtr cmpptr + * lockedptr = self._components.lock() + * if lockedptr: # <<<<<<<<<<<<<< + * for cmpptr in deref(lockedptr): + * yield Component.create(cmpptr) + */ + goto __pyx_L4; + } + + /* "khmer/_oxli/partitioning.pyx":136 + * yield Component.create(cmpptr) + * else: + * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< + * + * def tag_components(self): + */ + /*else*/ { + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 136, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_Raise(__pyx_t_5, 0, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(0, 136, __pyx_L1_error) + } + __pyx_L4:; + if (1); else __pyx_cur_scope = __pyx_cur_scope; + + /* "khmer/_oxli/partitioning.pyx":128 + * return Component.create(compptr) + * + * def components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[ComponentPtrSet] locked + * cdef ComponentPtr cmpptr + */ + + /* function exit code */ + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); __pyx_r = 0; + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_15generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ + +/* "khmer/_oxli/partitioning.pyx":138 + * raise MemoryError("Can't locked underlying Component set") + * + * def tag_components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[CpGuardedKmerCompMap] locked + * cdef pair[HashIntoType,ComponentPtr] cpair + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_14tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_14tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("tag_components (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_13tag_components(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_13tag_components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("tag_components", 0); + __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 138, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_15generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_tag_components, __pyx_n_s_StreamingPartitioner_tag_compone, __pyx_n_s_khmer__oxli_partitioning); if (unlikely(!gen)) __PYX_ERR(0, 138, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_15generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + bool __pyx_t_1; + std::map ::iterator __pyx_t_2; + std::map *__pyx_t_3; + std::pair __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("None", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L7_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 138, __pyx_L1_error) + + /* "khmer/_oxli/partitioning.pyx":141 + * cdef shared_ptr[CpGuardedKmerCompMap] locked + * cdef pair[HashIntoType,ComponentPtr] cpair + * lockedptr = self._tag_component_map.lock() # <<<<<<<<<<<<<< + * if lockedptr: + * for cpair in deref(lockedptr).data: + */ + __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_tag_component_map.lock(); + + /* "khmer/_oxli/partitioning.pyx":142 + * cdef pair[HashIntoType,ComponentPtr] cpair + * lockedptr = self._tag_component_map.lock() + * if lockedptr: # <<<<<<<<<<<<<< + * for cpair in deref(lockedptr).data: + * yield cpair.first, Component.create(cpair.second) + */ + __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); + if (__pyx_t_1) { + + /* "khmer/_oxli/partitioning.pyx":143 + * lockedptr = self._tag_component_map.lock() + * if lockedptr: + * for cpair in deref(lockedptr).data: # <<<<<<<<<<<<<< + * yield cpair.first, Component.create(cpair.second) + * else: + */ + __pyx_t_3 = &(*__pyx_cur_scope->__pyx_v_lockedptr).data; + __pyx_t_2 = __pyx_t_3->begin(); + for (;;) { + if (!(__pyx_t_2 != __pyx_t_3->end())) break; + __pyx_t_4 = *__pyx_t_2; + ++__pyx_t_2; + __pyx_cur_scope->__pyx_v_cpair = __pyx_t_4; + + /* "khmer/_oxli/partitioning.pyx":144 + * if lockedptr: + * for cpair in deref(lockedptr).data: + * yield cpair.first, Component.create(cpair.second) # <<<<<<<<<<<<<< + * else: + * raise MemoryError("Can't locked underlying Component set") + */ + __pyx_t_5 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_cur_scope->__pyx_v_cpair.first); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = ((PyObject *)__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(__pyx_cur_scope->__pyx_v_cpair.second)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 144, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6); + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_r = __pyx_t_7; + __pyx_t_7 = 0; + __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L7_resume_from_yield:; + __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; + __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 144, __pyx_L1_error) + + /* "khmer/_oxli/partitioning.pyx":143 + * lockedptr = self._tag_component_map.lock() + * if lockedptr: + * for cpair in deref(lockedptr).data: # <<<<<<<<<<<<<< + * yield cpair.first, Component.create(cpair.second) + * else: + */ + } + + /* "khmer/_oxli/partitioning.pyx":142 + * cdef pair[HashIntoType,ComponentPtr] cpair + * lockedptr = self._tag_component_map.lock() + * if lockedptr: # <<<<<<<<<<<<<< + * for cpair in deref(lockedptr).data: + * yield cpair.first, Component.create(cpair.second) + */ + goto __pyx_L4; + } + + /* "khmer/_oxli/partitioning.pyx":146 + * yield cpair.first, Component.create(cpair.second) + * else: + * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< + * + * def write_components(self, filename): + */ + /*else*/ { + __pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_Raise(__pyx_t_7, 0, 0, 0); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_ERR(0, 146, __pyx_L1_error) + } + __pyx_L4:; + if (1); else __pyx_cur_scope = __pyx_cur_scope; + + /* "khmer/_oxli/partitioning.pyx":138 + * raise MemoryError("Can't locked underlying Component set") + * + * def tag_components(self): # <<<<<<<<<<<<<< + * cdef shared_ptr[CpGuardedKmerCompMap] locked + * cdef pair[HashIntoType,ComponentPtr] cpair + */ + + /* function exit code */ + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); __pyx_r = 0; + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":148 + * raise MemoryError("Can't locked underlying Component set") + * + * def write_components(self, filename): # <<<<<<<<<<<<<< + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_17write_components(PyObject *__pyx_v_self, PyObject *__pyx_v_filename); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_17write_components(PyObject *__pyx_v_self, PyObject *__pyx_v_filename) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("write_components (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_16write_components(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_filename)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_16write_components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename) { + FILE *__pyx_v_fp; + khmer::ComponentPtr __pyx_v_cmpptr; + std::shared_ptr __pyx_v_lockedptr; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + char const *__pyx_t_3; + int __pyx_t_4; + bool __pyx_t_5; + std::set ::iterator __pyx_t_6; + khmer::ComponentPtrSet *__pyx_t_7; + khmer::ComponentPtr __pyx_t_8; + __Pyx_RefNannySetupContext("write_components", 0); + + /* "khmer/_oxli/partitioning.pyx":150 + * def write_components(self, filename): + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') # <<<<<<<<<<<<<< + * if fp == NULL: + * raise IOError("Can't open file.") + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_filename, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 150, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 150, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __Pyx_PyObject_AsString(__pyx_t_2); if (unlikely((!__pyx_t_3) && PyErr_Occurred())) __PYX_ERR(0, 150, __pyx_L1_error) + __pyx_v_fp = fopen(__pyx_t_3, ((char const *)"wb")); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "khmer/_oxli/partitioning.pyx":151 + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') + * if fp == NULL: # <<<<<<<<<<<<<< + * raise IOError("Can't open file.") + * + */ + __pyx_t_4 = ((__pyx_v_fp == NULL) != 0); + if (__pyx_t_4) { + + /* "khmer/_oxli/partitioning.pyx":152 + * fp = fopen(filename.encode('utf-8'), 'wb') + * if fp == NULL: + * raise IOError("Can't open file.") # <<<<<<<<<<<<<< + * + * cdef ComponentPtr cmpptr + */ + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 152, __pyx_L1_error) + + /* "khmer/_oxli/partitioning.pyx":151 + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') + * if fp == NULL: # <<<<<<<<<<<<<< + * raise IOError("Can't open file.") + * + */ + } + + /* "khmer/_oxli/partitioning.pyx":156 + * cdef ComponentPtr cmpptr + * cdef shared_ptr[ComponentPtrSet] lockedptr + * lockedptr = self._components.lock() # <<<<<<<<<<<<<< + * + * if lockedptr: + */ + __pyx_v_lockedptr = __pyx_v_self->_components.lock(); + + /* "khmer/_oxli/partitioning.pyx":158 + * lockedptr = self._components.lock() + * + * if lockedptr: # <<<<<<<<<<<<<< + * for cmpptr in deref(lockedptr): + * fprintf(fp, "%llu,%llu,%f\n", + */ + __pyx_t_5 = __pyx_v_lockedptr.operator bool(); + if (__pyx_t_5) { + + /* "khmer/_oxli/partitioning.pyx":159 + * + * if lockedptr: + * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< + * fprintf(fp, "%llu,%llu,%f\n", + * deref(cmpptr).component_id, + */ + __pyx_t_7 = &(*__pyx_v_lockedptr); + __pyx_t_6 = __pyx_t_7->begin(); + for (;;) { + if (!(__pyx_t_6 != __pyx_t_7->end())) break; + __pyx_t_8 = *__pyx_t_6; + ++__pyx_t_6; + __pyx_v_cmpptr = __pyx_t_8; + + /* "khmer/_oxli/partitioning.pyx":160 + * if lockedptr: + * for cmpptr in deref(lockedptr): + * fprintf(fp, "%llu,%llu,%f\n", # <<<<<<<<<<<<<< + * deref(cmpptr).component_id, + * deref(cmpptr).get_n_tags(), + */ + fprintf(__pyx_v_fp, ((char const *)"%llu,%llu,%f\n"), (*__pyx_v_cmpptr).component_id, (*__pyx_v_cmpptr).get_n_tags(), __pyx_f_5khmer_5_oxli_12partitioning_9Component__mean_tag_count(__pyx_v_cmpptr, __pyx_v_self->_graph_ptr)); + + /* "khmer/_oxli/partitioning.pyx":159 + * + * if lockedptr: + * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< + * fprintf(fp, "%llu,%llu,%f\n", + * deref(cmpptr).component_id, + */ + } + + /* "khmer/_oxli/partitioning.pyx":158 + * lockedptr = self._components.lock() + * + * if lockedptr: # <<<<<<<<<<<<<< + * for cmpptr in deref(lockedptr): + * fprintf(fp, "%llu,%llu,%f\n", + */ + } + + /* "khmer/_oxli/partitioning.pyx":164 + * deref(cmpptr).get_n_tags(), + * Component._mean_tag_count(cmpptr, self._graph_ptr)) + * fclose(fp) # <<<<<<<<<<<<<< + * + * property n_components: + */ + fclose(__pyx_v_fp); + + /* "khmer/_oxli/partitioning.pyx":148 + * raise MemoryError("Can't locked underlying Component set") + * + * def write_components(self, filename): # <<<<<<<<<<<<<< + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.write_components", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":167 + * + * property n_components: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_components() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli/partitioning.pyx":168 + * property n_components: + * def __get__(self): + * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< + * + * property n_tags: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 168, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":167 + * + * property n_components: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_components() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.n_components.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pyx":171 + * + * property n_tags: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_tags() + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "khmer/_oxli/partitioning.pyx":172 + * property n_tags: + * def __get__(self): + * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_tags()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 172, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "khmer/_oxli/partitioning.pyx":171 + * + * property n_tags: + * def __get__(self): # <<<<<<<<<<<<<< + * return deref(self._this).get_n_tags() + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.n_tags.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "khmer/_oxli/partitioning.pxd":26 + * cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map + * cdef CpHashtable * _graph_ptr + * cdef readonly uint64_t n_consumed # <<<<<<<<<<<<<< + * + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_uint64_t(__pyx_v_self->n_consumed); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 26, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.n_consumed.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "vector.to_py":67 + * + * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") + * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): # <<<<<<<<<<<<<< + * return [X_to_py(v[i]) for i in range(v.size())] + * + */ + +static PyObject *__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(const std::vector &__pyx_v_v) { + size_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + size_t __pyx_t_2; + size_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType", 0); + + /* "vector.to_py":68 + * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") + * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): + * return [X_to_py(v[i]) for i in range(v.size())] # <<<<<<<<<<<<<< + * + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_v_v.size(); + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + __pyx_t_4 = __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "vector.to_py":67 + * + * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") + * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): # <<<<<<<<<<<<<< + * return [X_to_py(v[i]) for i in range(v.size())] + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("vector.to_py.__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { + Py_ssize_t __pyx_v_length; + char *__pyx_v_data; + std::string __pyx_r; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); + + /* "string.from_py":15 + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< + * return string(data, length) + * + */ + __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(2, 15, __pyx_L1_error) + __pyx_v_data = __pyx_t_1; + + /* "string.from_py":16 + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * return string(data, length) # <<<<<<<<<<<<<< + * + * + */ + __pyx_r = std::string(__pyx_v_data, __pyx_v_length); + goto __pyx_L0; + + /* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static struct __pyx_vtabstruct_5khmer_5_oxli_12partitioning_Component __pyx_vtable_5khmer_5_oxli_12partitioning_Component; + +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning_Component(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *p; + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)o); + p->__pyx_vtab = __pyx_vtabptr_5khmer_5_oxli_12partitioning_Component; + new((void*)&(p->_this)) khmer::ComponentPtr(); + if (unlikely(__pyx_pw_5khmer_5_oxli_12partitioning_9Component_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning_Component(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)o; + #if PY_VERSION_HEX >= 0x030400a1 + if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + __Pyx_call_destructor(p->_this); + (*Py_TYPE(o)->tp_free)(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_9Component_component_id(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_12partitioning_9Component_12component_id_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_9Component__n_created(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_12partitioning_9Component_10_n_created_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_9Component__n_destroyed(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed_1__get__(o); +} + +static PyMethodDef __pyx_methods_5khmer_5_oxli_12partitioning_Component[] = { + {"tag_counts", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12tag_counts, METH_VARARGS|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_12partitioning_Component[] = { + {(char *)"component_id", __pyx_getprop_5khmer_5_oxli_12partitioning_9Component_component_id, 0, (char *)0, 0}, + {(char *)"_n_created", __pyx_getprop_5khmer_5_oxli_12partitioning_9Component__n_created, 0, (char *)0, 0}, + {(char *)"_n_destroyed", __pyx_getprop_5khmer_5_oxli_12partitioning_9Component__n_destroyed, 0, (char *)0, 0}, + {0, 0, 0, 0, 0} +}; + +static PySequenceMethods __pyx_tp_as_sequence_Component = { + __pyx_pw_5khmer_5_oxli_12partitioning_9Component_3__len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Component = { + __pyx_pw_5khmer_5_oxli_12partitioning_9Component_3__len__, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning_Component = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.partitioning.Component", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_12partitioning_Component, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_Component, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Component, /*tp_as_mapping*/ + __pyx_pw_5khmer_5_oxli_12partitioning_9Component_8__hash__, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + __pyx_pw_5khmer_5_oxli_12partitioning_9Component_10__richcmp__, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + __pyx_pw_5khmer_5_oxli_12partitioning_9Component_5__iter__, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5khmer_5_oxli_12partitioning_Component, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5khmer_5_oxli_12partitioning_Component, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_12partitioning_Component, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *p; + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)o); + new((void*)&(p->_this)) std::unique_ptr (); + new((void*)&(p->_components)) std::weak_ptr (); + new((void*)&(p->_tag_component_map)) std::weak_ptr (); + if (unlikely(__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning_StreamingPartitioner(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)o; + #if PY_VERSION_HEX >= 0x030400a1 + if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + __Pyx_call_destructor(p->_this); + __Pyx_call_destructor(p->_components); + __Pyx_call_destructor(p->_tag_component_map); + (*Py_TYPE(o)->tp_free)(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_components(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_tags(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags_1__get__(o); +} + +static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_consumed(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed_1__get__(o); +} + +static PyMethodDef __pyx_methods_5khmer_5_oxli_12partitioning_StreamingPartitioner[] = { + {"consume_sequence", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_3consume_sequence, METH_O, 0}, + {"consume_fasta", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_5consume_fasta, METH_O, 0}, + {"get_tag_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_7get_tag_component, METH_O, 0}, + {"get_nearest_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_9get_nearest_component, METH_O, 0}, + {"components", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_11components, METH_NOARGS, 0}, + {"tag_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_14tag_components, METH_NOARGS, 0}, + {"write_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_17write_components, METH_O, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_12partitioning_StreamingPartitioner[] = { + {(char *)"n_components", __pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_components, 0, (char *)0, 0}, + {(char *)"n_tags", __pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_tags, 0, (char *)0, 0}, + {(char *)"n_consumed", __pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_consumed, 0, (char *)0, 0}, + {0, 0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.partitioning.StreamingPartitioner", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_12partitioning_StreamingPartitioner, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5khmer_5_oxli_12partitioning_StreamingPartitioner, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5khmer_5_oxli_12partitioning_StreamingPartitioner, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_12partitioning_StreamingPartitioner, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__[8]; +static int __pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ = 0; + +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *p; + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__)))) { + o = (PyObject*)__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__[--__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__]; + memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__)); + (void) PyObject_INIT(o, t); + PyObject_GC_Track(o); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o); + new((void*)&(p->__pyx_t_0)) std::set ::iterator(); + return o; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o; + PyObject_GC_UnTrack(o); + __Pyx_call_destructor(p->__pyx_t_0); + Py_CLEAR(p->__pyx_v_self); + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__)))) { + __pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__[__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__++] = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static int __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o; + if (p->__pyx_v_self) { + e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o; + tmp = ((PyObject*)p->__pyx_v_self); + p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.partitioning.__pyx_scope_struct____iter__", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, /*tp_traverse*/ + __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components[8]; +static int __pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components = 0; + +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *p; + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components)))) { + o = (PyObject*)__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components[--__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components]; + memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components)); + (void) PyObject_INIT(o, t); + PyObject_GC_Track(o); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o); + new((void*)&(p->__pyx_v_cmpptr)) khmer::ComponentPtr(); + new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); + new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); + new((void*)&(p->__pyx_t_0)) std::set ::iterator(); + return o; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o; + PyObject_GC_UnTrack(o); + __Pyx_call_destructor(p->__pyx_v_cmpptr); + __Pyx_call_destructor(p->__pyx_v_locked); + __Pyx_call_destructor(p->__pyx_v_lockedptr); + __Pyx_call_destructor(p->__pyx_t_0); + Py_CLEAR(p->__pyx_v_self); + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components)))) { + __pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components[__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components++] = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static int __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o; + if (p->__pyx_v_self) { + e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o; + tmp = ((PyObject*)p->__pyx_v_self); + p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.partitioning.__pyx_scope_struct_1_components", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, /*tp_traverse*/ + __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components[8]; +static int __pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components = 0; + +static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *p; + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components)))) { + o = (PyObject*)__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components[--__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components]; + memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components)); + (void) PyObject_INIT(o, t); + PyObject_GC_Track(o); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o); + new((void*)&(p->__pyx_v_cpair)) std::pair (); + new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); + new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); + new((void*)&(p->__pyx_t_0)) std::map ::iterator(); + return o; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o; + PyObject_GC_UnTrack(o); + __Pyx_call_destructor(p->__pyx_v_cpair); + __Pyx_call_destructor(p->__pyx_v_locked); + __Pyx_call_destructor(p->__pyx_v_lockedptr); + __Pyx_call_destructor(p->__pyx_t_0); + Py_CLEAR(p->__pyx_v_self); + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components)))) { + __pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components[__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components++] = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static int __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o; + if (p->__pyx_v_self) { + e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o; + tmp = ((PyObject*)p->__pyx_v_self); + p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.partitioning.__pyx_scope_struct_2_tag_components", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, /*tp_traverse*/ + __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + #if PY_VERSION_HEX < 0x03020000 + { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, + #else + PyModuleDef_HEAD_INIT, + #endif + "partitioning", + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp_s_Can_t_locked_underlying_Componen, __pyx_k_Can_t_locked_underlying_Componen, sizeof(__pyx_k_Can_t_locked_underlying_Componen), 0, 0, 1, 0}, + {&__pyx_kp_s_Can_t_open_file, __pyx_k_Can_t_open_file, sizeof(__pyx_k_Can_t_open_file), 0, 0, 1, 0}, + {&__pyx_n_s_Component___iter, __pyx_k_Component___iter, sizeof(__pyx_k_Component___iter), 0, 0, 1, 1}, + {&__pyx_n_s_Countgraph, __pyx_k_Countgraph, sizeof(__pyx_k_Countgraph), 0, 0, 1, 1}, + {&__pyx_n_s_IOError, __pyx_k_IOError, sizeof(__pyx_k_IOError), 0, 0, 1, 1}, + {&__pyx_n_s_MemoryError, __pyx_k_MemoryError, sizeof(__pyx_k_MemoryError), 0, 0, 1, 1}, + {&__pyx_kp_s_Must_take_an_object_with_Hashtab, __pyx_k_Must_take_an_object_with_Hashtab, sizeof(__pyx_k_Must_take_an_object_with_Hashtab), 0, 0, 1, 0}, + {&__pyx_n_s_Nodegraph, __pyx_k_Nodegraph, sizeof(__pyx_k_Nodegraph), 0, 0, 1, 1}, + {&__pyx_n_s_NotImplementedError, __pyx_k_NotImplementedError, sizeof(__pyx_k_NotImplementedError), 0, 0, 1, 1}, + {&__pyx_kp_s_Operator_not_available, __pyx_k_Operator_not_available, sizeof(__pyx_k_Operator_not_available), 0, 0, 1, 0}, + {&__pyx_n_s_StreamingPartitioner_components, __pyx_k_StreamingPartitioner_components, sizeof(__pyx_k_StreamingPartitioner_components), 0, 0, 1, 1}, + {&__pyx_n_s_StreamingPartitioner_tag_compone, __pyx_k_StreamingPartitioner_tag_compone, sizeof(__pyx_k_StreamingPartitioner_tag_compone), 0, 0, 1, 1}, + {&__pyx_kp_s_Users_camille_w_khmer_khmer__ox, __pyx_k_Users_camille_w_khmer_khmer__ox, sizeof(__pyx_k_Users_camille_w_khmer_khmer__ox), 0, 0, 1, 0}, + {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, + {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, + {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, + {&__pyx_n_s_component, __pyx_k_component, sizeof(__pyx_k_component), 0, 0, 1, 1}, + {&__pyx_n_s_component_id, __pyx_k_component_id, sizeof(__pyx_k_component_id), 0, 0, 1, 1}, + {&__pyx_n_s_components, __pyx_k_components, sizeof(__pyx_k_components), 0, 0, 1, 1}, + {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, + {&__pyx_n_s_graph, __pyx_k_graph, sizeof(__pyx_k_graph), 0, 0, 1, 1}, + {&__pyx_n_s_graph_ptr, __pyx_k_graph_ptr, sizeof(__pyx_k_graph_ptr), 0, 0, 1, 1}, + {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, + {&__pyx_n_s_iter, __pyx_k_iter, sizeof(__pyx_k_iter), 0, 0, 1, 1}, + {&__pyx_n_s_khmer, __pyx_k_khmer, sizeof(__pyx_k_khmer), 0, 0, 1, 1}, + {&__pyx_n_s_khmer__oxli_partitioning, __pyx_k_khmer__oxli_partitioning, sizeof(__pyx_k_khmer__oxli_partitioning), 0, 0, 1, 1}, + {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, + {&__pyx_n_s_other, __pyx_k_other, sizeof(__pyx_k_other), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, + {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, + {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, + {&__pyx_n_s_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 0, 0, 1, 1}, + {&__pyx_n_s_tag_components, __pyx_k_tag_components, sizeof(__pyx_k_tag_components), 0, 0, 1, 1}, + {&__pyx_n_s_tag_counts, __pyx_k_tag_counts, sizeof(__pyx_k_tag_counts), 0, 0, 1, 1}, + {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, + {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, + {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, + {0, 0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_staticmethod = __Pyx_GetBuiltinName(__pyx_n_s_staticmethod); if (!__pyx_builtin_staticmethod) __PYX_ERR(0, 75, __pyx_L1_error) + __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(0, 56, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 94, __pyx_L1_error) + __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 136, __pyx_L1_error) + __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s_IOError); if (!__pyx_builtin_IOError) __PYX_ERR(0, 152, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitCachedConstants(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "khmer/_oxli/partitioning.pyx":56 + * return x.component_id == y.component_id + * else: + * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Operator_not_available); if (unlikely(!__pyx_tuple_)) __PYX_ERR(0, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple_); + __Pyx_GIVEREF(__pyx_tuple_); + + /* "khmer/_oxli/partitioning.pyx":94 + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< + * + * cdef CPyHashtable_Object* ptr = graph + */ + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(0, 94, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__2); + __Pyx_GIVEREF(__pyx_tuple__2); + + /* "khmer/_oxli/partitioning.pyx":106 + * + * def consume_sequence(self, sequence): + * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< + * self.n_consumed += 1 + * + */ + __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__3); + __Pyx_GIVEREF(__pyx_tuple__3); + + /* "khmer/_oxli/partitioning.pyx":110 + * + * def consume_fasta(self, filename): + * return deref(self._this).consume_fasta(filename.encode('utf-8')) # <<<<<<<<<<<<<< + * + * def get_tag_component(self, kmer): + */ + __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__4); + __Pyx_GIVEREF(__pyx_tuple__4); + + /* "khmer/_oxli/partitioning.pyx":114 + * def get_tag_component(self, kmer): + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if compptr == NULL: + * return None + */ + __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__5); + __Pyx_GIVEREF(__pyx_tuple__5); + + /* "khmer/_oxli/partitioning.pyx":122 + * def get_nearest_component(self, kmer): + * cdef ComponentPtr compptr + * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< + * if compptr == NULL: + * return None + */ + __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(0, 122, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__6); + __Pyx_GIVEREF(__pyx_tuple__6); + + /* "khmer/_oxli/partitioning.pyx":136 + * yield Component.create(cmpptr) + * else: + * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< + * + * def tag_components(self): + */ + __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(0, 136, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__7); + __Pyx_GIVEREF(__pyx_tuple__7); + + /* "khmer/_oxli/partitioning.pyx":146 + * yield cpair.first, Component.create(cpair.second) + * else: + * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< + * + * def write_components(self, filename): + */ + __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(0, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); + + /* "khmer/_oxli/partitioning.pyx":150 + * def write_components(self, filename): + * cdef FILE* fp + * fp = fopen(filename.encode('utf-8'), 'wb') # <<<<<<<<<<<<<< + * if fp == NULL: + * raise IOError("Can't open file.") + */ + __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(0, 150, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__9); + __Pyx_GIVEREF(__pyx_tuple__9); + + /* "khmer/_oxli/partitioning.pyx":152 + * fp = fopen(filename.encode('utf-8'), 'wb') + * if fp == NULL: + * raise IOError("Can't open file.") # <<<<<<<<<<<<<< + * + * cdef ComponentPtr cmpptr + */ + __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_s_Can_t_open_file); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); + + /* "khmer/_oxli/partitioning.pyx":76 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CPyHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + __pyx_tuple__11 = PyTuple_Pack(3, __pyx_n_s_component, __pyx_n_s_graph, __pyx_n_s_graph_ptr); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__11); + __Pyx_GIVEREF(__pyx_tuple__11); + __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_camille_w_khmer_khmer__ox, __pyx_n_s_tag_counts, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + __pyx_int_2 = PyInt_FromLong(2); if (unlikely(!__pyx_int_2)) __PYX_ERR(0, 1, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initpartitioning(void); /*proto*/ +PyMODINIT_FUNC initpartitioning(void) +#else +PyMODINIT_FUNC PyInit_partitioning(void); /*proto*/ +PyMODINIT_FUNC PyInit_partitioning(void) +#endif +{ + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannyDeclarations + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_partitioning(void)", 0); + if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_StopAsyncIteration_USED + if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4("partitioning", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_d); + __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + #if CYTHON_COMPILING_IN_PYPY + Py_INCREF(__pyx_b); + #endif + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + if (__pyx_module_is_main_khmer___oxli__partitioning) { + if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + } + #if PY_MAJOR_VERSION >= 3 + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "khmer._oxli.partitioning")) { + if (unlikely(PyDict_SetItemString(modules, "khmer._oxli.partitioning", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #endif + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global init code ---*/ + /*--- Variable export code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + __pyx_vtabptr_5khmer_5_oxli_12partitioning_Component = &__pyx_vtable_5khmer_5_oxli_12partitioning_Component; + __pyx_vtable_5khmer_5_oxli_12partitioning_Component.create = (struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *(*)(khmer::ComponentPtr))__pyx_f_5khmer_5_oxli_12partitioning_9Component_create; + __pyx_vtable_5khmer_5_oxli_12partitioning_Component._tag_counts = (std::vector (*)(khmer::ComponentPtr, khmer::Hashtable *))__pyx_f_5khmer_5_oxli_12partitioning_9Component__tag_counts; + __pyx_vtable_5khmer_5_oxli_12partitioning_Component._mean_tag_count = (float (*)(khmer::ComponentPtr, khmer::Hashtable *))__pyx_f_5khmer_5_oxli_12partitioning_9Component__mean_tag_count; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning_Component) < 0) __PYX_ERR(0, 23, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_12partitioning_Component.tp_print = 0; + if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_12partitioning_Component.tp_dict, __pyx_vtabptr_5khmer_5_oxli_12partitioning_Component) < 0) __PYX_ERR(0, 23, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_12partitioning_Component) < 0) __PYX_ERR(0, 23, __pyx_L1_error) + __pyx_ptype_5khmer_5_oxli_12partitioning_Component = &__pyx_type_5khmer_5_oxli_12partitioning_Component; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner) < 0) __PYX_ERR(0, 90, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner) < 0) __PYX_ERR(0, 90, __pyx_L1_error) + __pyx_ptype_5khmer_5_oxli_12partitioning_StreamingPartitioner = &__pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__) < 0) __PYX_ERR(0, 44, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__.tp_print = 0; + __pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ = &__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components) < 0) __PYX_ERR(0, 128, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components.tp_print = 0; + __pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components = &__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components) < 0) __PYX_ERR(0, 138, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components.tp_print = 0; + __pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components = &__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components; + /*--- Type import code ---*/ + /*--- Variable import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + + /* "khmer/_oxli/partitioning.pyx":20 + * + * from _oxli cimport * + * from .._khmer import Countgraph # <<<<<<<<<<<<<< + * from .._khmer import Nodegraph + * + */ + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_n_s_Countgraph); + __Pyx_GIVEREF(__pyx_n_s_Countgraph); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_Countgraph); + __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer, __pyx_t_1, 2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "khmer/_oxli/partitioning.pyx":21 + * from _oxli cimport * + * from .._khmer import Countgraph + * from .._khmer import Nodegraph # <<<<<<<<<<<<<< + * + * cdef class Component: + */ + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_n_s_Nodegraph); + __Pyx_GIVEREF(__pyx_n_s_Nodegraph); + PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Nodegraph); + __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer, __pyx_t_2, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "khmer/_oxli/partitioning.pyx":76 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CPyHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5khmer_5_oxli_12partitioning_9Component_12tag_counts, NULL, __pyx_n_s_khmer__oxli_partitioning); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + + /* "khmer/_oxli/partitioning.pyx":75 + * return counts + * + * @staticmethod # <<<<<<<<<<<<<< + * def tag_counts(Component component, graph): + * cdef CPyHashtable_Object* graph_ptr = graph + */ + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_staticmethod, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5khmer_5_oxli_12partitioning_Component->tp_dict, __pyx_n_s_tag_counts, __pyx_t_1) < 0) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5khmer_5_oxli_12partitioning_Component); + + /* "khmer/_oxli/partitioning.pyx":76 + * + * @staticmethod + * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< + * cdef CPyHashtable_Object* graph_ptr = graph + * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) + */ + __pyx_t_1 = __Pyx_GetNameInClass((PyObject *)__pyx_ptype_5khmer_5_oxli_12partitioning_Component, __pyx_n_s_tag_counts); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + + /* "khmer/_oxli/partitioning.pyx":75 + * return counts + * + * @staticmethod # <<<<<<<<<<<<<< + * def tag_counts(Component component, graph): + * cdef CPyHashtable_Object* graph_ptr = graph + */ + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_staticmethod, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5khmer_5_oxli_12partitioning_Component->tp_dict, __pyx_n_s_tag_counts, __pyx_t_1) < 0) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5khmer_5_oxli_12partitioning_Component); + + /* "khmer/_oxli/partitioning.pyx":1 + * import cython # <<<<<<<<<<<<<< + * from cython.operator cimport dereference as deref, preincrement as inc + * + */ + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + if (__pyx_m) { + if (__pyx_d) { + __Pyx_AddTraceback("init khmer._oxli.partitioning", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + Py_DECREF(__pyx_m); __pyx_m = 0; + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init khmer._oxli.partitioning"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); + if (unlikely(!result)) { + PyErr_Format(PyExc_NameError, +#if PY_MAJOR_VERSION >= 3 + "name '%U' is not defined", name); +#else + "name '%.200s' is not defined", PyString_AS_STRING(name)); +#endif + } + return result; +} + +/* RaiseDoubleKeywords */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AsString(kw_name)); + #endif +} + +/* ParseKeywords */ +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + continue; + } + name = first_kw_arg; + #if PY_MAJOR_VERSION < 3 + if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + while (*name) { + if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) + && _PyString_Eq(**name, key)) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + if ((**argname == key) || ( + (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) + && _PyString_Eq(**argname, key))) { + goto arg_passed_twice; + } + argname++; + } + } + } else + #endif + if (likely(PyUnicode_Check(key))) { + while (*name) { + int cmp = (**name == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**name, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + int cmp = (**argname == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**argname, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + argname++; + } + } + } else + goto invalid_keyword_type; + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTest */ +static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); +} +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact) +{ + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (none_allowed && obj == Py_None) return 1; + else if (exact) { + if (likely(Py_TYPE(obj) == type)) return 1; + #if PY_MAJOR_VERSION == 2 + else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + #endif + } + else { + if (likely(PyObject_TypeCheck(obj, type))) return 1; + } + __Pyx_RaiseArgumentTypeInvalid(name, obj, type); + return 0; +} + +/* PyIntBinop */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { + if (op1 == op2) { + Py_RETURN_TRUE; + } + #if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(op1))) { + const long b = intval; + long a = PyInt_AS_LONG(op1); + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + const long b = intval; + long a; + const digit* digits = ((PyLongObject*)op1)->ob_digit; + const Py_ssize_t size = Py_SIZE(op1); + if (likely(__Pyx_sst_abs(size) <= 1)) { + a = likely(size) ? digits[0] : 0; + if (size == -1) a = -a; + } else { + switch (size) { + case -2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 + default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); + #else + default: Py_RETURN_FALSE; + #endif + } + } + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = PyFloat_AS_DOUBLE(op1); + if ((double)a == (double)b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + return PyObject_RichCompare(op1, op2, Py_EQ); +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} +#endif + +/* RaiseException */ +#if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + __Pyx_PyThreadState_declare + Py_XINCREF(type); + if (!value || value == Py_None) + value = NULL; + else + Py_INCREF(value); + if (!tb || tb == Py_None) + tb = NULL; + else { + Py_INCREF(tb); + if (!PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + } + if (PyType_Check(type)) { +#if CYTHON_COMPILING_IN_PYPY + if (!value) { + Py_INCREF(Py_None); + value = Py_None; + } +#endif + PyErr_NormalizeException(&type, &value, &tb); + } else { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + value = type; + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + } + __Pyx_PyThreadState_assign + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } +#if PY_VERSION_HEX >= 0x03030000 + if (cause) { +#else + if (cause && cause != Py_None) { +#endif + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = PyThreadState_GET(); + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} +#endif + +/* RaiseTooManyValuesToUnpack */ + static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* RaiseNeedMoreValuesToUnpack */ + static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* IterFinish */ + static CYTHON_INLINE int __Pyx_IterFinish(void) { +#if CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = PyThreadState_GET(); + PyObject* exc_type = tstate->curexc_type; + if (unlikely(exc_type)) { + if (likely(exc_type == PyExc_StopIteration) || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) { + PyObject *exc_value, *exc_tb; + exc_value = tstate->curexc_value; + exc_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + Py_DECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + return 0; + } else { + return -1; + } + } + return 0; +#else + if (unlikely(PyErr_Occurred())) { + if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) { + PyErr_Clear(); + return 0; + } else { + return -1; + } + } + return 0; +#endif +} + +/* UnpackItemEndCheck */ + static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { + if (unlikely(retval)) { + Py_DECREF(retval); + __Pyx_RaiseTooManyValuesError(expected); + return -1; + } else { + return __Pyx_IterFinish(); + } + return 0; +} + +/* WriteUnraisableException */ + static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, + CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, + int full_traceback, CYTHON_UNUSED int nogil) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_PyThreadState_declare +#ifdef WITH_THREAD + PyGILState_STATE state; + if (nogil) + state = PyGILState_Ensure(); +#ifdef _MSC_VER + else state = (PyGILState_STATE)-1; +#endif +#endif + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + if (full_traceback) { + Py_XINCREF(old_exc); + Py_XINCREF(old_val); + Py_XINCREF(old_tb); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + PyErr_PrintEx(1); + } + #if PY_MAJOR_VERSION < 3 + ctx = PyString_FromString(name); + #else + ctx = PyUnicode_FromString(name); + #endif + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } +#ifdef WITH_THREAD + if (nogil) + PyGILState_Release(state); +#endif +} + +/* GetModuleGlobalName */ + static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + +/* SetVTable */ + static int __Pyx_SetVtable(PyObject *dict, void *vtable) { +#if PY_VERSION_HEX >= 0x02070000 + PyObject *ob = PyCapsule_New(vtable, 0, 0); +#else + PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); +#endif + if (!ob) + goto bad; + if (PyDict_SetItem(dict, __pyx_n_s_pyx_vtable, ob) < 0) + goto bad; + Py_DECREF(ob); + return 0; +bad: + Py_XDECREF(ob); + return -1; +} + +/* Import */ + static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_import; + py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); + if (!py_import) + goto bad; + #endif + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + { + #if PY_MAJOR_VERSION >= 3 + if (level == -1) { + if (strchr(__Pyx_MODULE_NAME, '.')) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(1); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, 1); + #endif + if (!module) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + #endif + if (!module) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(level); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, level); + #endif + } + } +bad: + #if PY_VERSION_HEX < 0x03030000 + Py_XDECREF(py_import); + #endif + Py_XDECREF(empty_list); + Py_XDECREF(empty_dict); + return module; +} + +/* ImportFrom */ + static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Format(PyExc_ImportError, + #if PY_MAJOR_VERSION < 3 + "cannot import name %.230s", PyString_AS_STRING(name)); + #else + "cannot import name %S", name); + #endif + } + return value; +} + +/* GetNameInClass */ + static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name) { + PyObject *result; + result = __Pyx_PyObject_GetAttrStr(nmspace, name); + if (!result) + result = __Pyx_GetModuleGlobalName(name); + return result; +} + +/* CodeObjectCache */ + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); +} + +/* AddTraceback */ + #include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, + 0, + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + py_code = __pyx_find_code_object(c_line ? c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + } + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + } +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value) { + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(uint64_t) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(uint64_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(uint64_t) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(uint64_t), + little, !is_unsigned); + } +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value) { + const khmer::HashIntoType neg_one = (khmer::HashIntoType) -1, const_zero = (khmer::HashIntoType) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(khmer::HashIntoType) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(khmer::HashIntoType) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::HashIntoType) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(khmer::HashIntoType), + little, !is_unsigned); + } +} + +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType(khmer::BoundedCounterType value) { + const khmer::BoundedCounterType neg_one = (khmer::BoundedCounterType) -1, const_zero = (khmer::BoundedCounterType) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(khmer::BoundedCounterType) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(khmer::BoundedCounterType) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::BoundedCounterType) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(khmer::BoundedCounterType) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(khmer::BoundedCounterType) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(khmer::BoundedCounterType), + little, !is_unsigned); + } +} + +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case -2: + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int) -1; + } + } else { + int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *x) { + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(uint64_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (uint64_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (uint64_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, digits[0]) + case 2: + if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 2 * PyLong_SHIFT) { + return (uint64_t) (((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 3 * PyLong_SHIFT) { + return (uint64_t) (((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 4 * PyLong_SHIFT) { + return (uint64_t) (((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (uint64_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(uint64_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (uint64_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(uint64_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, +digits[0]) + case -2: + if (8 * sizeof(uint64_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { + return (uint64_t) ((((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { + return (uint64_t) ((((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { + return (uint64_t) ((((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + } +#endif + if (sizeof(uint64_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + uint64_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (uint64_t) -1; + } + } else { + uint64_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (uint64_t) -1; + val = __Pyx_PyInt_As_uint64_t(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to uint64_t"); + return (uint64_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to uint64_t"); + return (uint64_t) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (size_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) + case -2: + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } +#endif + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + size_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (size_t) -1; + } + } else { + size_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case -2: + if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (long) -1; + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (long) -1; + val = __Pyx_PyInt_As_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* FetchCommonType */ + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* fake_module; + PyTypeObject* cached_type = NULL; + fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); + if (!fake_module) return NULL; + Py_INCREF(fake_module); + cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + if (cached_type) { + if (!PyType_Check((PyObject*)cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", + type->tp_name); + goto bad; + } + if (cached_type->tp_basicsize != type->tp_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + type->tp_name); + goto bad; + } + } else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + } +done: + Py_DECREF(fake_module); + return cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* SwapException */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* PyCFunctionFastCall */ + #if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + PyObject *result; + int flags; + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); +} +#endif // CYTHON_FAST_PYCCALL + +/* PyFunctionFastCall */ + #if CYTHON_FAST_PYCALL +#include "frameobject.h" +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = PyThreadState_GET(); + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif // CPython < 3.6 +#endif // CYTHON_FAST_PYCALL + +/* PyObjectCallMethO */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { +#else + if (likely(PyCFunction_Check(func))) { +#endif + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif + +/* PyObjectCallMethod1 */ + static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { + PyObject *method, *result = NULL; + method = __Pyx_PyObject_GetAttrStr(obj, method_name); + if (unlikely(!method)) goto done; +#if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(method))) { + PyObject *self = PyMethod_GET_SELF(method); + if (likely(self)) { + PyObject *args; + PyObject *function = PyMethod_GET_FUNCTION(method); + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(function)) { + PyObject *args[2] = {self, arg}; + result = __Pyx_PyFunction_FastCall(function, args, 2); + goto done; + } + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(function)) { + PyObject *args[2] = {self, arg}; + result = __Pyx_PyCFunction_FastCall(function, args, 2); + goto done; + } + #endif + args = PyTuple_New(2); + if (unlikely(!args)) goto done; + Py_INCREF(self); + PyTuple_SET_ITEM(args, 0, self); + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 1, arg); + Py_INCREF(function); + Py_DECREF(method); method = NULL; + result = __Pyx_PyObject_Call(function, args, NULL); + Py_DECREF(args); + Py_DECREF(function); + return result; + } + } +#endif + result = __Pyx_PyObject_CallOneArg(method, arg); +done: + Py_XDECREF(method); + return result; +} + +/* CoroutineBase */ + #include +#include +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); +static PyObject *__Pyx_Coroutine_Close(PyObject *self); +static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); +#define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) +#if 1 || PY_VERSION_HEX < 0x030300B0 +static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { + PyObject *et, *ev, *tb; + PyObject *value = NULL; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&et, &ev, &tb); + if (!et) { + Py_XDECREF(tb); + Py_XDECREF(ev); + Py_INCREF(Py_None); + *pvalue = Py_None; + return 0; + } + if (likely(et == PyExc_StopIteration)) { + if (!ev) { + Py_INCREF(Py_None); + value = Py_None; + } +#if PY_VERSION_HEX >= 0x030300A0 + else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + Py_DECREF(ev); + } +#endif + else if (unlikely(PyTuple_Check(ev))) { + if (PyTuple_GET_SIZE(ev) >= 1) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + value = PyTuple_GET_ITEM(ev, 0); + Py_INCREF(value); +#else + value = PySequence_ITEM(ev, 0); +#endif + } else { + Py_INCREF(Py_None); + value = Py_None; + } + Py_DECREF(ev); + } + else if (!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { + value = ev; + } + if (likely(value)) { + Py_XDECREF(tb); + Py_DECREF(et); + *pvalue = value; + return 0; + } + } else if (!PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + PyErr_NormalizeException(&et, &ev, &tb); + if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + Py_XDECREF(tb); + Py_DECREF(et); +#if PY_VERSION_HEX >= 0x030300A0 + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + Py_DECREF(ev); +#else + { + PyObject* args = __Pyx_PyObject_GetAttrStr(ev, __pyx_n_s_args); + Py_DECREF(ev); + if (likely(args)) { + value = PySequence_GetItem(args, 0); + Py_DECREF(args); + } + if (unlikely(!value)) { + __Pyx_ErrRestore(NULL, NULL, NULL); + Py_INCREF(Py_None); + value = Py_None; + } + } +#endif + *pvalue = value; + return 0; +} +#endif +static CYTHON_INLINE +void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { + PyObject *exc_type = self->exc_type; + PyObject *exc_value = self->exc_value; + PyObject *exc_traceback = self->exc_traceback; + self->exc_type = NULL; + self->exc_value = NULL; + self->exc_traceback = NULL; + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_traceback); +} +static CYTHON_INLINE +int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) { + if (unlikely(gen->is_running)) { + PyErr_SetString(PyExc_ValueError, + "generator already executing"); + return 1; + } + return 0; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) { + PyObject *retval; + __Pyx_PyThreadState_declare + assert(!self->is_running); + if (unlikely(self->resume_label == 0)) { + if (unlikely(value && value != Py_None)) { + PyErr_SetString(PyExc_TypeError, + "can't send non-None value to a " + "just-started generator"); + return NULL; + } + } + if (unlikely(self->resume_label == -1)) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + __Pyx_PyThreadState_assign + if (value) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#else + if (self->exc_traceback) { + PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyFrameObject *f = tb->tb_frame; + Py_XINCREF(__pyx_tstate->frame); + assert(f->f_back == NULL); + f->f_back = __pyx_tstate->frame; + } +#endif + __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, + &self->exc_traceback); + } else { + __Pyx_Coroutine_ExceptionClear(self); + } + self->is_running = 1; + retval = self->body((PyObject *) self, value); + self->is_running = 0; + if (retval) { + __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, + &self->exc_traceback); +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#else + if (self->exc_traceback) { + PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyFrameObject *f = tb->tb_frame; + Py_CLEAR(f->f_back); + } +#endif + } else { + __Pyx_Coroutine_ExceptionClear(self); + } + return retval; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) { + if (unlikely(!retval && !PyErr_Occurred())) { + PyErr_SetNone(PyExc_StopIteration); + } + return retval; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { + PyObject *ret; + PyObject *val = NULL; + __Pyx_Coroutine_Undelegate(gen); + __Pyx_PyGen_FetchStopIterationValue(&val); + ret = __Pyx_Coroutine_SendEx(gen, val); + Py_XDECREF(val); + return ret; +} +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { + PyObject *retval; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + PyObject *yf = gen->yieldfrom; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + { + if (value == Py_None) + ret = Py_TYPE(yf)->tp_iternext(yf); + else + ret = __Pyx_PyObject_CallMethod1(yf, __pyx_n_s_send, value); + } + gen->is_running = 0; + if (likely(ret)) { + return ret; + } + retval = __Pyx_Coroutine_FinishDelegation(gen); + } else { + retval = __Pyx_Coroutine_SendEx(gen, value); + } + return __Pyx_Coroutine_MethodReturn(retval); +} +static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { + PyObject *retval = NULL; + int err = 0; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + retval = __Pyx_Coroutine_Close(yf); + if (!retval) + return -1; + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + retval = __Pyx_Coroutine_Close(yf); + if (!retval) + return -1; + } else + #endif + { + PyObject *meth; + gen->is_running = 1; + meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_close); + if (unlikely(!meth)) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_WriteUnraisable(yf); + } + PyErr_Clear(); + } else { + retval = PyObject_CallFunction(meth, NULL); + Py_DECREF(meth); + if (!retval) + err = -1; + } + gen->is_running = 0; + } + Py_XDECREF(retval); + return err; +} +static PyObject *__Pyx_Generator_Next(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + PyObject *yf = gen->yieldfrom; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Generator_Next(yf); + } else + #endif + ret = Py_TYPE(yf)->tp_iternext(yf); + gen->is_running = 0; + if (likely(ret)) { + return ret; + } + return __Pyx_Coroutine_FinishDelegation(gen); + } + return __Pyx_Coroutine_SendEx(gen, Py_None); +} +static PyObject *__Pyx_Coroutine_Close(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject *retval, *raised_exception; + PyObject *yf = gen->yieldfrom; + int err = 0; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + Py_INCREF(yf); + err = __Pyx_Coroutine_CloseIter(gen, yf); + __Pyx_Coroutine_Undelegate(gen); + Py_DECREF(yf); + } + if (err == 0) + PyErr_SetNone(PyExc_GeneratorExit); + retval = __Pyx_Coroutine_SendEx(gen, NULL); + if (retval) { + Py_DECREF(retval); + PyErr_SetString(PyExc_RuntimeError, + "generator ignored GeneratorExit"); + return NULL; + } + raised_exception = PyErr_Occurred(); + if (!raised_exception + || raised_exception == PyExc_StopIteration + || raised_exception == PyExc_GeneratorExit + || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) + || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) + { + if (raised_exception) PyErr_Clear(); + Py_INCREF(Py_None); + return Py_None; + } + return NULL; +} +static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject *typ; + PyObject *tb = NULL; + PyObject *val = NULL; + PyObject *yf = gen->yieldfrom; + if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) + return NULL; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + Py_INCREF(yf); + if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + int err = __Pyx_Coroutine_CloseIter(gen, yf); + Py_DECREF(yf); + __Pyx_Coroutine_Undelegate(gen); + if (err < 0) + return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); + goto throw_here; + } + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Coroutine_Throw(yf, args); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + ret = __Pyx_Coroutine_Throw(yf, args); + } else + #endif + { + PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_throw); + if (unlikely(!meth)) { + Py_DECREF(yf); + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + gen->is_running = 0; + return NULL; + } + PyErr_Clear(); + __Pyx_Coroutine_Undelegate(gen); + gen->is_running = 0; + goto throw_here; + } + ret = PyObject_CallObject(meth, args); + Py_DECREF(meth); + } + gen->is_running = 0; + Py_DECREF(yf); + if (!ret) { + ret = __Pyx_Coroutine_FinishDelegation(gen); + } + return __Pyx_Coroutine_MethodReturn(ret); + } +throw_here: + __Pyx_Raise(typ, val, tb, NULL); + return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); +} +static int __Pyx_Coroutine_traverse(PyObject *self, visitproc visit, void *arg) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + Py_VISIT(gen->closure); + Py_VISIT(gen->classobj); + Py_VISIT(gen->yieldfrom); + Py_VISIT(gen->exc_type); + Py_VISIT(gen->exc_value); + Py_VISIT(gen->exc_traceback); + return 0; +} +static int __Pyx_Coroutine_clear(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + Py_CLEAR(gen->closure); + Py_CLEAR(gen->classobj); + Py_CLEAR(gen->yieldfrom); + Py_CLEAR(gen->exc_type); + Py_CLEAR(gen->exc_value); + Py_CLEAR(gen->exc_traceback); + Py_CLEAR(gen->gi_name); + Py_CLEAR(gen->gi_qualname); + return 0; +} +static void __Pyx_Coroutine_dealloc(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject_GC_UnTrack(gen); + if (gen->gi_weakreflist != NULL) + PyObject_ClearWeakRefs(self); + if (gen->resume_label > 0) { + PyObject_GC_Track(self); +#if PY_VERSION_HEX >= 0x030400a1 + if (PyObject_CallFinalizerFromDealloc(self)) +#else + Py_TYPE(gen)->tp_del(self); + if (self->ob_refcnt > 0) +#endif + { + return; + } + PyObject_GC_UnTrack(self); + } + __Pyx_Coroutine_clear(self); + PyObject_GC_Del(gen); +} +static void __Pyx_Coroutine_del(PyObject *self) { + PyObject *res; + PyObject *error_type, *error_value, *error_traceback; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + __Pyx_PyThreadState_declare + if (gen->resume_label <= 0) + return ; +#if PY_VERSION_HEX < 0x030400a1 + assert(self->ob_refcnt == 0); + self->ob_refcnt = 1; +#endif + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); + res = __Pyx_Coroutine_Close(self); + if (res == NULL) + PyErr_WriteUnraisable(self); + else + Py_DECREF(res); + __Pyx_ErrRestore(error_type, error_value, error_traceback); +#if PY_VERSION_HEX < 0x030400a1 + assert(self->ob_refcnt > 0); + if (--self->ob_refcnt == 0) { + return; + } + { + Py_ssize_t refcnt = self->ob_refcnt; + _Py_NewReference(self); + self->ob_refcnt = refcnt; + } +#if CYTHON_COMPILING_IN_CPYTHON + assert(PyType_IS_GC(self->ob_type) && + _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); + _Py_DEC_REFTOTAL; +#endif +#ifdef COUNT_ALLOCS + --Py_TYPE(self)->tp_frees; + --Py_TYPE(self)->tp_allocs; +#endif +#endif +} +static PyObject * +__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self) +{ + PyObject *name = self->gi_name; + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + tmp = self->gi_name; + Py_INCREF(value); + self->gi_name = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self) +{ + PyObject *name = self->gi_qualname; + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + tmp = self->gi_qualname; + Py_INCREF(value); + self->gi_qualname = value; + Py_XDECREF(tmp); + return 0; +} +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { + __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); + if (gen == NULL) + return NULL; + gen->body = body; + gen->closure = closure; + Py_XINCREF(closure); + gen->is_running = 0; + gen->resume_label = 0; + gen->classobj = NULL; + gen->yieldfrom = NULL; + gen->exc_type = NULL; + gen->exc_value = NULL; + gen->exc_traceback = NULL; + gen->gi_weakreflist = NULL; + Py_XINCREF(qualname); + gen->gi_qualname = qualname; + Py_XINCREF(name); + gen->gi_name = name; + Py_XINCREF(module_name); + gen->gi_modulename = module_name; + PyObject_GC_Track(gen); + return gen; +} + +/* PatchModuleWithCoroutine */ + static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code) { +#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + int result; + PyObject *globals, *result_obj; + globals = PyDict_New(); if (unlikely(!globals)) goto ignore; + result = PyDict_SetItemString(globals, "_cython_coroutine_type", + #ifdef __Pyx_Coroutine_USED + (PyObject*)__pyx_CoroutineType); + #else + Py_None); + #endif + if (unlikely(result < 0)) goto ignore; + result = PyDict_SetItemString(globals, "_cython_generator_type", + #ifdef __Pyx_Generator_USED + (PyObject*)__pyx_GeneratorType); + #else + Py_None); + #endif + if (unlikely(result < 0)) goto ignore; + if (unlikely(PyDict_SetItemString(globals, "_module", module) < 0)) goto ignore; + if (unlikely(PyDict_SetItemString(globals, "__builtins__", __pyx_b) < 0)) goto ignore; + result_obj = PyRun_String(py_code, Py_file_input, globals, globals); + if (unlikely(!result_obj)) goto ignore; + Py_DECREF(result_obj); + Py_DECREF(globals); + return module; +ignore: + Py_XDECREF(globals); + PyErr_WriteUnraisable(module); + if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, "Cython module failed to patch module with custom type", 1) < 0)) { + Py_DECREF(module); + module = NULL; + } +#else + py_code++; +#endif + return module; +} + +/* PatchGeneratorABC */ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) +static PyObject* __Pyx_patch_abc_module(PyObject *module); +static PyObject* __Pyx_patch_abc_module(PyObject *module) { + module = __Pyx_Coroutine_patch_module( + module, "" +"if _cython_generator_type is not None:\n" +" try: Generator = _module.Generator\n" +" except AttributeError: pass\n" +" else: Generator.register(_cython_generator_type)\n" +"if _cython_coroutine_type is not None:\n" +" try: Coroutine = _module.Coroutine\n" +" except AttributeError: pass\n" +" else: Coroutine.register(_cython_coroutine_type)\n" + ); + return module; +} +#endif +static int __Pyx_patch_abc(void) { +#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + static int abc_patched = 0; + if (!abc_patched) { + PyObject *module; + module = PyImport_ImportModule((PY_VERSION_HEX >= 0x03030000) ? "collections.abc" : "collections"); + if (!module) { + PyErr_WriteUnraisable(NULL); + if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, + ((PY_VERSION_HEX >= 0x03030000) ? + "Cython module failed to register with collections.abc module" : + "Cython module failed to register with collections module"), 1) < 0)) { + return -1; + } + } else { + module = __Pyx_patch_abc_module(module); + abc_patched = 1; + if (unlikely(!module)) + return -1; + Py_DECREF(module); + } + module = PyImport_ImportModule("backports_abc"); + if (module) { + module = __Pyx_patch_abc_module(module); + Py_XDECREF(module); + } + if (!module) { + PyErr_Clear(); + } + } +#else + if (0) __Pyx_Coroutine_patch_module(NULL, NULL); +#endif + return 0; +} + +/* Generator */ + static PyMethodDef __pyx_Generator_methods[] = { + {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, + (char*) PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, + {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, + (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, + {"close", (PyCFunction) __Pyx_Coroutine_Close, METH_NOARGS, + (char*) PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, + {0, 0, 0, 0} +}; +static PyMemberDef __pyx_Generator_memberlist[] = { + {(char *) "gi_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, + {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, + (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, + {0, 0, 0, 0, 0} +}; +static PyGetSetDef __pyx_Generator_getsets[] = { + {(char *) "__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, + (char*) PyDoc_STR("name of the generator"), 0}, + {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, + (char*) PyDoc_STR("qualified name of the generator"), 0}, + {0, 0, 0, 0, 0} +}; +static PyTypeObject __pyx_GeneratorType_type = { + PyVarObject_HEAD_INIT(0, 0) + "generator", + sizeof(__pyx_CoroutineObject), + 0, + (destructor) __Pyx_Coroutine_dealloc, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, + 0, + (traverseproc) __Pyx_Coroutine_traverse, + 0, + 0, + offsetof(__pyx_CoroutineObject, gi_weakreflist), + 0, + (iternextfunc) __Pyx_Generator_Next, + __pyx_Generator_methods, + __pyx_Generator_memberlist, + __pyx_Generator_getsets, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, +#if PY_VERSION_HEX >= 0x030400a1 + 0, +#else + __Pyx_Coroutine_del, +#endif + 0, +#if PY_VERSION_HEX >= 0x030400a1 + __Pyx_Coroutine_del, +#endif +}; +static int __pyx_Generator_init(void) { + __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; + __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; + __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); + if (unlikely(!__pyx_GeneratorType)) { + return -1; + } + return 0; +} + +/* CheckBinaryVersion */ + static int __Pyx_check_binary_version(void) { + char ctversion[4], rtversion[4]; + PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); + PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); + if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compiletime version %s of module '%.100s' " + "does not match runtime version %s", + ctversion, __Pyx_MODULE_NAME, rtversion); + return PyErr_WarnEx(NULL, message, 1); + } + return 0; +} + +/* InitStrings */ + static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else + if (t->is_unicode | t->is_str) { + if (t->intern) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); + } else { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { +#if PY_VERSION_HEX < 0x03030000 + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; + } + } + } +#endif + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +#else + if (__Pyx_PyUnicode_READY(o) == -1) return NULL; +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (PyUnicode_IS_ASCII(o)) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif + } else +#endif +#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) + if (PyByteArray_Check(o)) { + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); + } else +#endif + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + const char *name = NULL; + PyObject *res = NULL; +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + #if PY_MAJOR_VERSION < 3 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } + #else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } + #endif +#else + res = PyNumber_Int(x); +#endif + if (res) { +#if PY_MAJOR_VERSION < 3 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(b))) { + if (sizeof(Py_ssize_t) >= sizeof(long)) + return PyInt_AS_LONG(b); + else + return PyInt_AsSsize_t(x); + } +#endif + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)b)->ob_digit; + const Py_ssize_t size = Py_SIZE(b); + if (likely(__Pyx_sst_abs(size) <= 1)) { + ival = likely(size) ? digits[0] : 0; + if (size == -1) ival = -ival; + return ival; + } else { + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { + return PyInt_FromSize_t(ival); +} + + +#endif /* Py_PYTHON_H */ diff --git a/khmer/_oxli/traversal.cpp b/khmer/_oxli/traversal.cpp new file mode 100644 index 0000000000..b859552da1 --- /dev/null +++ b/khmer/_oxli/traversal.cpp @@ -0,0 +1,4360 @@ +/* Generated by Cython 0.25.1 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) + #error Cython requires Python 2.6+ or Python 3.2+. +#else +#define CYTHON_ABI "0_25_1" +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#ifndef HAVE_LONG_LONG + #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) + #define HAVE_LONG_LONG + #endif +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#ifdef PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#elif defined(PYSTON_VERSION) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #if PY_MAJOR_VERSION < 3 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #ifndef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if PY_VERSION_HEX < 0x030300F0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif +#endif +#if !defined(CYTHON_FAST_PYCCALL) +#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #include "longintrepr.h" + #undef SHIFT + #undef BASE + #undef MASK +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) + #define Py_OptimizeFlag 0 +#endif +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyClass_Type +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #define __Pyx_PyCFunctionFast _PyCFunctionFast +#endif +#if CYTHON_FAST_PYCCALL +#define __Pyx_PyFastCFunction_Check(func)\ + ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) +#else +#define __Pyx_PyFastCFunction_Check(func) 0 +#endif +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) +#else + #define CYTHON_PEP393_ENABLED 0 + #define PyUnicode_1BYTE_KIND 1 + #define PyUnicode_2BYTE_KIND 2 + #define PyUnicode_4BYTE_KIND 4 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) + #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_PYSTON + #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) +#else + #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) +#endif +#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) + #define PyObject_ASCII(o) PyObject_Repr(o) +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyStringObject PyUnicodeObject + #define PyString_Type PyUnicode_Type + #define PyString_Check PyUnicode_Check + #define PyString_CheckExact PyUnicode_CheckExact +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) + #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) +#else + #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) + #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#endif +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#if PY_MAJOR_VERSION >= 3 + #define PyIntObject PyLongObject + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define PyNumber_Int PyNumber_Long +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBoolObject PyLongObject +#endif +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY + #ifndef PyUnicode_InternFromString + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) + #endif +#endif +#if PY_VERSION_HEX < 0x030200A4 + typedef long Py_hash_t; + #define __Pyx_PyInt_FromHash_t PyInt_FromLong + #define __Pyx_PyInt_AsHash_t PyInt_AsLong +#else + #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t + #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) +#else + #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) +#endif +#if CYTHON_USE_ASYNC_SLOTS + #if PY_VERSION_HEX >= 0x030500B1 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods + #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) + #else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) + +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #define CYTHON_INLINE inline +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(U other) { return *ptr == other; }; + template bool operator !=(U other) { return *ptr != other; }; + private: + T *ptr; +}; + +#if defined(WIN32) || defined(MS_WINDOWS) + #define _USE_MATH_DEFINES +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + + +#define __PYX_ERR(f_index, lineno, Ln_error) \ +{ \ + __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ +} + +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#define __PYX_HAVE__khmer___oxli__traversal +#define __PYX_HAVE_API__khmer___oxli__traversal +#include +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include +#include +#include +#include +#include +#include +#include +#include "khmer.hh" +#include "kmer_hash.hh" +#include "_khmer.hh" +#include "hashtable.hh" +#include "traversal.hh" +#include "partitioning.hh" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#ifdef PYREX_WITHOUT_ASSERTIONS +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; + const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) && defined (_M_X64) + #define __Pyx_sst_abs(value) _abs64(value) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#else + #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize +#endif +#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#if PY_MAJOR_VERSION < 3 +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ + const Py_UNICODE *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen +#endif +#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) +#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) +#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) +#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +#if CYTHON_ASSUME_SAFE_MACROS +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#else +#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#endif +#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#else +#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) +#endif +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII +static int __Pyx_sys_getdefaultencoding_not_ascii; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + PyObject* ascii_chars_u = NULL; + PyObject* ascii_chars_b = NULL; + const char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + if (strcmp(default_encoding_c, "ascii") == 0) { + __Pyx_sys_getdefaultencoding_not_ascii = 0; + } else { + char ascii_chars[128]; + int c; + for (c = 0; c < 128; c++) { + ascii_chars[c] = c; + } + __Pyx_sys_getdefaultencoding_not_ascii = 1; + ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); + if (!ascii_chars_u) goto bad; + ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); + if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { + PyErr_Format( + PyExc_ValueError, + "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", + default_encoding_c); + goto bad; + } + Py_DECREF(ascii_chars_u); + Py_DECREF(ascii_chars_b); + } + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return -1; +} +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#else +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +static char* __PYX_DEFAULT_STRING_ENCODING; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; + strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + return -1; +} +#endif +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_d; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static PyObject *__pyx_empty_bytes; +static PyObject *__pyx_empty_unicode; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; + + +static const char *__pyx_f[] = { + "khmer/_oxli/traversal.pyx", + "khmer/_oxli/hashing.pxd", +}; + +/*--- Type declarations ---*/ +struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer; +struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser; +struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors; + +/* "hashing.pxd":4 + * from _oxli cimport CpKmer, HashIntoType, WordLength + * + * cdef class Kmer: # <<<<<<<<<<<<<< + * cdef shared_ptr[CpKmer] _this + * cdef readonly str kmer + */ +struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer { + PyObject_HEAD + struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer *__pyx_vtab; + std::shared_ptr _this; + PyObject *kmer; +}; + + +/* "khmer/_oxli/traversal.pxd":5 + * from _oxli cimport CpTraverser, CpHashtable + * + * cdef class Traverser: # <<<<<<<<<<<<<< + * cdef unique_ptr[CpTraverser] _this + * cdef CpHashtable * _graph_ptr + */ +struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser { + PyObject_HEAD + std::unique_ptr _this; + khmer::Hashtable *_graph_ptr; +}; + + +/* "khmer/_oxli/traversal.pyx":30 + * ''' + * + * def right_neighbors(self, hashing.Kmer kmer): # <<<<<<<<<<<<<< + * cdef KmerQueue kmer_q + * cdef CpKmer neighbor + */ +struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors { + PyObject_HEAD + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer; + khmer::KmerQueue __pyx_v_kmer_q; + khmer::Kmer __pyx_v_neighbor; + khmer::Kmer *__pyx_v_ptr; + struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self; +}; + + + +/* "hashing.pxd":4 + * from _oxli cimport CpKmer, HashIntoType, WordLength + * + * cdef class Kmer: # <<<<<<<<<<<<<< + * cdef shared_ptr[CpKmer] _this + * cdef readonly str kmer + */ + +struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer { + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*wrap)(khmer::Kmer *, khmer::WordLength); + struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*create)(khmer::HashIntoType, khmer::WordLength); +}; +static struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer *__pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer; + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*SetupContext)(const char*, int, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; +#ifdef WITH_THREAD + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + } +#else + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) +#endif + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ + const char* function_name); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* GetModuleGlobalName.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* ArgTypeTest.proto */ +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact); + +/* IncludeStringH.proto */ +#include + +/* GetVTable.proto */ +static void* __Pyx_GetVtable(PyObject *dict); + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* CodeObjectCache.proto */ +typedef struct { + PyCodeObject* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; +}; +static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static PyCodeObject *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* None.proto */ +#include + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectCallMethod1.proto */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* CoroutineBase.proto */ +typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); +typedef struct { + PyObject_HEAD + __pyx_coroutine_body_t body; + PyObject *closure; + PyObject *exc_type; + PyObject *exc_value; + PyObject *exc_traceback; + PyObject *gi_weakreflist; + PyObject *classobj; + PyObject *yieldfrom; + PyObject *gi_name; + PyObject *gi_qualname; + PyObject *gi_modulename; + int resume_label; + char is_running; +} __pyx_CoroutineObject; +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); +static int __Pyx_Coroutine_clear(PyObject *self); +#if 1 || PY_VERSION_HEX < 0x030300B0 +static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); +#else +#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) +#endif + +/* PatchModuleWithCoroutine.proto */ +static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code); + +/* PatchGeneratorABC.proto */ +static int __Pyx_patch_abc(void); + +/* Generator.proto */ +#define __Pyx_Generator_USED +static PyTypeObject *__pyx_GeneratorType = 0; +#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) +#define __Pyx_Generator_New(body, closure, name, qualname, module_name)\ + __Pyx__Coroutine_New(__pyx_GeneratorType, body, closure, name, qualname, module_name) +static PyObject *__Pyx_Generator_Next(PyObject *self); +static int __pyx_Generator_init(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(void); + +/* PyIdentifierFromString.proto */ +#if !defined(__Pyx_PyIdentifier_FromString) +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s) +#else + #define __Pyx_PyIdentifier_FromString(s) PyUnicode_FromString(s) +#endif +#endif + +/* ModuleImport.proto */ +static PyObject *__Pyx_ImportModule(const char *name); + +/* TypeImport.proto */ +static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict); + +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); + + +/* Module declarations from 'libcpp' */ + +/* Module declarations from 'libcpp.memory' */ + +/* Module declarations from 'libc.string' */ + +/* Module declarations from 'libcpp.string' */ + +/* Module declarations from 'libcpp.vector' */ + +/* Module declarations from 'libcpp.utility' */ + +/* Module declarations from 'libcpp.map' */ + +/* Module declarations from 'libcpp.set' */ + +/* Module declarations from 'libcpp.queue' */ + +/* Module declarations from 'libc.stdint' */ + +/* Module declarations from 'khmer._oxli._oxli' */ + +/* Module declarations from 'khmer._oxli.hashing' */ +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_7hashing_Kmer = 0; + +/* Module declarations from 'khmer._oxli.traversal' */ +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_9traversal_Traverser = 0; +static PyTypeObject *__pyx_ptype_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors = 0; +#define __Pyx_MODULE_NAME "khmer._oxli.traversal" +int __pyx_module_is_main_khmer___oxli__traversal = 0; + +/* Implementation of 'khmer._oxli.traversal' */ +static PyObject *__pyx_builtin_ValueError; +static const char __pyx_k_args[] = "args"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_send[] = "send"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_close[] = "close"; +static const char __pyx_k_graph[] = "graph"; +static const char __pyx_k_khmer[] = "_khmer"; +static const char __pyx_k_throw[] = "throw"; +static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_hashing[] = "hashing"; +static const char __pyx_k_Nodegraph[] = "Nodegraph"; +static const char __pyx_k_Countgraph[] = "Countgraph"; +static const char __pyx_k_ValueError[] = "ValueError"; +static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; +static const char __pyx_k_right_neighbors[] = "right_neighbors"; +static const char __pyx_k_khmer__oxli_traversal[] = "khmer._oxli.traversal"; +static const char __pyx_k_Traverser_right_neighbors[] = "Traverser.right_neighbors"; +static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; +static PyObject *__pyx_n_s_Countgraph; +static PyObject *__pyx_kp_s_Must_take_an_object_with_Hashtab; +static PyObject *__pyx_n_s_Nodegraph; +static PyObject *__pyx_n_s_Traverser_right_neighbors; +static PyObject *__pyx_n_s_ValueError; +static PyObject *__pyx_n_s_args; +static PyObject *__pyx_n_s_close; +static PyObject *__pyx_n_s_graph; +static PyObject *__pyx_n_s_hashing; +static PyObject *__pyx_n_s_import; +static PyObject *__pyx_n_s_khmer; +static PyObject *__pyx_n_s_khmer__oxli_traversal; +static PyObject *__pyx_n_s_main; +static PyObject *__pyx_n_s_pyx_vtable; +static PyObject *__pyx_n_s_right_neighbors; +static PyObject *__pyx_n_s_send; +static PyObject *__pyx_n_s_test; +static PyObject *__pyx_n_s_throw; +static int __pyx_pf_5khmer_5_oxli_9traversal_9Traverser___cinit__(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ +static PyObject *__pyx_pf_5khmer_5_oxli_9traversal_9Traverser_2right_neighbors(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer); /* proto */ +static PyObject *__pyx_tp_new_5khmer_5_oxli_9traversal_Traverser(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tuple_; + +/* "khmer/_oxli/traversal.pyx":13 + * cdef class Traverser: + * + * def __cinit__(self, graph): # <<<<<<<<<<<<<< + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') + */ + +/* Python wrapper */ +static int __pyx_pw_5khmer_5_oxli_9traversal_9Traverser_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5khmer_5_oxli_9traversal_9Traverser_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_graph = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_graph,0}; + PyObject* values[1] = {0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 13, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + } + __pyx_v_graph = values[0]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 13, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("khmer._oxli.traversal.Traverser.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5khmer_5_oxli_9traversal_9Traverser___cinit__(((struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)__pyx_v_self), __pyx_v_graph); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5khmer_5_oxli_9traversal_9Traverser___cinit__(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self, PyObject *__pyx_v_graph) { + khmer::khmer_KHashtable_Object *__pyx_v_ptr; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + khmer::Hashtable *__pyx_t_5; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "khmer/_oxli/traversal.pyx":14 + * + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< + * raise ValueError('Must take an object with Hashtable *') + * + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = (__pyx_t_3 != 0); + if (!__pyx_t_4) { + } else { + __pyx_t_1 = __pyx_t_4; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_3 = (__pyx_t_4 != 0); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + __pyx_t_3 = ((!__pyx_t_1) != 0); + if (__pyx_t_3) { + + /* "khmer/_oxli/traversal.pyx":15 + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< + * + * cdef CPyHashtable_Object* ptr = graph + */ + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 15, __pyx_L1_error) + + /* "khmer/_oxli/traversal.pyx":14 + * + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< + * raise ValueError('Must take an object with Hashtable *') + * + */ + } + + /* "khmer/_oxli/traversal.pyx":17 + * raise ValueError('Must take an object with Hashtable *') + * + * cdef CPyHashtable_Object* ptr = graph # <<<<<<<<<<<<<< + * self._graph_ptr = deref(ptr).hashtable + * + */ + __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); + + /* "khmer/_oxli/traversal.pyx":18 + * + * cdef CPyHashtable_Object* ptr = graph + * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< + * + * self._this.reset(new CpTraverser(self._graph_ptr)) + */ + __pyx_t_5 = (*__pyx_v_ptr).hashtable; + __pyx_v_self->_graph_ptr = __pyx_t_5; + + /* "khmer/_oxli/traversal.pyx":20 + * self._graph_ptr = deref(ptr).hashtable + * + * self._this.reset(new CpTraverser(self._graph_ptr)) # <<<<<<<<<<<<<< + * ''' + * def left_neighbors(self, Kmer kmer): + */ + __pyx_v_self->_this.reset(new khmer::Traverser(__pyx_v_self->_graph_ptr)); + + /* "khmer/_oxli/traversal.pyx":13 + * cdef class Traverser: + * + * def __cinit__(self, graph): # <<<<<<<<<<<<<< + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("khmer._oxli.traversal.Traverser.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_5khmer_5_oxli_9traversal_9Traverser_4generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ + +/* "khmer/_oxli/traversal.pyx":30 + * ''' + * + * def right_neighbors(self, hashing.Kmer kmer): # <<<<<<<<<<<<<< + * cdef KmerQueue kmer_q + * cdef CpKmer neighbor + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5khmer_5_oxli_9traversal_9Traverser_3right_neighbors(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ +static PyObject *__pyx_pw_5khmer_5_oxli_9traversal_9Traverser_3right_neighbors(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("right_neighbors (wrapper)", 0); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_kmer), __pyx_ptype_5khmer_5_oxli_7hashing_Kmer, 1, "kmer", 0))) __PYX_ERR(0, 30, __pyx_L1_error) + __pyx_r = __pyx_pf_5khmer_5_oxli_9traversal_9Traverser_2right_neighbors(((struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)__pyx_v_self), ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_kmer)); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5khmer_5_oxli_9traversal_9Traverser_2right_neighbors(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer) { + struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("right_neighbors", 0); + __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)__pyx_tp_new_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(__pyx_ptype_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 30, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __pyx_cur_scope->__pyx_v_kmer = __pyx_v_kmer; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_kmer); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_kmer); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9traversal_9Traverser_4generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_right_neighbors, __pyx_n_s_Traverser_right_neighbors, __pyx_n_s_khmer__oxli_traversal); if (unlikely(!gen)) __PYX_ERR(0, 30, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("khmer._oxli.traversal.Traverser.right_neighbors", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5khmer_5_oxli_9traversal_9Traverser_4generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("None", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L6_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 30, __pyx_L1_error) + + /* "khmer/_oxli/traversal.pyx":34 + * cdef CpKmer neighbor + * cdef CpKmer* ptr + * deref(self._this).traverse_right(deref(kmer._this), kmer_q) # <<<<<<<<<<<<<< + * while(kmer_q.empty() == 0): + * neighbor = kmer_q.back() + */ + (*__pyx_cur_scope->__pyx_v_self->_this).traverse_right((*__pyx_cur_scope->__pyx_v_kmer->_this), __pyx_cur_scope->__pyx_v_kmer_q); + + /* "khmer/_oxli/traversal.pyx":35 + * cdef CpKmer* ptr + * deref(self._this).traverse_right(deref(kmer._this), kmer_q) + * while(kmer_q.empty() == 0): # <<<<<<<<<<<<<< + * neighbor = kmer_q.back() + * ptr = new CpKmer(neighbor) + */ + while (1) { + __pyx_t_1 = ((__pyx_cur_scope->__pyx_v_kmer_q.empty() == 0) != 0); + if (!__pyx_t_1) break; + + /* "khmer/_oxli/traversal.pyx":36 + * deref(self._this).traverse_right(deref(kmer._this), kmer_q) + * while(kmer_q.empty() == 0): + * neighbor = kmer_q.back() # <<<<<<<<<<<<<< + * ptr = new CpKmer(neighbor) + * yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) + */ + __pyx_cur_scope->__pyx_v_neighbor = __pyx_cur_scope->__pyx_v_kmer_q.back(); + + /* "khmer/_oxli/traversal.pyx":37 + * while(kmer_q.empty() == 0): + * neighbor = kmer_q.back() + * ptr = new CpKmer(neighbor) # <<<<<<<<<<<<<< + * yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) + * kmer_q.pop() + */ + __pyx_cur_scope->__pyx_v_ptr = new khmer::Kmer(__pyx_cur_scope->__pyx_v_neighbor); + + /* "khmer/_oxli/traversal.pyx":38 + * neighbor = kmer_q.back() + * ptr = new CpKmer(neighbor) + * yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) # <<<<<<<<<<<<<< + * kmer_q.pop() + */ + __pyx_t_2 = ((PyObject *)__pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer->wrap(__pyx_cur_scope->__pyx_v_ptr, (*__pyx_cur_scope->__pyx_v_self->_graph_ptr).ksize())); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L6_resume_from_yield:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 38, __pyx_L1_error) + + /* "khmer/_oxli/traversal.pyx":39 + * ptr = new CpKmer(neighbor) + * yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) + * kmer_q.pop() # <<<<<<<<<<<<<< + */ + __pyx_cur_scope->__pyx_v_kmer_q.pop(); + } + if (1); else __pyx_cur_scope = __pyx_cur_scope; + + /* "khmer/_oxli/traversal.pyx":30 + * ''' + * + * def right_neighbors(self, hashing.Kmer kmer): # <<<<<<<<<<<<<< + * cdef KmerQueue kmer_q + * cdef CpKmer neighbor + */ + + /* function exit code */ + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("right_neighbors", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); __pyx_r = 0; + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_5khmer_5_oxli_9traversal_Traverser(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *p; + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)o); + new((void*)&(p->_this)) std::unique_ptr (); + if (unlikely(__pyx_pw_5khmer_5_oxli_9traversal_9Traverser_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_9traversal_Traverser(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *p = (struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)o; + #if PY_VERSION_HEX >= 0x030400a1 + if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + __Pyx_call_destructor(p->_this); + (*Py_TYPE(o)->tp_free)(o); +} + +static PyMethodDef __pyx_methods_5khmer_5_oxli_9traversal_Traverser[] = { + {"right_neighbors", (PyCFunction)__pyx_pw_5khmer_5_oxli_9traversal_9Traverser_3right_neighbors, METH_O, 0}, + {0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5khmer_5_oxli_9traversal_Traverser = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.traversal.Traverser", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_9traversal_Traverser, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5khmer_5_oxli_9traversal_Traverser, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_9traversal_Traverser, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *__pyx_freelist_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors[8]; +static int __pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors = 0; + +static PyObject *__pyx_tp_new_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *p; + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors)))) { + o = (PyObject*)__pyx_freelist_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors[--__pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors]; + memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors)); + (void) PyObject_INIT(o, t); + PyObject_GC_Track(o); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + p = ((struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o); + new((void*)&(p->__pyx_v_kmer_q)) khmer::KmerQueue(); + new((void*)&(p->__pyx_v_neighbor)) khmer::Kmer(); + return o; +} + +static void __pyx_tp_dealloc_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyObject *o) { + struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *p = (struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o; + PyObject_GC_UnTrack(o); + __Pyx_call_destructor(p->__pyx_v_kmer_q); + __Pyx_call_destructor(p->__pyx_v_neighbor); + Py_CLEAR(p->__pyx_v_kmer); + Py_CLEAR(p->__pyx_v_self); + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors)))) { + __pyx_freelist_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors[__pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors++] = ((struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static int __pyx_tp_traverse_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *p = (struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o; + if (p->__pyx_v_kmer) { + e = (*v)(((PyObject*)p->__pyx_v_kmer), a); if (e) return e; + } + if (p->__pyx_v_self) { + e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *p = (struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o; + tmp = ((PyObject*)p->__pyx_v_kmer); + p->__pyx_v_kmer = ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->__pyx_v_self); + p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyTypeObject __pyx_type_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors = { + PyVarObject_HEAD_INIT(0, 0) + "khmer._oxli.traversal.__pyx_scope_struct__right_neighbors", /*tp_name*/ + sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, /*tp_traverse*/ + __pyx_tp_clear_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + #if PY_VERSION_HEX < 0x03020000 + { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, + #else + PyModuleDef_HEAD_INIT, + #endif + "traversal", + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_n_s_Countgraph, __pyx_k_Countgraph, sizeof(__pyx_k_Countgraph), 0, 0, 1, 1}, + {&__pyx_kp_s_Must_take_an_object_with_Hashtab, __pyx_k_Must_take_an_object_with_Hashtab, sizeof(__pyx_k_Must_take_an_object_with_Hashtab), 0, 0, 1, 0}, + {&__pyx_n_s_Nodegraph, __pyx_k_Nodegraph, sizeof(__pyx_k_Nodegraph), 0, 0, 1, 1}, + {&__pyx_n_s_Traverser_right_neighbors, __pyx_k_Traverser_right_neighbors, sizeof(__pyx_k_Traverser_right_neighbors), 0, 0, 1, 1}, + {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, + {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, + {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, + {&__pyx_n_s_graph, __pyx_k_graph, sizeof(__pyx_k_graph), 0, 0, 1, 1}, + {&__pyx_n_s_hashing, __pyx_k_hashing, sizeof(__pyx_k_hashing), 0, 0, 1, 1}, + {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, + {&__pyx_n_s_khmer, __pyx_k_khmer, sizeof(__pyx_k_khmer), 0, 0, 1, 1}, + {&__pyx_n_s_khmer__oxli_traversal, __pyx_k_khmer__oxli_traversal, sizeof(__pyx_k_khmer__oxli_traversal), 0, 0, 1, 1}, + {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, + {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, + {&__pyx_n_s_right_neighbors, __pyx_k_right_neighbors, sizeof(__pyx_k_right_neighbors), 0, 0, 1, 1}, + {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, + {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, + {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, + {0, 0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 15, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitCachedConstants(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "khmer/_oxli/traversal.pyx":15 + * def __cinit__(self, graph): + * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< + * + * cdef CPyHashtable_Object* ptr = graph + */ + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple_)) __PYX_ERR(0, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple_); + __Pyx_GIVEREF(__pyx_tuple_); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC inittraversal(void); /*proto*/ +PyMODINIT_FUNC inittraversal(void) +#else +PyMODINIT_FUNC PyInit_traversal(void); /*proto*/ +PyMODINIT_FUNC PyInit_traversal(void) +#endif +{ + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannyDeclarations + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_traversal(void)", 0); + if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_StopAsyncIteration_USED + if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4("traversal", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_d); + __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + #if CYTHON_COMPILING_IN_PYPY + Py_INCREF(__pyx_b); + #endif + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + if (__pyx_module_is_main_khmer___oxli__traversal) { + if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + } + #if PY_MAJOR_VERSION >= 3 + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "khmer._oxli.traversal")) { + if (unlikely(PyDict_SetItemString(modules, "khmer._oxli.traversal", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #endif + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global init code ---*/ + /*--- Variable export code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_9traversal_Traverser) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_9traversal_Traverser.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "Traverser", (PyObject *)&__pyx_type_5khmer_5_oxli_9traversal_Traverser) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_ptype_5khmer_5_oxli_9traversal_Traverser = &__pyx_type_5khmer_5_oxli_9traversal_Traverser; + if (PyType_Ready(&__pyx_type_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors) < 0) __PYX_ERR(0, 30, __pyx_L1_error) + __pyx_type_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors.tp_print = 0; + __pyx_ptype_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors = &__pyx_type_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors; + /*--- Type import code ---*/ + __pyx_ptype_5khmer_5_oxli_7hashing_Kmer = __Pyx_ImportType("khmer._oxli.hashing", "Kmer", sizeof(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer), 1); if (unlikely(!__pyx_ptype_5khmer_5_oxli_7hashing_Kmer)) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer = (struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer*)__Pyx_GetVtable(__pyx_ptype_5khmer_5_oxli_7hashing_Kmer->tp_dict); if (unlikely(!__pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer)) __PYX_ERR(1, 4, __pyx_L1_error) + /*--- Variable import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + + /* "khmer/_oxli/traversal.pyx":4 + * from cython.operator cimport dereference as deref + * + * from .._khmer import Countgraph # <<<<<<<<<<<<<< + * from .._khmer import Nodegraph + * + */ + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_n_s_Countgraph); + __Pyx_GIVEREF(__pyx_n_s_Countgraph); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_Countgraph); + __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer, __pyx_t_1, 2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(0, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "khmer/_oxli/traversal.pyx":5 + * + * from .._khmer import Countgraph + * from .._khmer import Nodegraph # <<<<<<<<<<<<<< + * + * from _oxli cimport * + */ + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_n_s_Nodegraph); + __Pyx_GIVEREF(__pyx_n_s_Nodegraph); + PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Nodegraph); + __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer, __pyx_t_2, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(0, 5, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "khmer/_oxli/traversal.pyx":9 + * from _oxli cimport * + * cimport hashing + * import hashing # <<<<<<<<<<<<<< + * + * cdef class Traverser: + */ + __pyx_t_1 = __Pyx_Import(__pyx_n_s_hashing, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_hashing, __pyx_t_1) < 0) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "khmer/_oxli/traversal.pyx":1 + * from libcpp.memory cimport unique_ptr # <<<<<<<<<<<<<< + * from cython.operator cimport dereference as deref + * + */ + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + if (__pyx_m) { + if (__pyx_d) { + __Pyx_AddTraceback("init khmer._oxli.traversal", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + Py_DECREF(__pyx_m); __pyx_m = 0; + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init khmer._oxli.traversal"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); + if (unlikely(!result)) { + PyErr_Format(PyExc_NameError, +#if PY_MAJOR_VERSION >= 3 + "name '%U' is not defined", name); +#else + "name '%.200s' is not defined", PyString_AS_STRING(name)); +#endif + } + return result; +} + +/* RaiseDoubleKeywords */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AsString(kw_name)); + #endif +} + +/* ParseKeywords */ +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + continue; + } + name = first_kw_arg; + #if PY_MAJOR_VERSION < 3 + if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + while (*name) { + if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) + && _PyString_Eq(**name, key)) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + if ((**argname == key) || ( + (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) + && _PyString_Eq(**argname, key))) { + goto arg_passed_twice; + } + argname++; + } + } + } else + #endif + if (likely(PyUnicode_Check(key))) { + while (*name) { + int cmp = (**name == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**name, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + int cmp = (**argname == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**argname, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + argname++; + } + } + } else + goto invalid_keyword_type; + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* GetModuleGlobalName */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + +/* PyObjectCall */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyErrFetchRestore */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} +#endif + +/* RaiseException */ + #if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + __Pyx_PyThreadState_declare + Py_XINCREF(type); + if (!value || value == Py_None) + value = NULL; + else + Py_INCREF(value); + if (!tb || tb == Py_None) + tb = NULL; + else { + Py_INCREF(tb); + if (!PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + } + if (PyType_Check(type)) { +#if CYTHON_COMPILING_IN_PYPY + if (!value) { + Py_INCREF(Py_None); + value = Py_None; + } +#endif + PyErr_NormalizeException(&type, &value, &tb); + } else { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + value = type; + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + } + __Pyx_PyThreadState_assign + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } +#if PY_VERSION_HEX >= 0x03030000 + if (cause) { +#else + if (cause && cause != Py_None) { +#endif + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = PyThreadState_GET(); + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} +#endif + +/* ArgTypeTest */ + static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); +} +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact) +{ + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (none_allowed && obj == Py_None) return 1; + else if (exact) { + if (likely(Py_TYPE(obj) == type)) return 1; + #if PY_MAJOR_VERSION == 2 + else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + #endif + } + else { + if (likely(PyObject_TypeCheck(obj, type))) return 1; + } + __Pyx_RaiseArgumentTypeInvalid(name, obj, type); + return 0; +} + +/* GetVTable */ + static void* __Pyx_GetVtable(PyObject *dict) { + void* ptr; + PyObject *ob = PyObject_GetItem(dict, __pyx_n_s_pyx_vtable); + if (!ob) + goto bad; +#if PY_VERSION_HEX >= 0x02070000 + ptr = PyCapsule_GetPointer(ob, 0); +#else + ptr = PyCObject_AsVoidPtr(ob); +#endif + if (!ptr && !PyErr_Occurred()) + PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); + Py_DECREF(ob); + return ptr; +bad: + Py_XDECREF(ob); + return NULL; +} + +/* Import */ + static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_import; + py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); + if (!py_import) + goto bad; + #endif + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + { + #if PY_MAJOR_VERSION >= 3 + if (level == -1) { + if (strchr(__Pyx_MODULE_NAME, '.')) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(1); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, 1); + #endif + if (!module) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + #endif + if (!module) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(level); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, level); + #endif + } + } +bad: + #if PY_VERSION_HEX < 0x03030000 + Py_XDECREF(py_import); + #endif + Py_XDECREF(empty_list); + Py_XDECREF(empty_dict); + return module; +} + +/* ImportFrom */ + static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Format(PyExc_ImportError, + #if PY_MAJOR_VERSION < 3 + "cannot import name %.230s", PyString_AS_STRING(name)); + #else + "cannot import name %S", name); + #endif + } + return value; +} + +/* CodeObjectCache */ + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); +} + +/* AddTraceback */ + #include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, + 0, + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + py_code = __pyx_find_code_object(c_line ? c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + } + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case -2: + if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (long) -1; + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (long) -1; + val = __Pyx_PyInt_As_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case -2: + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int) -1; + } + } else { + int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FetchCommonType */ + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* fake_module; + PyTypeObject* cached_type = NULL; + fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); + if (!fake_module) return NULL; + Py_INCREF(fake_module); + cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + if (cached_type) { + if (!PyType_Check((PyObject*)cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", + type->tp_name); + goto bad; + } + if (cached_type->tp_basicsize != type->tp_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + type->tp_name); + goto bad; + } + } else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + } +done: + Py_DECREF(fake_module); + return cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* SwapException */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* PyCFunctionFastCall */ + #if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + PyObject *result; + int flags; + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); +} +#endif // CYTHON_FAST_PYCCALL + +/* PyFunctionFastCall */ + #if CYTHON_FAST_PYCALL +#include "frameobject.h" +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = PyThreadState_GET(); + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif // CPython < 3.6 +#endif // CYTHON_FAST_PYCALL + +/* PyObjectCallMethO */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { +#else + if (likely(PyCFunction_Check(func))) { +#endif + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif + +/* PyObjectCallMethod1 */ + static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { + PyObject *method, *result = NULL; + method = __Pyx_PyObject_GetAttrStr(obj, method_name); + if (unlikely(!method)) goto done; +#if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(method))) { + PyObject *self = PyMethod_GET_SELF(method); + if (likely(self)) { + PyObject *args; + PyObject *function = PyMethod_GET_FUNCTION(method); + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(function)) { + PyObject *args[2] = {self, arg}; + result = __Pyx_PyFunction_FastCall(function, args, 2); + goto done; + } + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(function)) { + PyObject *args[2] = {self, arg}; + result = __Pyx_PyCFunction_FastCall(function, args, 2); + goto done; + } + #endif + args = PyTuple_New(2); + if (unlikely(!args)) goto done; + Py_INCREF(self); + PyTuple_SET_ITEM(args, 0, self); + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 1, arg); + Py_INCREF(function); + Py_DECREF(method); method = NULL; + result = __Pyx_PyObject_Call(function, args, NULL); + Py_DECREF(args); + Py_DECREF(function); + return result; + } + } +#endif + result = __Pyx_PyObject_CallOneArg(method, arg); +done: + Py_XDECREF(method); + return result; +} + +/* CoroutineBase */ + #include +#include +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); +static PyObject *__Pyx_Coroutine_Close(PyObject *self); +static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); +#define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) +#if 1 || PY_VERSION_HEX < 0x030300B0 +static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { + PyObject *et, *ev, *tb; + PyObject *value = NULL; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&et, &ev, &tb); + if (!et) { + Py_XDECREF(tb); + Py_XDECREF(ev); + Py_INCREF(Py_None); + *pvalue = Py_None; + return 0; + } + if (likely(et == PyExc_StopIteration)) { + if (!ev) { + Py_INCREF(Py_None); + value = Py_None; + } +#if PY_VERSION_HEX >= 0x030300A0 + else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + Py_DECREF(ev); + } +#endif + else if (unlikely(PyTuple_Check(ev))) { + if (PyTuple_GET_SIZE(ev) >= 1) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + value = PyTuple_GET_ITEM(ev, 0); + Py_INCREF(value); +#else + value = PySequence_ITEM(ev, 0); +#endif + } else { + Py_INCREF(Py_None); + value = Py_None; + } + Py_DECREF(ev); + } + else if (!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { + value = ev; + } + if (likely(value)) { + Py_XDECREF(tb); + Py_DECREF(et); + *pvalue = value; + return 0; + } + } else if (!PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + PyErr_NormalizeException(&et, &ev, &tb); + if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + Py_XDECREF(tb); + Py_DECREF(et); +#if PY_VERSION_HEX >= 0x030300A0 + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + Py_DECREF(ev); +#else + { + PyObject* args = __Pyx_PyObject_GetAttrStr(ev, __pyx_n_s_args); + Py_DECREF(ev); + if (likely(args)) { + value = PySequence_GetItem(args, 0); + Py_DECREF(args); + } + if (unlikely(!value)) { + __Pyx_ErrRestore(NULL, NULL, NULL); + Py_INCREF(Py_None); + value = Py_None; + } + } +#endif + *pvalue = value; + return 0; +} +#endif +static CYTHON_INLINE +void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { + PyObject *exc_type = self->exc_type; + PyObject *exc_value = self->exc_value; + PyObject *exc_traceback = self->exc_traceback; + self->exc_type = NULL; + self->exc_value = NULL; + self->exc_traceback = NULL; + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_traceback); +} +static CYTHON_INLINE +int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) { + if (unlikely(gen->is_running)) { + PyErr_SetString(PyExc_ValueError, + "generator already executing"); + return 1; + } + return 0; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) { + PyObject *retval; + __Pyx_PyThreadState_declare + assert(!self->is_running); + if (unlikely(self->resume_label == 0)) { + if (unlikely(value && value != Py_None)) { + PyErr_SetString(PyExc_TypeError, + "can't send non-None value to a " + "just-started generator"); + return NULL; + } + } + if (unlikely(self->resume_label == -1)) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + __Pyx_PyThreadState_assign + if (value) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#else + if (self->exc_traceback) { + PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyFrameObject *f = tb->tb_frame; + Py_XINCREF(__pyx_tstate->frame); + assert(f->f_back == NULL); + f->f_back = __pyx_tstate->frame; + } +#endif + __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, + &self->exc_traceback); + } else { + __Pyx_Coroutine_ExceptionClear(self); + } + self->is_running = 1; + retval = self->body((PyObject *) self, value); + self->is_running = 0; + if (retval) { + __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, + &self->exc_traceback); +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#else + if (self->exc_traceback) { + PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyFrameObject *f = tb->tb_frame; + Py_CLEAR(f->f_back); + } +#endif + } else { + __Pyx_Coroutine_ExceptionClear(self); + } + return retval; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) { + if (unlikely(!retval && !PyErr_Occurred())) { + PyErr_SetNone(PyExc_StopIteration); + } + return retval; +} +static CYTHON_INLINE +PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { + PyObject *ret; + PyObject *val = NULL; + __Pyx_Coroutine_Undelegate(gen); + __Pyx_PyGen_FetchStopIterationValue(&val); + ret = __Pyx_Coroutine_SendEx(gen, val); + Py_XDECREF(val); + return ret; +} +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { + PyObject *retval; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + PyObject *yf = gen->yieldfrom; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + { + if (value == Py_None) + ret = Py_TYPE(yf)->tp_iternext(yf); + else + ret = __Pyx_PyObject_CallMethod1(yf, __pyx_n_s_send, value); + } + gen->is_running = 0; + if (likely(ret)) { + return ret; + } + retval = __Pyx_Coroutine_FinishDelegation(gen); + } else { + retval = __Pyx_Coroutine_SendEx(gen, value); + } + return __Pyx_Coroutine_MethodReturn(retval); +} +static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { + PyObject *retval = NULL; + int err = 0; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + retval = __Pyx_Coroutine_Close(yf); + if (!retval) + return -1; + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + retval = __Pyx_Coroutine_Close(yf); + if (!retval) + return -1; + } else + #endif + { + PyObject *meth; + gen->is_running = 1; + meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_close); + if (unlikely(!meth)) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_WriteUnraisable(yf); + } + PyErr_Clear(); + } else { + retval = PyObject_CallFunction(meth, NULL); + Py_DECREF(meth); + if (!retval) + err = -1; + } + gen->is_running = 0; + } + Py_XDECREF(retval); + return err; +} +static PyObject *__Pyx_Generator_Next(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + PyObject *yf = gen->yieldfrom; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Generator_Next(yf); + } else + #endif + ret = Py_TYPE(yf)->tp_iternext(yf); + gen->is_running = 0; + if (likely(ret)) { + return ret; + } + return __Pyx_Coroutine_FinishDelegation(gen); + } + return __Pyx_Coroutine_SendEx(gen, Py_None); +} +static PyObject *__Pyx_Coroutine_Close(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject *retval, *raised_exception; + PyObject *yf = gen->yieldfrom; + int err = 0; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + Py_INCREF(yf); + err = __Pyx_Coroutine_CloseIter(gen, yf); + __Pyx_Coroutine_Undelegate(gen); + Py_DECREF(yf); + } + if (err == 0) + PyErr_SetNone(PyExc_GeneratorExit); + retval = __Pyx_Coroutine_SendEx(gen, NULL); + if (retval) { + Py_DECREF(retval); + PyErr_SetString(PyExc_RuntimeError, + "generator ignored GeneratorExit"); + return NULL; + } + raised_exception = PyErr_Occurred(); + if (!raised_exception + || raised_exception == PyExc_StopIteration + || raised_exception == PyExc_GeneratorExit + || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) + || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) + { + if (raised_exception) PyErr_Clear(); + Py_INCREF(Py_None); + return Py_None; + } + return NULL; +} +static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject *typ; + PyObject *tb = NULL; + PyObject *val = NULL; + PyObject *yf = gen->yieldfrom; + if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) + return NULL; + if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) + return NULL; + if (yf) { + PyObject *ret; + Py_INCREF(yf); + if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + int err = __Pyx_Coroutine_CloseIter(gen, yf); + Py_DECREF(yf); + __Pyx_Coroutine_Undelegate(gen); + if (err < 0) + return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); + goto throw_here; + } + gen->is_running = 1; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Coroutine_Throw(yf, args); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + ret = __Pyx_Coroutine_Throw(yf, args); + } else + #endif + { + PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_throw); + if (unlikely(!meth)) { + Py_DECREF(yf); + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + gen->is_running = 0; + return NULL; + } + PyErr_Clear(); + __Pyx_Coroutine_Undelegate(gen); + gen->is_running = 0; + goto throw_here; + } + ret = PyObject_CallObject(meth, args); + Py_DECREF(meth); + } + gen->is_running = 0; + Py_DECREF(yf); + if (!ret) { + ret = __Pyx_Coroutine_FinishDelegation(gen); + } + return __Pyx_Coroutine_MethodReturn(ret); + } +throw_here: + __Pyx_Raise(typ, val, tb, NULL); + return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); +} +static int __Pyx_Coroutine_traverse(PyObject *self, visitproc visit, void *arg) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + Py_VISIT(gen->closure); + Py_VISIT(gen->classobj); + Py_VISIT(gen->yieldfrom); + Py_VISIT(gen->exc_type); + Py_VISIT(gen->exc_value); + Py_VISIT(gen->exc_traceback); + return 0; +} +static int __Pyx_Coroutine_clear(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + Py_CLEAR(gen->closure); + Py_CLEAR(gen->classobj); + Py_CLEAR(gen->yieldfrom); + Py_CLEAR(gen->exc_type); + Py_CLEAR(gen->exc_value); + Py_CLEAR(gen->exc_traceback); + Py_CLEAR(gen->gi_name); + Py_CLEAR(gen->gi_qualname); + return 0; +} +static void __Pyx_Coroutine_dealloc(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject_GC_UnTrack(gen); + if (gen->gi_weakreflist != NULL) + PyObject_ClearWeakRefs(self); + if (gen->resume_label > 0) { + PyObject_GC_Track(self); +#if PY_VERSION_HEX >= 0x030400a1 + if (PyObject_CallFinalizerFromDealloc(self)) +#else + Py_TYPE(gen)->tp_del(self); + if (self->ob_refcnt > 0) +#endif + { + return; + } + PyObject_GC_UnTrack(self); + } + __Pyx_Coroutine_clear(self); + PyObject_GC_Del(gen); +} +static void __Pyx_Coroutine_del(PyObject *self) { + PyObject *res; + PyObject *error_type, *error_value, *error_traceback; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + __Pyx_PyThreadState_declare + if (gen->resume_label <= 0) + return ; +#if PY_VERSION_HEX < 0x030400a1 + assert(self->ob_refcnt == 0); + self->ob_refcnt = 1; +#endif + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); + res = __Pyx_Coroutine_Close(self); + if (res == NULL) + PyErr_WriteUnraisable(self); + else + Py_DECREF(res); + __Pyx_ErrRestore(error_type, error_value, error_traceback); +#if PY_VERSION_HEX < 0x030400a1 + assert(self->ob_refcnt > 0); + if (--self->ob_refcnt == 0) { + return; + } + { + Py_ssize_t refcnt = self->ob_refcnt; + _Py_NewReference(self); + self->ob_refcnt = refcnt; + } +#if CYTHON_COMPILING_IN_CPYTHON + assert(PyType_IS_GC(self->ob_type) && + _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); + _Py_DEC_REFTOTAL; +#endif +#ifdef COUNT_ALLOCS + --Py_TYPE(self)->tp_frees; + --Py_TYPE(self)->tp_allocs; +#endif +#endif +} +static PyObject * +__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self) +{ + PyObject *name = self->gi_name; + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + tmp = self->gi_name; + Py_INCREF(value); + self->gi_name = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self) +{ + PyObject *name = self->gi_qualname; + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + tmp = self->gi_qualname; + Py_INCREF(value); + self->gi_qualname = value; + Py_XDECREF(tmp); + return 0; +} +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { + __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); + if (gen == NULL) + return NULL; + gen->body = body; + gen->closure = closure; + Py_XINCREF(closure); + gen->is_running = 0; + gen->resume_label = 0; + gen->classobj = NULL; + gen->yieldfrom = NULL; + gen->exc_type = NULL; + gen->exc_value = NULL; + gen->exc_traceback = NULL; + gen->gi_weakreflist = NULL; + Py_XINCREF(qualname); + gen->gi_qualname = qualname; + Py_XINCREF(name); + gen->gi_name = name; + Py_XINCREF(module_name); + gen->gi_modulename = module_name; + PyObject_GC_Track(gen); + return gen; +} + +/* PatchModuleWithCoroutine */ + static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code) { +#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + int result; + PyObject *globals, *result_obj; + globals = PyDict_New(); if (unlikely(!globals)) goto ignore; + result = PyDict_SetItemString(globals, "_cython_coroutine_type", + #ifdef __Pyx_Coroutine_USED + (PyObject*)__pyx_CoroutineType); + #else + Py_None); + #endif + if (unlikely(result < 0)) goto ignore; + result = PyDict_SetItemString(globals, "_cython_generator_type", + #ifdef __Pyx_Generator_USED + (PyObject*)__pyx_GeneratorType); + #else + Py_None); + #endif + if (unlikely(result < 0)) goto ignore; + if (unlikely(PyDict_SetItemString(globals, "_module", module) < 0)) goto ignore; + if (unlikely(PyDict_SetItemString(globals, "__builtins__", __pyx_b) < 0)) goto ignore; + result_obj = PyRun_String(py_code, Py_file_input, globals, globals); + if (unlikely(!result_obj)) goto ignore; + Py_DECREF(result_obj); + Py_DECREF(globals); + return module; +ignore: + Py_XDECREF(globals); + PyErr_WriteUnraisable(module); + if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, "Cython module failed to patch module with custom type", 1) < 0)) { + Py_DECREF(module); + module = NULL; + } +#else + py_code++; +#endif + return module; +} + +/* PatchGeneratorABC */ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) +static PyObject* __Pyx_patch_abc_module(PyObject *module); +static PyObject* __Pyx_patch_abc_module(PyObject *module) { + module = __Pyx_Coroutine_patch_module( + module, "" +"if _cython_generator_type is not None:\n" +" try: Generator = _module.Generator\n" +" except AttributeError: pass\n" +" else: Generator.register(_cython_generator_type)\n" +"if _cython_coroutine_type is not None:\n" +" try: Coroutine = _module.Coroutine\n" +" except AttributeError: pass\n" +" else: Coroutine.register(_cython_coroutine_type)\n" + ); + return module; +} +#endif +static int __Pyx_patch_abc(void) { +#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + static int abc_patched = 0; + if (!abc_patched) { + PyObject *module; + module = PyImport_ImportModule((PY_VERSION_HEX >= 0x03030000) ? "collections.abc" : "collections"); + if (!module) { + PyErr_WriteUnraisable(NULL); + if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, + ((PY_VERSION_HEX >= 0x03030000) ? + "Cython module failed to register with collections.abc module" : + "Cython module failed to register with collections module"), 1) < 0)) { + return -1; + } + } else { + module = __Pyx_patch_abc_module(module); + abc_patched = 1; + if (unlikely(!module)) + return -1; + Py_DECREF(module); + } + module = PyImport_ImportModule("backports_abc"); + if (module) { + module = __Pyx_patch_abc_module(module); + Py_XDECREF(module); + } + if (!module) { + PyErr_Clear(); + } + } +#else + if (0) __Pyx_Coroutine_patch_module(NULL, NULL); +#endif + return 0; +} + +/* Generator */ + static PyMethodDef __pyx_Generator_methods[] = { + {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, + (char*) PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, + {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, + (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, + {"close", (PyCFunction) __Pyx_Coroutine_Close, METH_NOARGS, + (char*) PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, + {0, 0, 0, 0} +}; +static PyMemberDef __pyx_Generator_memberlist[] = { + {(char *) "gi_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, + {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, + (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, + {0, 0, 0, 0, 0} +}; +static PyGetSetDef __pyx_Generator_getsets[] = { + {(char *) "__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, + (char*) PyDoc_STR("name of the generator"), 0}, + {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, + (char*) PyDoc_STR("qualified name of the generator"), 0}, + {0, 0, 0, 0, 0} +}; +static PyTypeObject __pyx_GeneratorType_type = { + PyVarObject_HEAD_INIT(0, 0) + "generator", + sizeof(__pyx_CoroutineObject), + 0, + (destructor) __Pyx_Coroutine_dealloc, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, + 0, + (traverseproc) __Pyx_Coroutine_traverse, + 0, + 0, + offsetof(__pyx_CoroutineObject, gi_weakreflist), + 0, + (iternextfunc) __Pyx_Generator_Next, + __pyx_Generator_methods, + __pyx_Generator_memberlist, + __pyx_Generator_getsets, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, +#if PY_VERSION_HEX >= 0x030400a1 + 0, +#else + __Pyx_Coroutine_del, +#endif + 0, +#if PY_VERSION_HEX >= 0x030400a1 + __Pyx_Coroutine_del, +#endif +}; +static int __pyx_Generator_init(void) { + __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; + __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; + __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); + if (unlikely(!__pyx_GeneratorType)) { + return -1; + } + return 0; +} + +/* CheckBinaryVersion */ + static int __Pyx_check_binary_version(void) { + char ctversion[4], rtversion[4]; + PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); + PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); + if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compiletime version %s of module '%.100s' " + "does not match runtime version %s", + ctversion, __Pyx_MODULE_NAME, rtversion); + return PyErr_WarnEx(NULL, message, 1); + } + return 0; +} + +/* ModuleImport */ + #ifndef __PYX_HAVE_RT_ImportModule +#define __PYX_HAVE_RT_ImportModule +static PyObject *__Pyx_ImportModule(const char *name) { + PyObject *py_name = 0; + PyObject *py_module = 0; + py_name = __Pyx_PyIdentifier_FromString(name); + if (!py_name) + goto bad; + py_module = PyImport_Import(py_name); + Py_DECREF(py_name); + return py_module; +bad: + Py_XDECREF(py_name); + return 0; +} +#endif + +/* TypeImport */ + #ifndef __PYX_HAVE_RT_ImportType +#define __PYX_HAVE_RT_ImportType +static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, + size_t size, int strict) +{ + PyObject *py_module = 0; + PyObject *result = 0; + PyObject *py_name = 0; + char warning[200]; + Py_ssize_t basicsize; +#ifdef Py_LIMITED_API + PyObject *py_basicsize; +#endif + py_module = __Pyx_ImportModule(module_name); + if (!py_module) + goto bad; + py_name = __Pyx_PyIdentifier_FromString(class_name); + if (!py_name) + goto bad; + result = PyObject_GetAttr(py_module, py_name); + Py_DECREF(py_name); + py_name = 0; + Py_DECREF(py_module); + py_module = 0; + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + module_name, class_name); + goto bad; + } +#ifndef Py_LIMITED_API + basicsize = ((PyTypeObject *)result)->tp_basicsize; +#else + py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); + if (!py_basicsize) + goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; +#endif + if (!strict && (size_t)basicsize > size) { + PyOS_snprintf(warning, sizeof(warning), + "%s.%s size changed, may indicate binary incompatibility. Expected %zd, got %zd", + module_name, class_name, basicsize, size); + if (PyErr_WarnEx(NULL, warning, 0) < 0) goto bad; + } + else if ((size_t)basicsize != size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s has the wrong size, try recompiling. Expected %zd, got %zd", + module_name, class_name, basicsize, size); + goto bad; + } + return (PyTypeObject *)result; +bad: + Py_XDECREF(py_module); + Py_XDECREF(result); + return NULL; +} +#endif + +/* InitStrings */ + static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else + if (t->is_unicode | t->is_str) { + if (t->intern) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); + } else { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { +#if PY_VERSION_HEX < 0x03030000 + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; + } + } + } +#endif + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +#else + if (__Pyx_PyUnicode_READY(o) == -1) return NULL; +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (PyUnicode_IS_ASCII(o)) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif + } else +#endif +#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) + if (PyByteArray_Check(o)) { + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); + } else +#endif + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + const char *name = NULL; + PyObject *res = NULL; +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + #if PY_MAJOR_VERSION < 3 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } + #else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } + #endif +#else + res = PyNumber_Int(x); +#endif + if (res) { +#if PY_MAJOR_VERSION < 3 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(b))) { + if (sizeof(Py_ssize_t) >= sizeof(long)) + return PyInt_AS_LONG(b); + else + return PyInt_AsSsize_t(x); + } +#endif + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)b)->ob_digit; + const Py_ssize_t size = Py_SIZE(b); + if (likely(__Pyx_sst_abs(size) <= 1)) { + ival = likely(size) ? digits[0] : 0; + if (size == -1) ival = -ival; + return ival; + } else { + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { + return PyInt_FromSize_t(ival); +} + + +#endif /* Py_PYTHON_H */ From 28095cf23e998fe754e3d8d40d3b53240180715d Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 14 Nov 2016 17:25:01 -0800 Subject: [PATCH 037/185] Add normalized streaming option --- scripts/partition-streaming.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/partition-streaming.py b/scripts/partition-streaming.py index 73ff98d724..d9b3bdabaa 100644 --- a/scripts/partition-streaming.py +++ b/scripts/partition-streaming.py @@ -9,7 +9,7 @@ from khmer.khmer_args import (build_counting_args, add_loadgraph_args, report_on_config, info, calculate_graphsize, sanitize_help, check_argument_range) -from khmer import _oxli +from khmer._oxli.partitioning import StreamingPartitioner import argparse from khmer.utils import (write_record, broken_paired_reader, ReadBundle) @@ -27,10 +27,12 @@ def main(): parser = build_counting_args() parser.add_argument('-stats-dir', default='component_stats') parser.add_argument('samples', nargs='+') + parser.add_argument('-Z', dest='norm', default=10) + parser.add_argument('--stats-interval', default=50000) args = parser.parse_args() graph = khmer_args.create_countgraph(args) - partitioner = _oxli.StreamingPartitioner(graph) + partitioner = StreamingPartitioner(graph) try: os.mkdir(args.stats_dir) @@ -40,11 +42,19 @@ def main(): for sample in args.samples: print('== Starting {0} =='.format(sample)) for n, read in enumerate(screed.open(sample)): - if n % 500 == 0: + if n % 1000 == 0: print (n, '...', sep='') - if n > 0 and n % 10000 == 0: + if args.stats_interval > 0 and n > 0 and n % args.stats_interval == 0: write_stats(partitioner, args.stats_dir, n, sample) - partitioner.consume_sequence(read.sequence) + cov, _, _ = graph.get_median_count(read.sequence) + if cov < args.norm: + graph.consume(read.sequence) + else: + seq, pos = graph.trim_on_abundance(read.sequence, 2) + if len(seq) < args.ksize: + continue + partitioner.consume_sequence(read.sequence) + write_stats(partitioner, args.stats_dir, n, sample) if __name__ == '__main__': From 748f5f5c9ece178ea50e10f47b2a8b237fcc094e Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 15 Nov 2016 12:49:37 -0800 Subject: [PATCH 038/185] basic wrapping of FastxParser --- khmer/_oxli/_oxli.pxd | 25 ++++++++++ khmer/_oxli/streaming.pxd | 35 +++++++++++++ khmer/_oxli/streaming.pyx | 102 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 khmer/_oxli/streaming.pxd create mode 100644 khmer/_oxli/streaming.pyx diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index d34b96e4c1..9c7876f9e6 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -47,6 +47,31 @@ cdef extern from "kmer_hash.hh" namespace "khmer": string _revhash(HashIntoType, WordLength) +######################################################################## +# +# ReadParser: read parsing stuff, Read object +# +######################################################################## + + +cdef extern from "read_parsers.hh": + cdef cppclass CpSequence "khmer::read_parsers::Read": + string name + string annotations + string sequence + string quality + + void reset() + + ctypedef pair[CpSequence,CpSequence] CpSequencePair "khmer::read_parsers::ReadPair" + + cdef cppclass CpFastxParser "khmer::read_parsers::FastxParser": + CpFastxParser(const char *) + bool is_complete() + void imprint_next_read(CpSequence&) except + + void imprint_next_read_pair(CpSequencePair&, uint8_t) except + + + ######################################################################## # diff --git a/khmer/_oxli/streaming.pxd b/khmer/_oxli/streaming.pxd new file mode 100644 index 0000000000..b87886e914 --- /dev/null +++ b/khmer/_oxli/streaming.pxd @@ -0,0 +1,35 @@ +from libcpp.memory cimport unique_ptr +from libcpp cimport bool + +from _oxli cimport CpSequence, CpFastxParser + +cdef class Sequence: + cdef unique_ptr[CpSequence] _this + + +cdef class SequencePair: + cdef Sequence first + cdef Sequence second + + +cdef class ReadBundle: + cdef list reads + + +cdef class FastxParser: + cdef unique_ptr[CpFastxParser] _this + cdef Sequence _next(self) + + +cpdef tuple broken_paired_reader(FastxParser fastx_iter, int min_length=*, + bool force_single=*, bool require_paired=*) + +cdef tuple _split_left_right(str name) + +cdef bool check_is_pair(Sequence first, Sequence second) + +cdef bool check_is_left(str name) + +cdef bool check_is_right(str name) + + diff --git a/khmer/_oxli/streaming.pyx b/khmer/_oxli/streaming.pyx new file mode 100644 index 0000000000..b5ba546377 --- /dev/null +++ b/khmer/_oxli/streaming.pyx @@ -0,0 +1,102 @@ +from cython.operator cimport dereference as deref +from libcpp cimport bool + +from _oxli cimport CpSequence + + +cdef class Sequence: + + def __cinit__(self, create=False): + if create is True: + self._this.reset(new CpSequence()) + + @property + def name(self): + return deref(self._this).name + + @property + def annotations(self): + return deref(self._this).annotations + + @property + def sequence(self): + return deref(self._this).sequence + + @property + def quality(self): + return deref(self._this).quality + + @staticmethod + def create(str sequence, str name, str annotations=None, str quality=None): + cdef Sequence seq = Sequence(create=True) + deref(seq._this).sequence = sequence + deref(seq._this).name = name + if annotations is not None: + deref(seq._this).annotations = annotations + if quality is not None: + deref(seq._this).quality = quality + +cdef class SequencePair: + + def __cinit__(self, Sequence first, Sequence second): + self.first = first + self.second = second + + +cdef class ReadBundle: + + def __cinit__(self, *raw_records): + self.reads = [r for r in raw_records if r] + + @property + def num_reads(self): + return len(self.reads) + + @property + def total_length(self): + return sum([len(r.sequence) for r in self.reads]) + + +cdef class FastxParser: + + def __cinit__(self, str filename): + self._this.reset(new CpFastxParser(filename.encode())) + + cdef Sequence _next(self): + cdef Sequence seq = Sequence(create=True) + if deref(self._this).is_complete(): + return None + else: + deref(self._this).imprint_next_read(deref(seq._this)) + return seq + + def __iter__(self): + cdef Sequence seq = self._next() + while seq is not None: + yield seq + seq = self._next() + + +cpdef tuple broken_paired_reader(FastxParser fastx_iter, + int min_length=None, + bool force_single=False, + bool require_paired=False): + pass + + +cdef tuple _split_left_right(str name): + pass + + +cdef bool check_is_pair(Sequence first, Sequence second): + pass + + +cdef bool check_is_left(str name): + pass + + +cdef bool check_is_right(str name): + pass + + From 60f3597c71dcfa6c72d92a11900b218d2bd4bc5e Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 16 Nov 2016 17:09:23 -0800 Subject: [PATCH 039/185] Add cythonized BrokenPairedReader --- khmer/_oxli/streaming.pxd | 27 ++- khmer/_oxli/streaming.pyx | 292 ++++++++++++++++++++++++++++++--- tests/test_cython_streaming.py | 154 +++++++++++++++++ 3 files changed, 451 insertions(+), 22 deletions(-) create mode 100644 tests/test_cython_streaming.py diff --git a/khmer/_oxli/streaming.pxd b/khmer/_oxli/streaming.pxd index b87886e914..ec1176af36 100644 --- a/khmer/_oxli/streaming.pxd +++ b/khmer/_oxli/streaming.pxd @@ -1,11 +1,19 @@ from libcpp.memory cimport unique_ptr from libcpp cimport bool +from libcpp.string cimport string from _oxli cimport CpSequence, CpFastxParser +cdef inline void move_cpsequence(CpSequence& move_from, CpSequence& move_to) +cdef inline void move_sequence(Sequence move_from, Sequence move_to) + cdef class Sequence: cdef unique_ptr[CpSequence] _this + cdef void take(self, Sequence other) + @staticmethod + cdef Sequence _create(str name, str sequence, + str annotations=*, str quality=*) cdef class SequencePair: cdef Sequence first @@ -18,18 +26,31 @@ cdef class ReadBundle: cdef class FastxParser: cdef unique_ptr[CpFastxParser] _this + + cdef bool _imprint_next(self, Sequence seq) + cdef bool _cp_imprint_next(self, CpSequence& seq) cdef Sequence _next(self) -cpdef tuple broken_paired_reader(FastxParser fastx_iter, int min_length=*, - bool force_single=*, bool require_paired=*) +cdef class BrokenPairedReader: + + cdef FastxParser parser + cdef readonly int min_length + cdef readonly bool force_single + cdef readonly bool require_paired + cdef readonly Sequence record + + cdef tuple _next(self) cdef tuple _split_left_right(str name) -cdef bool check_is_pair(Sequence first, Sequence second) +cdef bool _check_is_pair(Sequence first, Sequence second) cdef bool check_is_left(str name) cdef bool check_is_right(str name) +cdef inline bool is_valid_dna(const char base) + +cdef inline bool sanitize_sequence(string& sequence) diff --git a/khmer/_oxli/streaming.pyx b/khmer/_oxli/streaming.pyx index b5ba546377..68063f0e61 100644 --- a/khmer/_oxli/streaming.pyx +++ b/khmer/_oxli/streaming.pyx @@ -1,15 +1,38 @@ +# cython: c_string_type=unicode, c_string_encoding=utf8 +from __future__ import print_function from cython.operator cimport dereference as deref from libcpp cimport bool +from libcpp.string cimport string + +import sys from _oxli cimport CpSequence +cdef inline void move_cpsequence(CpSequence& move_from, CpSequence& move_to): + move_to.name.assign(move_from.name) + move_to.sequence.assign(move_from.sequence) + move_to.annotations.assign(move_from.annotations) + move_to.quality.assign(move_from.quality) + +cdef inline void move_sequence(Sequence move_from, Sequence move_to): + move_cpsequence(deref(move_from._this), deref(move_to._this)) + cdef class Sequence: def __cinit__(self, create=False): if create is True: self._this.reset(new CpSequence()) - + + cdef void take(self, Sequence other): + move_cpsequence(deref(other._this), deref(self._this)) + + def __str__(self): + return self.sequence + + def __len__(self): + return deref(self._this).sequence.length() + @property def name(self): return deref(self._this).name @@ -27,14 +50,29 @@ cdef class Sequence: return deref(self._this).quality @staticmethod - def create(str sequence, str name, str annotations=None, str quality=None): + def create(str name, str sequence, str annotations=None, str quality=None): + cdef Sequence seq = Sequence(create=True) + deref(seq._this).sequence = sequence.encode('UTF-8') + deref(seq._this).name = name.encode('UTF-8') + if annotations is not None: + deref(seq._this).annotations = annotations.encode('UTF-8') + if quality is not None: + deref(seq._this).quality = quality.encode('UTF-8') + + return seq + + @staticmethod + cdef Sequence _create(str name, str sequence, str annotations=None, str quality=None): cdef Sequence seq = Sequence(create=True) - deref(seq._this).sequence = sequence - deref(seq._this).name = name + deref(seq._this).sequence = sequence.encode('UTF-8') + deref(seq._this).name = name.encode('UTF-8') if annotations is not None: - deref(seq._this).annotations = annotations + deref(seq._this).annotations = annotations.encode('UTF-8') if quality is not None: - deref(seq._this).quality = quality + deref(seq._this).quality = quality.encode('UTF-8') + + return seq + cdef class SequencePair: @@ -57,18 +95,71 @@ cdef class ReadBundle: return sum([len(r.sequence) for r in self.reads]) +def print_error(msg): + """Print the given message to 'stderr'.""" + + print(msg, file=sys.stderr) + + +class UnpairedReadsError(ValueError): + """ValueError with refs to the read pair in question.""" + + def __init__(self, msg, r1, r2): + r1_name = "" + r2_name = "" + if r1: + r1_name = r1.name + if r2: + r2_name = r2.name + + msg = msg + '\n"{0}"\n"{1}"'.format(r1_name, r2_name) + ValueError.__init__(self, msg) + self.read1 = r1 + self.read2 = r2 + + +cdef inline bool is_valid_dna(const char base): + return base == 'A' or base == 'C' or base == 'G'\ + or base == 'T' or base == 'N' + + +cdef inline bool sanitize_sequence(string& sequence): + cdef int i = 0 + for i in range(sequence.length()): + sequence[i] &= 0xdf + if not is_valid_dna(sequence[i]): + return False + if sequence[i] == 'N': + sequence[i] = 'A' + return True + + cdef class FastxParser: def __cinit__(self, str filename): self._this.reset(new CpFastxParser(filename.encode())) - cdef Sequence _next(self): - cdef Sequence seq = Sequence(create=True) + cdef bool _cp_imprint_next(self, CpSequence& seq): if deref(self._this).is_complete(): - return None + return False + else: + deref(self._this).imprint_next_read(seq) + return True + + cdef bool _imprint_next(self, Sequence seq): + if deref(self._this).is_complete(): + return False else: deref(self._this).imprint_next_read(deref(seq._this)) + return True + + cdef Sequence _next(self): + cdef Sequence seq = Sequence(create=True) + cdef bool res = self._imprint(seq) + if res: return seq + else: + return None def __iter__(self): cdef Sequence seq = self._next() @@ -77,26 +168,189 @@ cdef class FastxParser: seq = self._next() -cpdef tuple broken_paired_reader(FastxParser fastx_iter, - int min_length=None, - bool force_single=False, - bool require_paired=False): - pass +cdef class BrokenPairedReader: + + def __cinit__(self, FastxParser parser, + int min_length=-1, + bool force_single=False, + bool require_paired=False): + + if force_single and require_paired: + raise ValueError("force_single and require_paired cannot both be set!") + + self.parser = parser + self.min_length = min_length + self.force_single = force_single + self.require_paired = require_paired + + self.record = None + + def __iter__(self): + cdef Sequence first + cdef Sequence second + cdef object err + cdef int found + cdef int read_num = 0 + cdef bool passed_length = True + + found, first, second, err = self._next() + while (found != 0): + if err is not None: + raise err + + if self.min_length > 0: + if found == 1: + if len(first) < self.min_length: + passed_length = False + else: + if len(first) < self.min_length or len(second) < self.min_length: + passed_length = False + + if passed_length: + yield read_num, found == 2, first, second + read_num += found + else: + yield read_num, found == 2, first, second + read_num += found + found, first, second, err = self._next() + passed_length = True + + cdef tuple _next(self): + cdef bool has_next + cdef Sequence first, second + + if self.record == None: + # No previous sequence. Try to get one. + first = Sequence(create=True) + has_next = self.parser._imprint_next(first) + # And none left? We're outta here. + if not has_next: + return 0, None, None, None + else: + first = self.record + + second = Sequence(create=True) + has_next = self.parser._imprint_next(second) + + # check if paired + if has_next: + if _check_is_pair(first, second) and not self.force_single: + self.record = None + return 2, first, second, None # found 2 proper records + else: # orphan. + if self.require_paired: + err = UnpairedReadsError( + "Unpaired reads when require_paired is set!", + first, second) + return 0, None, None, err + self.record = second + return 1, first, None, None + else: + # ran out of reads, handle last record + if self.require_paired: + err = UnpairedReadsError("Unpaired reads when require_paired " + "is set!", first, None) + return 0, None, None, err + self.record = None + return 1, first, None, None cdef tuple _split_left_right(str name): - pass + """Split record name at the first whitespace and return both parts. + + RHS is set to an empty string if not present. + """ + cdef list parts = name.split(None, 1) + cdef str lhs = parts[0] + cdef str rhs = parts[1] if len(parts) > 1 else '' + return lhs, rhs + + +cdef bool _check_is_pair(Sequence first, Sequence second): + """Check if the two sequence records belong to the same fragment. + In an matching pair the records are left and right pairs + of each other, respectively. Returns True or False as appropriate. -cdef bool check_is_pair(Sequence first, Sequence second): - pass + Handles both Casava formats: seq/1 and seq/2, and 'seq::... 1::...' + and 'seq::... 2::...'. + + Also handles the default format of the SRA toolkit's fastq-dump: + 'Accession seq/1' + """ + #if hasattr(first, 'quality') or hasattr(second, 'quality'): + # if not (hasattr(first, 'quality') and hasattr(first, 'quality')): + # raise ValueError("both records must be same type (FASTA or FASTQ)") + + cdef str lhs1, rhs1, lhs2, rhs2 + lhs1, rhs1 = _split_left_right(first.name) + lhs2, rhs2 = _split_left_right(second.name) + + # handle 'name/1' + cdef str subpart1, subpart2 + if lhs1.endswith('/1') and lhs2.endswith('/2'): + subpart1 = lhs1.split('/', 1)[0] + subpart2 = lhs2.split('/', 1)[0] + + if subpart1 and subpart1 == subpart2: + return True + + # handle '@name 1:rst' + elif lhs1 == lhs2 and rhs1.startswith('1:') and rhs2.startswith('2:'): + return True + + # handle @name seq/1 + elif lhs1 == lhs2 and rhs1.endswith('/1') and rhs2.endswith('/2'): + subpart1 = rhs1.split('/', 1)[0] + subpart2 = rhs2.split('/', 1)[0] + + if subpart1 and subpart1 == subpart2: + return True + + return False + + +def check_is_pair(Sequence first, Sequence second): + cdef int res = _check_is_pair(first, second) + return res == 1 cdef bool check_is_left(str name): - pass + """Check if the name belongs to a 'left' sequence (/1). + + Returns True or False. + + Handles both Casava formats: seq/1 and 'seq::... 1::...' + """ + cdef str lhs, rhs + lhs, rhs = _split_left_right(name) + if lhs.endswith('/1'): # handle 'name/1' + return True + elif rhs.startswith('1:'): # handle '@name 1:rst' + return True + + elif rhs.endswith('/1'): # handles '@name seq/1' + return True + + return False cdef bool check_is_right(str name): - pass + """Check if the name belongs to a 'right' sequence (/2). + + Returns True or False. + + Handles both Casava formats: seq/2 and 'seq::... 2::...' + """ + cdef str lhs, rhs + lhs, rhs = _split_left_right(name) + if lhs.endswith('/2'): # handle 'name/2' + return True + elif rhs.startswith('2:'): # handle '@name 2:rst' + return True + + elif rhs.endswith('/2'): # handles '@name seq/2' + return True + return False diff --git a/tests/test_cython_streaming.py b/tests/test_cython_streaming.py new file mode 100644 index 0000000000..740c08b05c --- /dev/null +++ b/tests/test_cython_streaming.py @@ -0,0 +1,154 @@ +from __future__ import print_function +from __future__ import absolute_import + +import gc +import itertools +import random + +import khmer +from khmer._oxli.streaming import Sequence, FastxParser, BrokenPairedReader +from khmer.khmer_args import estimate_optimal_with_K_and_f as optimal_fp +from khmer import reverse_complement as revcomp +from khmer import reverse_hash as revhash +from . import khmer_tst_utils as utils + +import pytest +import screed + + +def teardown(): + utils.cleanup() + + +def gather(filename, **kw): + stream = FastxParser(str(filename)) + itr = BrokenPairedReader(stream, **kw) + + x = [] + m = 0 + num = 0 + for num, is_pair, read1, read2 in itr: + if is_pair: + x.append((read1.name, read2.name)) + else: + x.append((read1.name, None)) + m += 1 + + return x, num, m + + +@pytest.fixture +def create_fastx(tmpdir): + def func(reads, fmt='fa'): + assert fmt in ['fa','fq'] + fastx_fn = tmpdir.join('test.'+fmt) + for record in reads: + if fmt == 'fa': + fastx_fn.write('>{0}\n{1}\n'.format(record.name, + record.sequence), + mode='a') + else: + fastx_fn.write('@{0}\n{1}\n+\n{2}\n'.format(record.name, + record.sequence, + record.quality), + mode='a') + print(fastx_fn.read()) + return fastx_fn + return func + + +def test_BrokenPairedReader_force_single(create_fastx): + reads = [Sequence.create('seq1/1', 'A' * 5), + Sequence.create('seq1/2', 'A' * 4), + Sequence.create('seq2/1', 'A' * 5), + Sequence.create('seq3/1', 'A' * 3), + Sequence.create('seq3/2', 'A' * 5)] + + x, n, m = gather(create_fastx(reads), force_single=True) + + expected = [('seq1/1', None), + ('seq1/2', None), + ('seq2/1', None), + ('seq3/1', None), + ('seq3/2', None)] + assert x == expected, x + assert m == 5 + assert n == 4, n + + +def test_BrokenPairedReader_OnPairs_filter_length_require_paired(create_fastx): + reads = [Sequence.create('seq1/1', 'A' * 5), + Sequence.create('seq1/2', 'A' * 4), + Sequence.create('seq3/1', 'A' * 3), + Sequence.create('seq3/2', 'A' * 5)] + + + x, n, m = gather(create_fastx(reads), min_length=4, require_paired=True) + + expected = [('seq1/1', 'seq1/2')] + assert x == expected, x + assert m == 1 + assert n == 0, n + + +def test_BrokenPairedReader_OnPairs_filter_length_require_paired_2(create_fastx): + reads = [Sequence.create('seq1/1', 'A' * 5), + Sequence.create('seq1/2', 'A' * 4), + Sequence.create('seq3/1', 'A' * 5), + Sequence.create('seq3/2', 'A' * 3)] + x, n, m = gather(create_fastx(reads), min_length=4, require_paired=True) + + expected = [('seq1/1', 'seq1/2')] + assert x == expected, x + assert m == 1 + assert n == 0, n + + +def test_BrokenPairedReader_OnPairs_filter_length_required_paired_3(create_fastx): + reads = [Sequence.create('seq1/1', 'A' * 5), + Sequence.create('seq1/2', 'A' * 4), + Sequence.create('seq3/1', 'A' * 3), + Sequence.create('seq3/2', 'A' * 3)] + x, n, m = gather(create_fastx(reads), min_length=4, require_paired=True) + + expected = [('seq1/1', 'seq1/2')] + assert x == expected, x + assert m == 1 + assert n == 0, n + + +def test_BrokenPairedReader_OnPairs_filter_length_require_paired_4(create_fastx): + reads = [Sequence.create('seq1/1', 'A' * 3), + Sequence.create('seq1/2', 'A' * 3), + Sequence.create('seq3/1', 'A' * 5), + Sequence.create('seq3/2', 'A' * 5)] + + x, n, m = gather(create_fastx(reads), min_length=4, require_paired=True) + + expected = [('seq3/1', 'seq3/2')] + assert x == expected, x + assert m == 1 + assert n == 0, n + +''' +def test_BrokenPairedReader_lowercase(): + reads = [Sequence.create('seq1/1', 'acgtn'), + Sequence.create('seq1/2', 'AcGtN'), + Sequence.create('seq1/2', 'aCgTn')] + + results = [] + parser = FastxParser(create_fastx(reads)) + for num, is_pair, read1, read2 in broken_paired_reader(parser): + results.append((read1, read2)) + + a, b = results[0] + assert a.sequence == 'acgtn' + assert a.cleaned_seq == 'ACGTA' + assert b.sequence == 'AcGtN' + assert b.cleaned_seq == 'ACGTA' + + c, d = results[1] + assert c.sequence == 'aCgTn' + assert c.cleaned_seq == 'ACGTA' + assert d is None +''' From e38002818c67ecba5e4b17594c3957a929eff08d Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 16 Nov 2016 17:22:13 -0800 Subject: [PATCH 040/185] Remove Cython-generated cpp files from main branch --- .gitignore | 1 + khmer/_oxli/hashing.cpp | 3569 --------------- khmer/_oxli/partitioning.cpp | 7995 ---------------------------------- khmer/_oxli/traversal.cpp | 4360 ------------------ 4 files changed, 1 insertion(+), 15924 deletions(-) delete mode 100644 khmer/_oxli/hashing.cpp delete mode 100644 khmer/_oxli/partitioning.cpp delete mode 100644 khmer/_oxli/traversal.cpp diff --git a/.gitignore b/.gitignore index 7df2e67534..91884b037d 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ pylint_report.txt pep8_report.txt pep257_report.txt .cache/ +khmer/_oxli/*.cpp diff --git a/khmer/_oxli/hashing.cpp b/khmer/_oxli/hashing.cpp deleted file mode 100644 index 2a5066a163..0000000000 --- a/khmer/_oxli/hashing.cpp +++ /dev/null @@ -1,3569 +0,0 @@ -/* Generated by Cython 0.25.1 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) - #error Cython requires Python 2.6+ or Python 3.2+. -#else -#define CYTHON_ABI "0_25_1" -#include -#ifndef offsetof - #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#endif -#ifndef DL_IMPORT - #define DL_IMPORT(t) t -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#ifndef HAVE_LONG_LONG - #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) - #define HAVE_LONG_LONG - #endif -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef Py_HUGE_VAL - #define Py_HUGE_VAL HUGE_VAL -#endif -#ifdef PYPY_VERSION - #define CYTHON_COMPILING_IN_PYPY 1 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #undef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 0 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #undef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #undef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 1 - #undef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 0 - #undef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 0 - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 -#elif defined(PYSTON_VERSION) - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 1 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 -#else - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 1 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #if PY_MAJOR_VERSION < 3 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #elif !defined(CYTHON_USE_ASYNC_SLOTS) - #define CYTHON_USE_ASYNC_SLOTS 1 - #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) - #define CYTHON_USE_PYLONG_INTERNALS 1 - #endif - #ifndef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 1 - #endif - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #if PY_VERSION_HEX < 0x030300F0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #elif !defined(CYTHON_USE_UNICODE_WRITER) - #define CYTHON_USE_UNICODE_WRITER 1 - #endif - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #ifndef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 1 - #endif - #ifndef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 1 - #endif -#endif -#if !defined(CYTHON_FAST_PYCCALL) -#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) -#endif -#if CYTHON_USE_PYLONG_INTERNALS - #include "longintrepr.h" - #undef SHIFT - #undef BASE - #undef MASK -#endif -#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) - #define Py_OptimizeFlag 0 -#endif -#define __PYX_BUILD_PY_SSIZE_T "n" -#define CYTHON_FORMAT_SSIZE_T "z" -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyClass_Type -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyType_Type -#endif -#ifndef Py_TPFLAGS_CHECKTYPES - #define Py_TPFLAGS_CHECKTYPES 0 -#endif -#ifndef Py_TPFLAGS_HAVE_INDEX - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#ifndef Py_TPFLAGS_HAVE_NEWBUFFER - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#ifndef Py_TPFLAGS_HAVE_FINALIZE - #define Py_TPFLAGS_HAVE_FINALIZE 0 -#endif -#ifndef METH_FASTCALL - #define METH_FASTCALL 0x80 - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); -#else - #define __Pyx_PyCFunctionFast _PyCFunctionFast -#endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) -#else -#define __Pyx_PyFastCFunction_Check(func) 0 -#endif -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) - #define CYTHON_PEP393_ENABLED 1 - #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ - 0 : _PyUnicode_Ready((PyObject *)(op))) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) - #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) - #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) -#else - #define CYTHON_PEP393_ENABLED 0 - #define PyUnicode_1BYTE_KIND 1 - #define PyUnicode_2BYTE_KIND 2 - #define PyUnicode_4BYTE_KIND 4 - #define __Pyx_PyUnicode_READY(op) (0) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) - #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) -#endif -#if CYTHON_COMPILING_IN_PYPY - #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) -#else - #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ - PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) - #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) - #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) - #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) - #define PyObject_Malloc(s) PyMem_Malloc(s) - #define PyObject_Free(p) PyMem_Free(p) - #define PyObject_Realloc(p) PyMem_Realloc(p) -#endif -#if CYTHON_COMPILING_IN_PYSTON - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) -#else - #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) -#endif -#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) -#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) -#else - #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) -#endif -#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) - #define PyObject_ASCII(o) PyObject_Repr(o) -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyStringObject PyUnicodeObject - #define PyString_Type PyUnicode_Type - #define PyString_Check PyUnicode_Check - #define PyString_CheckExact PyUnicode_CheckExact -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) - #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) -#else - #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) - #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) -#endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) -#endif -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) -#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) -#if PY_MAJOR_VERSION >= 3 - #define PyIntObject PyLongObject - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define PyNumber_Int PyNumber_Long -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBoolObject PyLongObject -#endif -#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY - #ifndef PyUnicode_InternFromString - #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) - #endif -#endif -#if PY_VERSION_HEX < 0x030200A4 - typedef long Py_hash_t; - #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong -#else - #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) -#endif -#if CYTHON_USE_ASYNC_SLOTS - #if PY_VERSION_HEX >= 0x030500B1 - #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods - #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) - #else - typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; - } __Pyx_PyAsyncMethodsStruct; - #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) - #endif -#else - #define __Pyx_PyType_AsAsync(obj) NULL -#endif -#ifndef CYTHON_RESTRICT - #if defined(__GNUC__) - #define CYTHON_RESTRICT __restrict__ - #elif defined(_MSC_VER) && _MSC_VER >= 1400 - #define CYTHON_RESTRICT __restrict - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_RESTRICT restrict - #else - #define CYTHON_RESTRICT - #endif -#endif -#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) - -#ifndef __cplusplus - #error "Cython files generated with the C++ option must be compiled with a C++ compiler." -#endif -#ifndef CYTHON_INLINE - #define CYTHON_INLINE inline -#endif -template -void __Pyx_call_destructor(T& x) { - x.~T(); -} -template -class __Pyx_FakeReference { - public: - __Pyx_FakeReference() : ptr(NULL) { } - __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } - T *operator->() { return ptr; } - operator T&() { return *ptr; } - template bool operator ==(U other) { return *ptr == other; }; - template bool operator !=(U other) { return *ptr != other; }; - private: - T *ptr; -}; - -#if defined(WIN32) || defined(MS_WINDOWS) - #define _USE_MATH_DEFINES -#endif -#include -#ifdef NAN -#define __PYX_NAN() ((float) NAN) -#else -static CYTHON_INLINE float __PYX_NAN() { - float value; - memset(&value, 0xFF, sizeof(value)); - return value; -} -#endif -#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) -#define __Pyx_truncl trunc -#else -#define __Pyx_truncl truncl -#endif - - -#define __PYX_ERR(f_index, lineno, Ln_error) \ -{ \ - __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ -} - -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) -#endif - -#ifndef __PYX_EXTERN_C - #ifdef __cplusplus - #define __PYX_EXTERN_C extern "C" - #else - #define __PYX_EXTERN_C extern - #endif -#endif - -#define __PYX_HAVE__khmer___oxli__hashing -#define __PYX_HAVE_API__khmer___oxli__hashing -#include -#include "ios" -#include "new" -#include "stdexcept" -#include "typeinfo" -#include -#include -#include -#include -#include -#include -#include -#include -#include "khmer.hh" -#include "kmer_hash.hh" -#include "_khmer.hh" -#include "hashtable.hh" -#include "traversal.hh" -#include "partitioning.hh" -#ifdef _OPENMP -#include -#endif /* _OPENMP */ - -#ifdef PYREX_WITHOUT_ASSERTIONS -#define CYTHON_WITHOUT_ASSERTIONS -#endif - -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif -typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; - const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; - -#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 -#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 -#define __PYX_DEFAULT_STRING_ENCODING "" -#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString -#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#define __Pyx_uchar_cast(c) ((unsigned char)c) -#define __Pyx_long_cast(x) ((long)x) -#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ - (sizeof(type) < sizeof(Py_ssize_t)) ||\ - (sizeof(type) > sizeof(Py_ssize_t) &&\ - likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX) &&\ - (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ - v == (type)PY_SSIZE_T_MIN))) ||\ - (sizeof(type) == sizeof(Py_ssize_t) &&\ - (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX))) ) -#if defined (__cplusplus) && __cplusplus >= 201103L - #include - #define __Pyx_sst_abs(value) std::abs(value) -#elif SIZEOF_INT >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) abs(value) -#elif SIZEOF_LONG >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) labs(value) -#elif defined (_MSC_VER) && defined (_M_X64) - #define __Pyx_sst_abs(value) _abs64(value) -#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define __Pyx_sst_abs(value) llabs(value) -#elif defined (__GNUC__) - #define __Pyx_sst_abs(value) __builtin_llabs(value) -#else - #define __Pyx_sst_abs(value) ((value<0) ? -value : value) -#endif -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); -#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) -#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); -#if PY_MAJOR_VERSION < 3 - #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#else - #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize -#endif -#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) -#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) -#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) -#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) -#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -#if PY_MAJOR_VERSION < 3 -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) -{ - const Py_UNICODE *u_end = u; - while (*u_end++) ; - return (size_t)(u_end - u - 1); -} -#else -#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen -#endif -#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) -#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode -#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode -#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) -#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) -#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -#if CYTHON_ASSUME_SAFE_MACROS -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) -#else -#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) -#endif -#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) -#if PY_MAJOR_VERSION >= 3 -#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) -#else -#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) -#endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII -static int __Pyx_sys_getdefaultencoding_not_ascii; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - PyObject* ascii_chars_u = NULL; - PyObject* ascii_chars_b = NULL; - const char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - if (strcmp(default_encoding_c, "ascii") == 0) { - __Pyx_sys_getdefaultencoding_not_ascii = 0; - } else { - char ascii_chars[128]; - int c; - for (c = 0; c < 128; c++) { - ascii_chars[c] = c; - } - __Pyx_sys_getdefaultencoding_not_ascii = 1; - ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); - if (!ascii_chars_u) goto bad; - ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); - if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { - PyErr_Format( - PyExc_ValueError, - "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", - default_encoding_c); - goto bad; - } - Py_DECREF(ascii_chars_u); - Py_DECREF(ascii_chars_b); - } - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - Py_XDECREF(ascii_chars_u); - Py_XDECREF(ascii_chars_b); - return -1; -} -#endif -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) -#else -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT -static char* __PYX_DEFAULT_STRING_ENCODING; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); - if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; - strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - return -1; -} -#endif -#endif - - -/* Test for GCC > 2.95 */ -#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) - #define likely(x) __builtin_expect(!!(x), 1) - #define unlikely(x) __builtin_expect(!!(x), 0) -#else /* !__GNUC__ or GCC < 2.95 */ - #define likely(x) (x) - #define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_d; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static PyObject *__pyx_empty_bytes; -static PyObject *__pyx_empty_unicode; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; - - -static const char *__pyx_f[] = { - "khmer/_oxli/hashing.pyx", - "stringsource", -}; - -/*--- Type declarations ---*/ -struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer; - -/* "khmer/_oxli/hashing.pxd":4 - * from _oxli cimport CpKmer, HashIntoType, WordLength - * - * cdef class Kmer: # <<<<<<<<<<<<<< - * cdef shared_ptr[CpKmer] _this - * cdef readonly str kmer - */ -struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer { - PyObject_HEAD - struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer *__pyx_vtab; - std::shared_ptr _this; - PyObject *kmer; -}; - - - -/* "khmer/_oxli/hashing.pyx":8 - * from _oxli cimport _revhash - * - * cdef class Kmer: # <<<<<<<<<<<<<< - * - * def __cinit__(self, str kmer=None): - */ - -struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer { - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*wrap)(khmer::Kmer *, khmer::WordLength); - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*create)(khmer::HashIntoType, khmer::WordLength); -}; -static struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer *__pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer; - -/* --- Runtime support code (head) --- */ -/* Refnanny.proto */ -#ifndef CYTHON_REFNANNY - #define CYTHON_REFNANNY 0 -#endif -#if CYTHON_REFNANNY - typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); - void (*FinishContext)(void**); - } __Pyx_RefNannyAPIStruct; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); - #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; -#ifdef WITH_THREAD - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - if (acquire_gil) {\ - PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - PyGILState_Release(__pyx_gilstate_save);\ - } else {\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - } -#else - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) -#endif - #define __Pyx_RefNannyFinishContext()\ - __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) -#else - #define __Pyx_RefNannyDeclarations - #define __Pyx_RefNannySetupContext(name, acquire_gil) - #define __Pyx_RefNannyFinishContext() - #define __Pyx_INCREF(r) Py_INCREF(r) - #define __Pyx_DECREF(r) Py_DECREF(r) - #define __Pyx_GOTREF(r) - #define __Pyx_GIVEREF(r) - #define __Pyx_XINCREF(r) Py_XINCREF(r) - #define __Pyx_XDECREF(r) Py_XDECREF(r) - #define __Pyx_XGOTREF(r) - #define __Pyx_XGIVEREF(r) -#endif -#define __Pyx_XDECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_XDECREF(tmp);\ - } while (0) -#define __Pyx_DECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_DECREF(tmp);\ - } while (0) -#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) -#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) - -/* RaiseDoubleKeywords.proto */ -static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); - -/* ParseKeywords.proto */ -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ - const char* function_name); - -/* RaiseArgTupleInvalid.proto */ -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); - -/* ArgTypeTest.proto */ -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact); - -/* PyObjectGetAttrStr.proto */ -#if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { - PyTypeObject* tp = Py_TYPE(obj); - if (likely(tp->tp_getattro)) - return tp->tp_getattro(obj, attr_name); -#if PY_MAJOR_VERSION < 3 - if (likely(tp->tp_getattr)) - return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); -#endif - return PyObject_GetAttr(obj, attr_name); -} -#else -#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) -#endif - -/* PyObjectCall.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); -#else -#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) -#endif - -/* IncludeCppStringH.proto */ -#include - -/* decode_c_bytes.proto */ -static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( - const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, - const char* encoding, const char* errors, - PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)); - -/* decode_cpp_string.proto */ -static CYTHON_INLINE PyObject* __Pyx_decode_cpp_string( - std::string cppstring, Py_ssize_t start, Py_ssize_t stop, - const char* encoding, const char* errors, - PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { - return __Pyx_decode_c_bytes( - cppstring.data(), cppstring.size(), start, stop, encoding, errors, decode_func); -} - -/* SetVTable.proto */ -static int __Pyx_SetVtable(PyObject *dict, void *vtable); - -/* CodeObjectCache.proto */ -typedef struct { - PyCodeObject* code_object; - int code_line; -} __Pyx_CodeObjectCacheEntry; -struct __Pyx_CodeObjectCache { - int count; - int max_count; - __Pyx_CodeObjectCacheEntry* entries; -}; -static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); -static PyCodeObject *__pyx_find_code_object(int code_line); -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); - -/* AddTraceback.proto */ -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename); - -/* None.proto */ -#include - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE khmer::HashIntoType __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(PyObject *); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); - -/* CheckBinaryVersion.proto */ -static int __Pyx_check_binary_version(void); - -/* InitStrings.proto */ -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); - -static struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_f_5khmer_5_oxli_7hashing_4Kmer_wrap(khmer::Kmer *__pyx_v_cpkmer, khmer::WordLength __pyx_v_K); /* proto*/ -static struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_f_5khmer_5_oxli_7hashing_4Kmer_create(khmer::HashIntoType __pyx_v_tag, khmer::WordLength __pyx_v_K); /* proto*/ - -/* Module declarations from 'libcpp' */ - -/* Module declarations from 'libcpp.memory' */ - -/* Module declarations from 'libc.string' */ - -/* Module declarations from 'libcpp.string' */ - -/* Module declarations from 'libcpp.vector' */ - -/* Module declarations from 'libcpp.utility' */ - -/* Module declarations from 'libcpp.map' */ - -/* Module declarations from 'libcpp.set' */ - -/* Module declarations from 'libcpp.queue' */ - -/* Module declarations from 'libc.stdint' */ - -/* Module declarations from 'khmer._oxli._oxli' */ - -/* Module declarations from 'khmer._oxli.hashing' */ -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_7hashing_Kmer = 0; -static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ -static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_string(std::string const &); /*proto*/ -static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_string(std::string const &); /*proto*/ -static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(std::string const &); /*proto*/ -static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string(std::string const &); /*proto*/ -static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_string(std::string const &); /*proto*/ -#define __Pyx_MODULE_NAME "khmer._oxli.hashing" -int __pyx_module_is_main_khmer___oxli__hashing = 0; - -/* Implementation of 'khmer._oxli.hashing' */ -static const char __pyx_k_kmer[] = "kmer"; -static const char __pyx_k_main[] = "__main__"; -static const char __pyx_k_test[] = "__test__"; -static const char __pyx_k_utf_8[] = "utf-8"; -static const char __pyx_k_encode[] = "encode"; -static const char __pyx_k_kmer_u[] = "kmer_u"; -static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; -static PyObject *__pyx_n_s_encode; -static PyObject *__pyx_n_s_kmer; -static PyObject *__pyx_n_s_kmer_u; -static PyObject *__pyx_n_s_main; -static PyObject *__pyx_n_s_pyx_vtable; -static PyObject *__pyx_n_s_test; -static PyObject *__pyx_kp_s_utf_8; -static int __pyx_pf_5khmer_5_oxli_7hashing_4Kmer___cinit__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ -static Py_ssize_t __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_2__len__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4__str__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ -static Py_hash_t __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6__hash__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_f___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_r___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_u___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4kmer___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self); /* proto */ -static PyObject *__pyx_tp_new_5khmer_5_oxli_7hashing_Kmer(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tuple_; - -/* "khmer/_oxli/hashing.pyx":10 - * cdef class Kmer: - * - * def __cinit__(self, str kmer=None): # <<<<<<<<<<<<<< - * self.kmer = kmer - * if self.kmer is not None: - */ - -/* Python wrapper */ -static int __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_kmer = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_kmer,0}; - PyObject* values[1] = {0}; - values[0] = ((PyObject*)Py_None); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_kmer); - if (value) { values[0] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 10, __pyx_L3_error) - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_kmer = ((PyObject*)values[0]); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 10, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_kmer), (&PyString_Type), 1, "kmer", 1))) __PYX_ERR(0, 10, __pyx_L1_error) - __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer___cinit__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self), __pyx_v_kmer); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5khmer_5_oxli_7hashing_4Kmer___cinit__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self, PyObject *__pyx_v_kmer) { - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - std::string __pyx_t_5; - Py_ssize_t __pyx_t_6; - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "khmer/_oxli/hashing.pyx":11 - * - * def __cinit__(self, str kmer=None): - * self.kmer = kmer # <<<<<<<<<<<<<< - * if self.kmer is not None: - * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) - */ - __Pyx_INCREF(__pyx_v_kmer); - __Pyx_GIVEREF(__pyx_v_kmer); - __Pyx_GOTREF(__pyx_v_self->kmer); - __Pyx_DECREF(__pyx_v_self->kmer); - __pyx_v_self->kmer = __pyx_v_kmer; - - /* "khmer/_oxli/hashing.pyx":12 - * def __cinit__(self, str kmer=None): - * self.kmer = kmer - * if self.kmer is not None: # <<<<<<<<<<<<<< - * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) - * else: - */ - __pyx_t_1 = (__pyx_v_self->kmer != ((PyObject*)Py_None)); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "khmer/_oxli/hashing.pyx":13 - * self.kmer = kmer - * if self.kmer is not None: - * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) # <<<<<<<<<<<<<< - * else: - * self._this.reset(new CpKmer()) - */ - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 13, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 13, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_5 = __pyx_convert_string_from_py_std__in_string(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 13, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_6 = PyObject_Length(__pyx_v_kmer); if (unlikely(__pyx_t_6 == -1)) __PYX_ERR(0, 13, __pyx_L1_error) - __pyx_v_self->_this.reset(new khmer::Kmer(__pyx_t_5, __pyx_t_6)); - - /* "khmer/_oxli/hashing.pyx":12 - * def __cinit__(self, str kmer=None): - * self.kmer = kmer - * if self.kmer is not None: # <<<<<<<<<<<<<< - * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) - * else: - */ - goto __pyx_L3; - } - - /* "khmer/_oxli/hashing.pyx":15 - * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) - * else: - * self._this.reset(new CpKmer()) # <<<<<<<<<<<<<< - * - * def __len__(self): - */ - /*else*/ { - __pyx_v_self->_this.reset(new khmer::Kmer()); - } - __pyx_L3:; - - /* "khmer/_oxli/hashing.pyx":10 - * cdef class Kmer: - * - * def __cinit__(self, str kmer=None): # <<<<<<<<<<<<<< - * self.kmer = kmer - * if self.kmer is not None: - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pyx":17 - * self._this.reset(new CpKmer()) - * - * def __len__(self): # <<<<<<<<<<<<<< - * return len(self.kmer) - * - */ - -/* Python wrapper */ -static Py_ssize_t __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_3__len__(PyObject *__pyx_v_self); /*proto*/ -static Py_ssize_t __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_3__len__(PyObject *__pyx_v_self) { - Py_ssize_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_2__len__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static Py_ssize_t __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_2__len__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { - Py_ssize_t __pyx_r; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - Py_ssize_t __pyx_t_2; - __Pyx_RefNannySetupContext("__len__", 0); - - /* "khmer/_oxli/hashing.pyx":18 - * - * def __len__(self): - * return len(self.kmer) # <<<<<<<<<<<<<< - * - * def __str__(self): - */ - __pyx_t_1 = __pyx_v_self->kmer; - __Pyx_INCREF(__pyx_t_1); - __pyx_t_2 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 18, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_2; - goto __pyx_L0; - - /* "khmer/_oxli/hashing.pyx":17 - * self._this.reset(new CpKmer()) - * - * def __len__(self): # <<<<<<<<<<<<<< - * return len(self.kmer) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.__len__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pyx":20 - * return len(self.kmer) - * - * def __str__(self): # <<<<<<<<<<<<<< - * return self.kmer - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_5__str__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_5__str__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4__str__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4__str__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__str__", 0); - - /* "khmer/_oxli/hashing.pyx":21 - * - * def __str__(self): - * return self.kmer # <<<<<<<<<<<<<< - * - * def __hash__(self): - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_self->kmer); - __pyx_r = __pyx_v_self->kmer; - goto __pyx_L0; - - /* "khmer/_oxli/hashing.pyx":20 - * return len(self.kmer) - * - * def __str__(self): # <<<<<<<<<<<<<< - * return self.kmer - * - */ - - /* function exit code */ - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pyx":23 - * return self.kmer - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return self.kmer_u - * - */ - -/* Python wrapper */ -static Py_hash_t __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_7__hash__(PyObject *__pyx_v_self); /*proto*/ -static Py_hash_t __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_7__hash__(PyObject *__pyx_v_self) { - Py_hash_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6__hash__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static Py_hash_t __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6__hash__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { - Py_hash_t __pyx_r; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - Py_hash_t __pyx_t_2; - __Pyx_RefNannySetupContext("__hash__", 0); - - /* "khmer/_oxli/hashing.pyx":24 - * - * def __hash__(self): - * return self.kmer_u # <<<<<<<<<<<<<< - * - * @property - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_kmer_u); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 24, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_AsHash_t(__pyx_t_1); if (unlikely((__pyx_t_2 == (Py_hash_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 24, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_2; - goto __pyx_L0; - - /* "khmer/_oxli/hashing.pyx":23 - * return self.kmer - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return self.kmer_u - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.__hash__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pyx":27 - * - * @property - * def kmer_f(self): # <<<<<<<<<<<<<< - * return deref(self._this).kmer_f - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_f_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_f_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_f___get__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_f___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli/hashing.pyx":28 - * @property - * def kmer_f(self): - * return deref(self._this).kmer_f # <<<<<<<<<<<<<< - * - * @property - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_v_self->_this).kmer_f); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/hashing.pyx":27 - * - * @property - * def kmer_f(self): # <<<<<<<<<<<<<< - * return deref(self._this).kmer_f - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.kmer_f.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pyx":31 - * - * @property - * def kmer_r(self): # <<<<<<<<<<<<<< - * return deref(self._this).kmer_r - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_r_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_r_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_r___get__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_r___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli/hashing.pyx":32 - * @property - * def kmer_r(self): - * return deref(self._this).kmer_r # <<<<<<<<<<<<<< - * - * @property - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_v_self->_this).kmer_r); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 32, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/hashing.pyx":31 - * - * @property - * def kmer_r(self): # <<<<<<<<<<<<<< - * return deref(self._this).kmer_r - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.kmer_r.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pyx":35 - * - * @property - * def kmer_u(self): # <<<<<<<<<<<<<< - * return deref(self._this).kmer_u - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_u_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_u_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_u___get__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_6kmer_u___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli/hashing.pyx":36 - * @property - * def kmer_u(self): - * return deref(self._this).kmer_u # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType((*__pyx_v_self->_this).kmer_u); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 36, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/hashing.pyx":35 - * - * @property - * def kmer_u(self): # <<<<<<<<<<<<<< - * return deref(self._this).kmer_u - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.kmer_u.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pyx":39 - * - * @staticmethod - * cdef Kmer wrap(CpKmer * cpkmer, WordLength K): # <<<<<<<<<<<<<< - * cdef Kmer kmer = Kmer() - * kmer._this.reset(cpkmer) - */ - -static struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_f_5khmer_5_oxli_7hashing_4Kmer_wrap(khmer::Kmer *__pyx_v_cpkmer, khmer::WordLength __pyx_v_K) { - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer = 0; - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - khmer::HashIntoType __pyx_t_2; - __Pyx_RefNannySetupContext("wrap", 0); - - /* "khmer/_oxli/hashing.pyx":40 - * @staticmethod - * cdef Kmer wrap(CpKmer * cpkmer, WordLength K): - * cdef Kmer kmer = Kmer() # <<<<<<<<<<<<<< - * kmer._this.reset(cpkmer) - * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') - */ - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_7hashing_Kmer), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 40, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_kmer = ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_t_1); - __pyx_t_1 = 0; - - /* "khmer/_oxli/hashing.pyx":41 - * cdef Kmer wrap(CpKmer * cpkmer, WordLength K): - * cdef Kmer kmer = Kmer() - * kmer._this.reset(cpkmer) # <<<<<<<<<<<<<< - * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') - * return kmer - */ - __pyx_v_kmer->_this.reset(__pyx_v_cpkmer); - - /* "khmer/_oxli/hashing.pyx":42 - * cdef Kmer kmer = Kmer() - * kmer._this.reset(cpkmer) - * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') # <<<<<<<<<<<<<< - * return kmer - * - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_kmer), __pyx_n_s_kmer_u); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(__pyx_t_1); if (unlikely((__pyx_t_2 == ((khmer::HashIntoType)-1)) && PyErr_Occurred())) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_decode_cpp_string(khmer::_revhash(__pyx_t_2, __pyx_v_K), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (!(likely(PyString_CheckExact(__pyx_t_1))||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GIVEREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_v_kmer->kmer); - __Pyx_DECREF(__pyx_v_kmer->kmer); - __pyx_v_kmer->kmer = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; - - /* "khmer/_oxli/hashing.pyx":43 - * kmer._this.reset(cpkmer) - * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') - * return kmer # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __Pyx_XDECREF(((PyObject *)__pyx_r)); - __Pyx_INCREF(((PyObject *)__pyx_v_kmer)); - __pyx_r = __pyx_v_kmer; - goto __pyx_L0; - - /* "khmer/_oxli/hashing.pyx":39 - * - * @staticmethod - * cdef Kmer wrap(CpKmer * cpkmer, WordLength K): # <<<<<<<<<<<<<< - * cdef Kmer kmer = Kmer() - * kmer._this.reset(cpkmer) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.wrap", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF((PyObject *)__pyx_v_kmer); - __Pyx_XGIVEREF((PyObject *)__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pyx":46 - * - * @staticmethod - * cdef Kmer create(HashIntoType tag, WordLength K): # <<<<<<<<<<<<<< - * cdef Kmer kmer = Kmer() - * deref(kmer._this).set_from_unique_hash(tag, K) - */ - -static struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_f_5khmer_5_oxli_7hashing_4Kmer_create(khmer::HashIntoType __pyx_v_tag, khmer::WordLength __pyx_v_K) { - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer = 0; - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - khmer::HashIntoType __pyx_t_2; - __Pyx_RefNannySetupContext("create", 0); - - /* "khmer/_oxli/hashing.pyx":47 - * @staticmethod - * cdef Kmer create(HashIntoType tag, WordLength K): - * cdef Kmer kmer = Kmer() # <<<<<<<<<<<<<< - * deref(kmer._this).set_from_unique_hash(tag, K) - * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') - */ - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_7hashing_Kmer), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 47, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_kmer = ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_t_1); - __pyx_t_1 = 0; - - /* "khmer/_oxli/hashing.pyx":48 - * cdef Kmer create(HashIntoType tag, WordLength K): - * cdef Kmer kmer = Kmer() - * deref(kmer._this).set_from_unique_hash(tag, K) # <<<<<<<<<<<<<< - * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') - * return kmer - */ - (*__pyx_v_kmer->_this).set_from_unique_hash(__pyx_v_tag, __pyx_v_K); - - /* "khmer/_oxli/hashing.pyx":49 - * cdef Kmer kmer = Kmer() - * deref(kmer._this).set_from_unique_hash(tag, K) - * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') # <<<<<<<<<<<<<< - * return kmer - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_kmer), __pyx_n_s_kmer_u); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(__pyx_t_1); if (unlikely((__pyx_t_2 == ((khmer::HashIntoType)-1)) && PyErr_Occurred())) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_decode_cpp_string(khmer::_revhash(__pyx_t_2, __pyx_v_K), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (!(likely(PyString_CheckExact(__pyx_t_1))||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 49, __pyx_L1_error) - __Pyx_GIVEREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_v_kmer->kmer); - __Pyx_DECREF(__pyx_v_kmer->kmer); - __pyx_v_kmer->kmer = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; - - /* "khmer/_oxli/hashing.pyx":50 - * deref(kmer._this).set_from_unique_hash(tag, K) - * kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') - * return kmer # <<<<<<<<<<<<<< - */ - __Pyx_XDECREF(((PyObject *)__pyx_r)); - __Pyx_INCREF(((PyObject *)__pyx_v_kmer)); - __pyx_r = __pyx_v_kmer; - goto __pyx_L0; - - /* "khmer/_oxli/hashing.pyx":46 - * - * @staticmethod - * cdef Kmer create(HashIntoType tag, WordLength K): # <<<<<<<<<<<<<< - * cdef Kmer kmer = Kmer() - * deref(kmer._this).set_from_unique_hash(tag, K) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.hashing.Kmer.create", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF((PyObject *)__pyx_v_kmer); - __Pyx_XGIVEREF((PyObject *)__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/hashing.pxd":6 - * cdef class Kmer: - * cdef shared_ptr[CpKmer] _this - * cdef readonly str kmer # <<<<<<<<<<<<<< - * - * @staticmethod - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_4kmer_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_4kmer_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4kmer___get__(((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_7hashing_4Kmer_4kmer___get__(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_self->kmer); - __pyx_r = __pyx_v_self->kmer; - goto __pyx_L0; - - /* function exit code */ - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - -static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { - Py_ssize_t __pyx_v_length; - char *__pyx_v_data; - std::string __pyx_r; - __Pyx_RefNannyDeclarations - char *__pyx_t_1; - __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); - - /* "string.from_py":15 - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< - * return string(data, length) - * - */ - __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(1, 15, __pyx_L1_error) - __pyx_v_data = __pyx_t_1; - - /* "string.from_py":16 - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - * return string(data, length) # <<<<<<<<<<<<<< - * - * - */ - __pyx_r = std::string(__pyx_v_data, __pyx_v_length); - goto __pyx_L0; - - /* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "string.to_py":31 - * - * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) - * cdef extern from *: - */ - -static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_string(std::string const &__pyx_v_s) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__pyx_convert_PyObject_string_to_py_std__in_string", 0); - - /* "string.to_py":32 - * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): - * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< - * cdef extern from *: - * cdef object __Pyx_PyUnicode_FromStringAndSize(char*, size_t) - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 32, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "string.to_py":31 - * - * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) - * cdef extern from *: - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("string.to_py.__pyx_convert_PyObject_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "string.to_py":37 - * - * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) - * cdef extern from *: - */ - -static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_string(std::string const &__pyx_v_s) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__pyx_convert_PyUnicode_string_to_py_std__in_string", 0); - - /* "string.to_py":38 - * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): - * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< - * cdef extern from *: - * cdef object __Pyx_PyStr_FromStringAndSize(char*, size_t) - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyUnicode_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 38, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "string.to_py":37 - * - * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) - * cdef extern from *: - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("string.to_py.__pyx_convert_PyUnicode_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "string.to_py":43 - * - * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) - * cdef extern from *: - */ - -static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(std::string const &__pyx_v_s) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__pyx_convert_PyStr_string_to_py_std__in_string", 0); - - /* "string.to_py":44 - * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): - * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< - * cdef extern from *: - * cdef object __Pyx_PyBytes_FromStringAndSize(char*, size_t) - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyStr_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "string.to_py":43 - * - * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) - * cdef extern from *: - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("string.to_py.__pyx_convert_PyStr_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "string.to_py":49 - * - * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) - * cdef extern from *: - */ - -static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string(std::string const &__pyx_v_s) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__pyx_convert_PyBytes_string_to_py_std__in_string", 0); - - /* "string.to_py":50 - * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): - * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< - * cdef extern from *: - * cdef object __Pyx_PyByteArray_FromStringAndSize(char*, size_t) - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 50, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "string.to_py":49 - * - * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) - * cdef extern from *: - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("string.to_py.__pyx_convert_PyBytes_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "string.to_py":55 - * - * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) - * - */ - -static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_string(std::string const &__pyx_v_s) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__pyx_convert_PyByteArray_string_to_py_std__in_string", 0); - - /* "string.to_py":56 - * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): - * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyByteArray_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 56, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "string.to_py":55 - * - * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("string.to_py.__pyx_convert_PyByteArray_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer __pyx_vtable_5khmer_5_oxli_7hashing_Kmer; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_7hashing_Kmer(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)o); - p->__pyx_vtab = __pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer; - new((void*)&(p->_this)) std::shared_ptr (); - p->kmer = ((PyObject*)Py_None); Py_INCREF(Py_None); - if (unlikely(__pyx_pw_5khmer_5_oxli_7hashing_4Kmer_1__cinit__(o, a, k) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_7hashing_Kmer(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *p = (struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - __Pyx_call_destructor(p->_this); - Py_CLEAR(p->kmer); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_f(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_f_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_r(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_r_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_u(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_6kmer_u_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_4kmer_1__get__(o); -} - -static PyMethodDef __pyx_methods_5khmer_5_oxli_7hashing_Kmer[] = { - {0, 0, 0, 0} -}; - -static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_7hashing_Kmer[] = { - {(char *)"kmer_f", __pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_f, 0, (char *)0, 0}, - {(char *)"kmer_r", __pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_r, 0, (char *)0, 0}, - {(char *)"kmer_u", __pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer_u, 0, (char *)0, 0}, - {(char *)"kmer", __pyx_getprop_5khmer_5_oxli_7hashing_4Kmer_kmer, 0, (char *)0, 0}, - {0, 0, 0, 0, 0} -}; - -static PySequenceMethods __pyx_tp_as_sequence_Kmer = { - __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_3__len__, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Kmer = { - __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_3__len__, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyTypeObject __pyx_type_5khmer_5_oxli_7hashing_Kmer = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.hashing.Kmer", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_7hashing_Kmer, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - &__pyx_tp_as_sequence_Kmer, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Kmer, /*tp_as_mapping*/ - __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_7__hash__, /*tp_hash*/ - 0, /*tp_call*/ - __pyx_pw_5khmer_5_oxli_7hashing_4Kmer_5__str__, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5khmer_5_oxli_7hashing_Kmer, /*tp_methods*/ - 0, /*tp_members*/ - __pyx_getsets_5khmer_5_oxli_7hashing_Kmer, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_7hashing_Kmer, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyMethodDef __pyx_methods[] = { - {0, 0, 0, 0} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - #if PY_VERSION_HEX < 0x03020000 - { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, - #else - PyModuleDef_HEAD_INIT, - #endif - "hashing", - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, - {&__pyx_n_s_kmer, __pyx_k_kmer, sizeof(__pyx_k_kmer), 0, 0, 1, 1}, - {&__pyx_n_s_kmer_u, __pyx_k_kmer_u, sizeof(__pyx_k_kmer_u), 0, 0, 1, 1}, - {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, - {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, - {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, - {0, 0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - return 0; -} - -static int __Pyx_InitCachedConstants(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - - /* "khmer/_oxli/hashing.pyx":13 - * self.kmer = kmer - * if self.kmer is not None: - * self._this.reset(new CpKmer(kmer.encode('utf-8'), len(kmer))) # <<<<<<<<<<<<<< - * else: - * self._this.reset(new CpKmer()) - */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(0, 13, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple_); - __Pyx_GIVEREF(__pyx_tuple_); - __Pyx_RefNannyFinishContext(); - return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC inithashing(void); /*proto*/ -PyMODINIT_FUNC inithashing(void) -#else -PyMODINIT_FUNC PyInit_hashing(void); /*proto*/ -PyMODINIT_FUNC PyInit_hashing(void) -#endif -{ - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannyDeclarations - #if CYTHON_REFNANNY - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); - if (!__Pyx_RefNanny) { - PyErr_Clear(); - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); - if (!__Pyx_RefNanny) - Py_FatalError("failed to import 'refnanny' module"); - } - #endif - __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_hashing(void)", 0); - if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) - #ifdef __Pyx_CyFunction_USED - if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_FusedFunction_USED - if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Coroutine_USED - if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Generator_USED - if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_StopAsyncIteration_USED - if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - /*--- Library function declarations ---*/ - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4("hashing", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) - Py_INCREF(__pyx_d); - __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) - #if CYTHON_COMPILING_IN_PYPY - Py_INCREF(__pyx_b); - #endif - if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); - /*--- Initialize various global constants etc. ---*/ - if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - if (__pyx_module_is_main_khmer___oxli__hashing) { - if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) - } - #if PY_MAJOR_VERSION >= 3 - { - PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) - if (!PyDict_GetItemString(modules, "khmer._oxli.hashing")) { - if (unlikely(PyDict_SetItemString(modules, "khmer._oxli.hashing", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) - } - } - #endif - /*--- Builtin init code ---*/ - if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - /*--- Constants init code ---*/ - if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - /*--- Global init code ---*/ - /*--- Variable export code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - __pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer = &__pyx_vtable_5khmer_5_oxli_7hashing_Kmer; - __pyx_vtable_5khmer_5_oxli_7hashing_Kmer.wrap = (struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*)(khmer::Kmer *, khmer::WordLength))__pyx_f_5khmer_5_oxli_7hashing_4Kmer_wrap; - __pyx_vtable_5khmer_5_oxli_7hashing_Kmer.create = (struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*)(khmer::HashIntoType, khmer::WordLength))__pyx_f_5khmer_5_oxli_7hashing_4Kmer_create; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_7hashing_Kmer) < 0) __PYX_ERR(0, 8, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_7hashing_Kmer.tp_print = 0; - if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_7hashing_Kmer.tp_dict, __pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer) < 0) __PYX_ERR(0, 8, __pyx_L1_error) - if (PyObject_SetAttrString(__pyx_m, "Kmer", (PyObject *)&__pyx_type_5khmer_5_oxli_7hashing_Kmer) < 0) __PYX_ERR(0, 8, __pyx_L1_error) - __pyx_ptype_5khmer_5_oxli_7hashing_Kmer = &__pyx_type_5khmer_5_oxli_7hashing_Kmer; - /*--- Type import code ---*/ - /*--- Variable import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - - /* "khmer/_oxli/hashing.pyx":1 - * from libcpp.string cimport string # <<<<<<<<<<<<<< - * from libcpp.memory cimport make_shared - * from libc.stdint cimport uint64_t - */ - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "string.to_py":55 - * - * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") - * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< - * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) - * - */ - - /*--- Wrapped vars code ---*/ - - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - if (__pyx_m) { - if (__pyx_d) { - __Pyx_AddTraceback("init khmer._oxli.hashing", __pyx_clineno, __pyx_lineno, __pyx_filename); - } - Py_DECREF(__pyx_m); __pyx_m = 0; - } else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ImportError, "init khmer._oxli.hashing"); - } - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -/* --- Runtime support code --- */ -/* Refnanny */ -#if CYTHON_REFNANNY -static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { - PyObject *m = NULL, *p = NULL; - void *r = NULL; - m = PyImport_ImportModule((char *)modname); - if (!m) goto end; - p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); - if (!p) goto end; - r = PyLong_AsVoidPtr(p); -end: - Py_XDECREF(p); - Py_XDECREF(m); - return (__Pyx_RefNannyAPIStruct *)r; -} -#endif - -/* RaiseDoubleKeywords */ -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AsString(kw_name)); - #endif -} - -/* ParseKeywords */ -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - continue; - } - name = first_kw_arg; - #if PY_MAJOR_VERSION < 3 - if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { - while (*name) { - if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) - && _PyString_Eq(**name, key)) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - if ((**argname == key) || ( - (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) - && _PyString_Eq(**argname, key))) { - goto arg_passed_twice; - } - argname++; - } - } - } else - #endif - if (likely(PyUnicode_Check(key))) { - while (*name) { - int cmp = (**name == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**name, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - int cmp = (**argname == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**argname, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) goto arg_passed_twice; - argname++; - } - } - } else - goto invalid_keyword_type; - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -/* RaiseArgTupleInvalid */ -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *more_or_less; - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", - func_name, more_or_less, num_expected, - (num_expected == 1) ? "" : "s", num_found); -} - -/* ArgTypeTest */ -static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); -} -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - if (none_allowed && obj == Py_None) return 1; - else if (exact) { - if (likely(Py_TYPE(obj) == type)) return 1; - #if PY_MAJOR_VERSION == 2 - else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } - else { - if (likely(PyObject_TypeCheck(obj, type))) return 1; - } - __Pyx_RaiseArgumentTypeInvalid(name, obj, type); - return 0; -} - -/* PyObjectCall */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyObject *result; - ternaryfunc call = func->ob_type->tp_call; - if (unlikely(!call)) - return PyObject_Call(func, arg, kw); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = (*call)(func, arg, kw); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* decode_c_bytes */ -static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( - const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, - const char* encoding, const char* errors, - PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { - if (unlikely((start < 0) | (stop < 0))) { - if (start < 0) { - start += length; - if (start < 0) - start = 0; - } - if (stop < 0) - stop += length; - } - if (stop > length) - stop = length; - length = stop - start; - if (unlikely(length <= 0)) - return PyUnicode_FromUnicode(NULL, 0); - cstring += start; - if (decode_func) { - return decode_func(cstring, length, errors); - } else { - return PyUnicode_Decode(cstring, length, encoding, errors); - } -} - -/* SetVTable */ -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { -#if PY_VERSION_HEX >= 0x02070000 - PyObject *ob = PyCapsule_New(vtable, 0, 0); -#else - PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); -#endif - if (!ob) - goto bad; - if (PyDict_SetItem(dict, __pyx_n_s_pyx_vtable, ob) < 0) - goto bad; - Py_DECREF(ob); - return 0; -bad: - Py_XDECREF(ob); - return -1; -} - -/* CodeObjectCache */ -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { - int start = 0, mid = 0, end = count - 1; - if (end >= 0 && code_line > entries[end].code_line) { - return count; - } - while (start < end) { - mid = start + (end - start) / 2; - if (code_line < entries[mid].code_line) { - end = mid; - } else if (code_line > entries[mid].code_line) { - start = mid + 1; - } else { - return mid; - } - } - if (code_line <= entries[mid].code_line) { - return mid; - } else { - return mid + 1; - } -} -static PyCodeObject *__pyx_find_code_object(int code_line) { - PyCodeObject* code_object; - int pos; - if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { - return NULL; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { - return NULL; - } - code_object = __pyx_code_cache.entries[pos].code_object; - Py_INCREF(code_object); - return code_object; -} -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { - int pos, i; - __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; - if (unlikely(!code_line)) { - return; - } - if (unlikely(!entries)) { - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); - if (likely(entries)) { - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = 64; - __pyx_code_cache.count = 1; - entries[0].code_line = code_line; - entries[0].code_object = code_object; - Py_INCREF(code_object); - } - return; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { - PyCodeObject* tmp = entries[pos].code_object; - entries[pos].code_object = code_object; - Py_DECREF(tmp); - return; - } - if (__pyx_code_cache.count == __pyx_code_cache.max_count) { - int new_max = __pyx_code_cache.max_count + 64; - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( - __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); - if (unlikely(!entries)) { - return; - } - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = new_max; - } - for (i=__pyx_code_cache.count; i>pos; i--) { - entries[i] = entries[i-1]; - } - entries[pos].code_line = code_line; - entries[pos].code_object = code_object; - __pyx_code_cache.count++; - Py_INCREF(code_object); -} - -/* AddTraceback */ -#include "compile.h" -#include "frameobject.h" -#include "traceback.h" -static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( - const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(filename); - #else - py_srcfile = PyUnicode_FromString(filename); - #endif - if (!py_srcfile) goto bad; - if (c_line) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_code = __Pyx_PyCode_New( - 0, - 0, - 0, - 0, - 0, - __pyx_empty_bytes, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - py_line, - __pyx_empty_bytes /*PyObject *lnotab*/ - ); - Py_DECREF(py_srcfile); - Py_DECREF(py_funcname); - return py_code; -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - return NULL; -} -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - py_code = __pyx_find_code_object(c_line ? c_line : py_line); - if (!py_code) { - py_code = __Pyx_CreateCodeObjectForTraceback( - funcname, c_line, py_line, filename); - if (!py_code) goto bad; - __pyx_insert_code_object(c_line ? c_line : py_line, py_code); - } - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - __pyx_d, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - __Pyx_PyFrame_SetLineNumber(py_frame, py_line); - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value) { - const khmer::HashIntoType neg_one = (khmer::HashIntoType) -1, const_zero = (khmer::HashIntoType) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(khmer::HashIntoType) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(khmer::HashIntoType) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::HashIntoType) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(khmer::HashIntoType), - little, !is_unsigned); - } -} - -/* CIntFromPyVerify */ -#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) -#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) -#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ - {\ - func_type value = func_value;\ - if (sizeof(target_type) < sizeof(func_type)) {\ - if (unlikely(value != (func_type) (target_type) value)) {\ - func_type zero = 0;\ - if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ - return (target_type) -1;\ - if (is_unsigned && unlikely(value < zero))\ - goto raise_neg_overflow;\ - else\ - goto raise_overflow;\ - }\ - }\ - return (target_type) value;\ - } - -/* CIntFromPy */ -static CYTHON_INLINE khmer::HashIntoType __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(PyObject *x) { - const khmer::HashIntoType neg_one = (khmer::HashIntoType) -1, const_zero = (khmer::HashIntoType) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(khmer::HashIntoType) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (khmer::HashIntoType) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (khmer::HashIntoType) 0; - case 1: __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, digit, digits[0]) - case 2: - if (8 * sizeof(khmer::HashIntoType) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) >= 2 * PyLong_SHIFT) { - return (khmer::HashIntoType) (((((khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(khmer::HashIntoType) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) >= 3 * PyLong_SHIFT) { - return (khmer::HashIntoType) (((((((khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(khmer::HashIntoType) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) >= 4 * PyLong_SHIFT) { - return (khmer::HashIntoType) (((((((((khmer::HashIntoType)digits[3]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (khmer::HashIntoType) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(khmer::HashIntoType) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(khmer::HashIntoType, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(khmer::HashIntoType, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (khmer::HashIntoType) 0; - case -1: __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, digit, +digits[0]) - case -2: - if (8 * sizeof(khmer::HashIntoType) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) - 1 > 2 * PyLong_SHIFT) { - return (khmer::HashIntoType) (((khmer::HashIntoType)-1)*(((((khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(khmer::HashIntoType) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) - 1 > 2 * PyLong_SHIFT) { - return (khmer::HashIntoType) ((((((khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(khmer::HashIntoType) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) - 1 > 3 * PyLong_SHIFT) { - return (khmer::HashIntoType) (((khmer::HashIntoType)-1)*(((((((khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(khmer::HashIntoType) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) - 1 > 3 * PyLong_SHIFT) { - return (khmer::HashIntoType) ((((((((khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(khmer::HashIntoType) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) - 1 > 4 * PyLong_SHIFT) { - return (khmer::HashIntoType) (((khmer::HashIntoType)-1)*(((((((((khmer::HashIntoType)digits[3]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(khmer::HashIntoType) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(khmer::HashIntoType, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(khmer::HashIntoType) - 1 > 4 * PyLong_SHIFT) { - return (khmer::HashIntoType) ((((((((((khmer::HashIntoType)digits[3]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[2]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[1]) << PyLong_SHIFT) | (khmer::HashIntoType)digits[0]))); - } - } - break; - } -#endif - if (sizeof(khmer::HashIntoType) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(khmer::HashIntoType, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::HashIntoType) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(khmer::HashIntoType, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - khmer::HashIntoType val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (khmer::HashIntoType) -1; - } - } else { - khmer::HashIntoType val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (khmer::HashIntoType) -1; - val = __Pyx_PyInt_As_khmer_3a__3a_HashIntoType(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to khmer::HashIntoType"); - return (khmer::HashIntoType) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to khmer::HashIntoType"); - return (khmer::HashIntoType) -1; -} - -/* CIntToPy */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - -/* CIntFromPy */ -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(long) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (long) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { - return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { - return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { - return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (long) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(long) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) - case -2: - if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - } -#endif - if (sizeof(long) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - long val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (long) -1; - } - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (long) -1; - val = __Pyx_PyInt_As_long(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to long"); - return (long) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to long"); - return (long) -1; -} - -/* CIntFromPy */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) -1, const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (int) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { - return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { - return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { - return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (int) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) - case -2: - if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - } -#endif - if (sizeof(int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - int val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (int) -1; - } - } else { - int val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (int) -1; - val = __Pyx_PyInt_As_int(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to int"); - return (int) -1; -} - -/* CheckBinaryVersion */ -static int __Pyx_check_binary_version(void) { - char ctversion[4], rtversion[4]; - PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); - PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); - if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { - char message[200]; - PyOS_snprintf(message, sizeof(message), - "compiletime version %s of module '%.100s' " - "does not match runtime version %s", - ctversion, __Pyx_MODULE_NAME, rtversion); - return PyErr_WarnEx(NULL, message, 1); - } - return 0; -} - -/* InitStrings */ -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { - return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { - Py_ssize_t ignore; - return __Pyx_PyObject_AsStringAndSize(o, &ignore); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { -#if PY_VERSION_HEX < 0x03030000 - char* defenc_c; - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - PyUnicode_AsASCIIString(o); - return NULL; - } - } - } -#endif - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; -#else - if (__Pyx_PyUnicode_READY(o) == -1) return NULL; -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (PyUnicode_IS_ASCII(o)) { - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - PyUnicode_AsASCIIString(o); - return NULL; - } -#else - return PyUnicode_AsUTF8AndSize(o, length); -#endif -#endif - } else -#endif -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) - if (PyByteArray_Check(o)) { - *length = PyByteArray_GET_SIZE(o); - return PyByteArray_AS_STRING(o); - } else -#endif - { - char* result; - int r = PyBytes_AsStringAndSize(o, &result, length); - if (unlikely(r < 0)) { - return NULL; - } else { - return result; - } - } -} -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - int is_true = x == Py_True; - if (is_true | (x == Py_False) | (x == Py_None)) return is_true; - else return PyObject_IsTrue(x); -} -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { -#if CYTHON_USE_TYPE_SLOTS - PyNumberMethods *m; -#endif - const char *name = NULL; - PyObject *res = NULL; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return __Pyx_NewRef(x); -#if CYTHON_USE_TYPE_SLOTS - m = Py_TYPE(x)->tp_as_number; - #if PY_MAJOR_VERSION < 3 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } - #else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } - #endif -#else - res = PyNumber_Int(x); -#endif - if (res) { -#if PY_MAJOR_VERSION < 3 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject *x; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(b))) { - if (sizeof(Py_ssize_t) >= sizeof(long)) - return PyInt_AS_LONG(b); - else - return PyInt_AsSsize_t(x); - } -#endif - if (likely(PyLong_CheckExact(b))) { - #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)b)->ob_digit; - const Py_ssize_t size = Py_SIZE(b); - if (likely(__Pyx_sst_abs(size) <= 1)) { - ival = likely(size) ? digits[0] : 0; - if (size == -1) ival = -ival; - return ival; - } else { - switch (size) { - case 2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - } - } - #endif - return PyLong_AsSsize_t(b); - } - x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { - return PyInt_FromSize_t(ival); -} - - -#endif /* Py_PYTHON_H */ diff --git a/khmer/_oxli/partitioning.cpp b/khmer/_oxli/partitioning.cpp deleted file mode 100644 index 7de9d62c58..0000000000 --- a/khmer/_oxli/partitioning.cpp +++ /dev/null @@ -1,7995 +0,0 @@ -/* Generated by Cython 0.25.1 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) - #error Cython requires Python 2.6+ or Python 3.2+. -#else -#define CYTHON_ABI "0_25_1" -#include -#ifndef offsetof - #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#endif -#ifndef DL_IMPORT - #define DL_IMPORT(t) t -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#ifndef HAVE_LONG_LONG - #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) - #define HAVE_LONG_LONG - #endif -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef Py_HUGE_VAL - #define Py_HUGE_VAL HUGE_VAL -#endif -#ifdef PYPY_VERSION - #define CYTHON_COMPILING_IN_PYPY 1 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #undef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 0 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #undef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #undef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 1 - #undef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 0 - #undef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 0 - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 -#elif defined(PYSTON_VERSION) - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 1 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 -#else - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 1 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #if PY_MAJOR_VERSION < 3 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #elif !defined(CYTHON_USE_ASYNC_SLOTS) - #define CYTHON_USE_ASYNC_SLOTS 1 - #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) - #define CYTHON_USE_PYLONG_INTERNALS 1 - #endif - #ifndef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 1 - #endif - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #if PY_VERSION_HEX < 0x030300F0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #elif !defined(CYTHON_USE_UNICODE_WRITER) - #define CYTHON_USE_UNICODE_WRITER 1 - #endif - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #ifndef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 1 - #endif - #ifndef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 1 - #endif -#endif -#if !defined(CYTHON_FAST_PYCCALL) -#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) -#endif -#if CYTHON_USE_PYLONG_INTERNALS - #include "longintrepr.h" - #undef SHIFT - #undef BASE - #undef MASK -#endif -#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) - #define Py_OptimizeFlag 0 -#endif -#define __PYX_BUILD_PY_SSIZE_T "n" -#define CYTHON_FORMAT_SSIZE_T "z" -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyClass_Type -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyType_Type -#endif -#ifndef Py_TPFLAGS_CHECKTYPES - #define Py_TPFLAGS_CHECKTYPES 0 -#endif -#ifndef Py_TPFLAGS_HAVE_INDEX - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#ifndef Py_TPFLAGS_HAVE_NEWBUFFER - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#ifndef Py_TPFLAGS_HAVE_FINALIZE - #define Py_TPFLAGS_HAVE_FINALIZE 0 -#endif -#ifndef METH_FASTCALL - #define METH_FASTCALL 0x80 - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); -#else - #define __Pyx_PyCFunctionFast _PyCFunctionFast -#endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) -#else -#define __Pyx_PyFastCFunction_Check(func) 0 -#endif -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) - #define CYTHON_PEP393_ENABLED 1 - #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ - 0 : _PyUnicode_Ready((PyObject *)(op))) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) - #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) - #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) -#else - #define CYTHON_PEP393_ENABLED 0 - #define PyUnicode_1BYTE_KIND 1 - #define PyUnicode_2BYTE_KIND 2 - #define PyUnicode_4BYTE_KIND 4 - #define __Pyx_PyUnicode_READY(op) (0) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) - #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) -#endif -#if CYTHON_COMPILING_IN_PYPY - #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) -#else - #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ - PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) - #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) - #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) - #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) - #define PyObject_Malloc(s) PyMem_Malloc(s) - #define PyObject_Free(p) PyMem_Free(p) - #define PyObject_Realloc(p) PyMem_Realloc(p) -#endif -#if CYTHON_COMPILING_IN_PYSTON - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) -#else - #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) -#endif -#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) -#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) -#else - #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) -#endif -#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) - #define PyObject_ASCII(o) PyObject_Repr(o) -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyStringObject PyUnicodeObject - #define PyString_Type PyUnicode_Type - #define PyString_Check PyUnicode_Check - #define PyString_CheckExact PyUnicode_CheckExact -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) - #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) -#else - #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) - #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) -#endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) -#endif -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) -#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) -#if PY_MAJOR_VERSION >= 3 - #define PyIntObject PyLongObject - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define PyNumber_Int PyNumber_Long -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBoolObject PyLongObject -#endif -#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY - #ifndef PyUnicode_InternFromString - #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) - #endif -#endif -#if PY_VERSION_HEX < 0x030200A4 - typedef long Py_hash_t; - #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong -#else - #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) -#endif -#if CYTHON_USE_ASYNC_SLOTS - #if PY_VERSION_HEX >= 0x030500B1 - #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods - #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) - #else - typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; - } __Pyx_PyAsyncMethodsStruct; - #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) - #endif -#else - #define __Pyx_PyType_AsAsync(obj) NULL -#endif -#ifndef CYTHON_RESTRICT - #if defined(__GNUC__) - #define CYTHON_RESTRICT __restrict__ - #elif defined(_MSC_VER) && _MSC_VER >= 1400 - #define CYTHON_RESTRICT __restrict - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_RESTRICT restrict - #else - #define CYTHON_RESTRICT - #endif -#endif -#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) - -#ifndef __cplusplus - #error "Cython files generated with the C++ option must be compiled with a C++ compiler." -#endif -#ifndef CYTHON_INLINE - #define CYTHON_INLINE inline -#endif -template -void __Pyx_call_destructor(T& x) { - x.~T(); -} -template -class __Pyx_FakeReference { - public: - __Pyx_FakeReference() : ptr(NULL) { } - __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } - T *operator->() { return ptr; } - operator T&() { return *ptr; } - template bool operator ==(U other) { return *ptr == other; }; - template bool operator !=(U other) { return *ptr != other; }; - private: - T *ptr; -}; - -#if defined(WIN32) || defined(MS_WINDOWS) - #define _USE_MATH_DEFINES -#endif -#include -#ifdef NAN -#define __PYX_NAN() ((float) NAN) -#else -static CYTHON_INLINE float __PYX_NAN() { - float value; - memset(&value, 0xFF, sizeof(value)); - return value; -} -#endif -#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) -#define __Pyx_truncl trunc -#else -#define __Pyx_truncl truncl -#endif - - -#define __PYX_ERR(f_index, lineno, Ln_error) \ -{ \ - __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ -} - -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) -#endif - -#ifndef __PYX_EXTERN_C - #ifdef __cplusplus - #define __PYX_EXTERN_C extern "C" - #else - #define __PYX_EXTERN_C extern - #endif -#endif - -#define __PYX_HAVE__khmer___oxli__partitioning -#define __PYX_HAVE_API__khmer___oxli__partitioning -#include -#include "ios" -#include "new" -#include "stdexcept" -#include "typeinfo" -#include -#include -#include -#include -#include -#include -#include -#include -#include "khmer.hh" -#include "kmer_hash.hh" -#include "_khmer.hh" -#include "hashtable.hh" -#include "traversal.hh" -#include "partitioning.hh" -#include -#include -#ifdef _OPENMP -#include -#endif /* _OPENMP */ - -#ifdef PYREX_WITHOUT_ASSERTIONS -#define CYTHON_WITHOUT_ASSERTIONS -#endif - -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif -typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; - const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; - -#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 -#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 -#define __PYX_DEFAULT_STRING_ENCODING "" -#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString -#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#define __Pyx_uchar_cast(c) ((unsigned char)c) -#define __Pyx_long_cast(x) ((long)x) -#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ - (sizeof(type) < sizeof(Py_ssize_t)) ||\ - (sizeof(type) > sizeof(Py_ssize_t) &&\ - likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX) &&\ - (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ - v == (type)PY_SSIZE_T_MIN))) ||\ - (sizeof(type) == sizeof(Py_ssize_t) &&\ - (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX))) ) -#if defined (__cplusplus) && __cplusplus >= 201103L - #include - #define __Pyx_sst_abs(value) std::abs(value) -#elif SIZEOF_INT >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) abs(value) -#elif SIZEOF_LONG >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) labs(value) -#elif defined (_MSC_VER) && defined (_M_X64) - #define __Pyx_sst_abs(value) _abs64(value) -#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define __Pyx_sst_abs(value) llabs(value) -#elif defined (__GNUC__) - #define __Pyx_sst_abs(value) __builtin_llabs(value) -#else - #define __Pyx_sst_abs(value) ((value<0) ? -value : value) -#endif -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); -#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) -#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); -#if PY_MAJOR_VERSION < 3 - #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#else - #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize -#endif -#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) -#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) -#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) -#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) -#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -#if PY_MAJOR_VERSION < 3 -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) -{ - const Py_UNICODE *u_end = u; - while (*u_end++) ; - return (size_t)(u_end - u - 1); -} -#else -#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen -#endif -#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) -#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode -#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode -#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) -#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) -#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -#if CYTHON_ASSUME_SAFE_MACROS -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) -#else -#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) -#endif -#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) -#if PY_MAJOR_VERSION >= 3 -#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) -#else -#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) -#endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII -static int __Pyx_sys_getdefaultencoding_not_ascii; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - PyObject* ascii_chars_u = NULL; - PyObject* ascii_chars_b = NULL; - const char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - if (strcmp(default_encoding_c, "ascii") == 0) { - __Pyx_sys_getdefaultencoding_not_ascii = 0; - } else { - char ascii_chars[128]; - int c; - for (c = 0; c < 128; c++) { - ascii_chars[c] = c; - } - __Pyx_sys_getdefaultencoding_not_ascii = 1; - ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); - if (!ascii_chars_u) goto bad; - ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); - if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { - PyErr_Format( - PyExc_ValueError, - "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", - default_encoding_c); - goto bad; - } - Py_DECREF(ascii_chars_u); - Py_DECREF(ascii_chars_b); - } - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - Py_XDECREF(ascii_chars_u); - Py_XDECREF(ascii_chars_b); - return -1; -} -#endif -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) -#else -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT -static char* __PYX_DEFAULT_STRING_ENCODING; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); - if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; - strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - return -1; -} -#endif -#endif - - -/* Test for GCC > 2.95 */ -#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) - #define likely(x) __builtin_expect(!!(x), 1) - #define unlikely(x) __builtin_expect(!!(x), 0) -#else /* !__GNUC__ or GCC < 2.95 */ - #define likely(x) (x) - #define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_d; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static PyObject *__pyx_empty_bytes; -static PyObject *__pyx_empty_unicode; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; - - -static const char *__pyx_f[] = { - "khmer/_oxli/partitioning.pyx", - "khmer/_oxli/partitioning.pxd", - "stringsource", -}; - -/*--- Type declarations ---*/ -struct __pyx_obj_5khmer_5_oxli_12partitioning_Component; -struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner; -struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__; -struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components; -struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components; - -/* "khmer/_oxli/partitioning.pxd":8 - * from _oxli cimport CpHashtable, CpStreamingPartitioner, BoundedCounterType - * - * cdef class Component: # <<<<<<<<<<<<<< - * cdef ComponentPtr _this - * - */ -struct __pyx_obj_5khmer_5_oxli_12partitioning_Component { - PyObject_HEAD - struct __pyx_vtabstruct_5khmer_5_oxli_12partitioning_Component *__pyx_vtab; - khmer::ComponentPtr _this; -}; - - -/* "khmer/_oxli/partitioning.pxd":21 - * - * - * cdef class StreamingPartitioner: # <<<<<<<<<<<<<< - * cdef unique_ptr[CpStreamingPartitioner] _this - * cdef weak_ptr[ComponentPtrSet] _components - */ -struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner { - PyObject_HEAD - std::unique_ptr _this; - std::weak_ptr _components; - std::weak_ptr _tag_component_map; - khmer::Hashtable *_graph_ptr; - uint64_t n_consumed; -}; - - -/* "khmer/_oxli/partitioning.pyx":44 - * return deref(self._this).get_n_tags() - * - * def __iter__(self): # <<<<<<<<<<<<<< - * cdef HashIntoType tag - * for tag in deref(self._this).tags: - */ -struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ { - PyObject_HEAD - struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self; - khmer::HashIntoType __pyx_v_tag; - std::set ::iterator __pyx_t_0; - std::set *__pyx_t_1; -}; - - -/* "khmer/_oxli/partitioning.pyx":128 - * return Component.create(compptr) - * - * def components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[ComponentPtrSet] locked - * cdef ComponentPtr cmpptr - */ -struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components { - PyObject_HEAD - khmer::ComponentPtr __pyx_v_cmpptr; - std::shared_ptr __pyx_v_locked; - std::shared_ptr __pyx_v_lockedptr; - struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self; - std::set ::iterator __pyx_t_0; - khmer::ComponentPtrSet *__pyx_t_1; -}; - - -/* "khmer/_oxli/partitioning.pyx":138 - * raise MemoryError("Can't locked underlying Component set") - * - * def tag_components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[CpGuardedKmerCompMap] locked - * cdef pair[HashIntoType,ComponentPtr] cpair - */ -struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components { - PyObject_HEAD - std::pair __pyx_v_cpair; - std::shared_ptr __pyx_v_locked; - std::shared_ptr __pyx_v_lockedptr; - struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self; - std::map ::iterator __pyx_t_0; - std::map *__pyx_t_1; -}; - - - -/* "khmer/_oxli/partitioning.pyx":23 - * from .._khmer import Nodegraph - * - * cdef class Component: # <<<<<<<<<<<<<< - * - * def __cinit__(self, Component other=None): - */ - -struct __pyx_vtabstruct_5khmer_5_oxli_12partitioning_Component { - struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *(*create)(khmer::ComponentPtr); - std::vector (*_tag_counts)(khmer::ComponentPtr, khmer::Hashtable *); - float (*_mean_tag_count)(khmer::ComponentPtr, khmer::Hashtable *); -}; -static struct __pyx_vtabstruct_5khmer_5_oxli_12partitioning_Component *__pyx_vtabptr_5khmer_5_oxli_12partitioning_Component; - -/* --- Runtime support code (head) --- */ -/* Refnanny.proto */ -#ifndef CYTHON_REFNANNY - #define CYTHON_REFNANNY 0 -#endif -#if CYTHON_REFNANNY - typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); - void (*FinishContext)(void**); - } __Pyx_RefNannyAPIStruct; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); - #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; -#ifdef WITH_THREAD - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - if (acquire_gil) {\ - PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - PyGILState_Release(__pyx_gilstate_save);\ - } else {\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - } -#else - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) -#endif - #define __Pyx_RefNannyFinishContext()\ - __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) -#else - #define __Pyx_RefNannyDeclarations - #define __Pyx_RefNannySetupContext(name, acquire_gil) - #define __Pyx_RefNannyFinishContext() - #define __Pyx_INCREF(r) Py_INCREF(r) - #define __Pyx_DECREF(r) Py_DECREF(r) - #define __Pyx_GOTREF(r) - #define __Pyx_GIVEREF(r) - #define __Pyx_XINCREF(r) Py_XINCREF(r) - #define __Pyx_XDECREF(r) Py_XDECREF(r) - #define __Pyx_XGOTREF(r) - #define __Pyx_XGIVEREF(r) -#endif -#define __Pyx_XDECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_XDECREF(tmp);\ - } while (0) -#define __Pyx_DECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_DECREF(tmp);\ - } while (0) -#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) -#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) - -/* PyObjectGetAttrStr.proto */ -#if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { - PyTypeObject* tp = Py_TYPE(obj); - if (likely(tp->tp_getattro)) - return tp->tp_getattro(obj, attr_name); -#if PY_MAJOR_VERSION < 3 - if (likely(tp->tp_getattr)) - return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); -#endif - return PyObject_GetAttr(obj, attr_name); -} -#else -#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) -#endif - -/* GetBuiltinName.proto */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name); - -/* RaiseDoubleKeywords.proto */ -static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); - -/* ParseKeywords.proto */ -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ - const char* function_name); - -/* RaiseArgTupleInvalid.proto */ -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); - -/* ArgTypeTest.proto */ -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact); - -/* PyIntBinop.proto */ -#if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); -#else -#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ - PyObject_RichCompare(op1, op2, Py_EQ) - #endif - -/* PyObjectCall.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); -#else -#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) -#endif - -/* PyThreadStateGet.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; -#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); -#else -#define __Pyx_PyThreadState_declare -#define __Pyx_PyThreadState_assign -#endif - -/* PyErrFetchRestore.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) -#endif - -/* RaiseException.proto */ -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); - -/* RaiseTooManyValuesToUnpack.proto */ -static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); - -/* RaiseNeedMoreValuesToUnpack.proto */ -static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); - -/* IterFinish.proto */ -static CYTHON_INLINE int __Pyx_IterFinish(void); - -/* UnpackItemEndCheck.proto */ -static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); - -/* WriteUnraisableException.proto */ -static void __Pyx_WriteUnraisable(const char *name, int clineno, - int lineno, const char *filename, - int full_traceback, int nogil); - -/* GetModuleGlobalName.proto */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); - -/* ListCompAppend.proto */ -#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS -static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { - PyListObject* L = (PyListObject*) list; - Py_ssize_t len = Py_SIZE(list); - if (likely(L->allocated > len)) { - Py_INCREF(x); - PyList_SET_ITEM(list, len, x); - Py_SIZE(list) = len+1; - return 0; - } - return PyList_Append(list, x); -} -#else -#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) -#endif - -/* IncludeStringH.proto */ -#include - -/* SetVTable.proto */ -static int __Pyx_SetVtable(PyObject *dict, void *vtable); - -/* Import.proto */ -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); - -/* ImportFrom.proto */ -static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); - -/* GetNameInClass.proto */ -static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name); - -/* CodeObjectCache.proto */ -typedef struct { - PyCodeObject* code_object; - int code_line; -} __Pyx_CodeObjectCacheEntry; -struct __Pyx_CodeObjectCache { - int count; - int max_count; - __Pyx_CodeObjectCacheEntry* entries; -}; -static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); -static PyCodeObject *__pyx_find_code_object(int code_line); -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); - -/* AddTraceback.proto */ -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename); - -/* None.proto */ -#include - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value); - -/* CppExceptionConversion.proto */ -#ifndef __Pyx_CppExn2PyErr -#include -#include -#include -#include -static void __Pyx_CppExn2PyErr() { - try { - if (PyErr_Occurred()) - ; // let the latest Python exn pass through and ignore the current one - else - throw; - } catch (const std::bad_alloc& exn) { - PyErr_SetString(PyExc_MemoryError, exn.what()); - } catch (const std::bad_cast& exn) { - PyErr_SetString(PyExc_TypeError, exn.what()); - } catch (const std::bad_typeid& exn) { - PyErr_SetString(PyExc_TypeError, exn.what()); - } catch (const std::domain_error& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::invalid_argument& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::ios_base::failure& exn) { - PyErr_SetString(PyExc_IOError, exn.what()); - } catch (const std::out_of_range& exn) { - PyErr_SetString(PyExc_IndexError, exn.what()); - } catch (const std::overflow_error& exn) { - PyErr_SetString(PyExc_OverflowError, exn.what()); - } catch (const std::range_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::underflow_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::exception& exn) { - PyErr_SetString(PyExc_RuntimeError, exn.what()); - } - catch (...) - { - PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); - } -} -#endif - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType(khmer::BoundedCounterType value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); - -/* FetchCommonType.proto */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); - -/* SwapException.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); -#endif - -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/* PyFunctionFastCall.proto */ -#if CYTHON_FAST_PYCALL -#define __Pyx_PyFunction_FastCall(func, args, nargs)\ - __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) -#endif -#endif - -/* PyObjectCallMethO.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - -/* PyObjectCallMethod1.proto */ -static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); - -/* CoroutineBase.proto */ -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); -typedef struct { - PyObject_HEAD - __pyx_coroutine_body_t body; - PyObject *closure; - PyObject *exc_type; - PyObject *exc_value; - PyObject *exc_traceback; - PyObject *gi_weakreflist; - PyObject *classobj; - PyObject *yieldfrom; - PyObject *gi_name; - PyObject *gi_qualname; - PyObject *gi_modulename; - int resume_label; - char is_running; -} __pyx_CoroutineObject; -static __pyx_CoroutineObject *__Pyx__Coroutine_New( - PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *closure, - PyObject *name, PyObject *qualname, PyObject *module_name); -static int __Pyx_Coroutine_clear(PyObject *self); -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); -#else -#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) -#endif - -/* PatchModuleWithCoroutine.proto */ -static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code); - -/* PatchGeneratorABC.proto */ -static int __Pyx_patch_abc(void); - -/* Generator.proto */ -#define __Pyx_Generator_USED -static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) -#define __Pyx_Generator_New(body, closure, name, qualname, module_name)\ - __Pyx__Coroutine_New(__pyx_GeneratorType, body, closure, name, qualname, module_name) -static PyObject *__Pyx_Generator_Next(PyObject *self); -static int __pyx_Generator_init(void); - -/* CheckBinaryVersion.proto */ -static int __Pyx_check_binary_version(void); - -/* InitStrings.proto */ -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); - -static struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(khmer::ComponentPtr __pyx_v_ptr); /* proto*/ -static std::vector __pyx_f_5khmer_5_oxli_12partitioning_9Component__tag_counts(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph); /* proto*/ -static float __pyx_f_5khmer_5_oxli_12partitioning_9Component__mean_tag_count(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph); /* proto*/ - -/* Module declarations from 'libcpp' */ - -/* Module declarations from 'libcpp.memory' */ - -/* Module declarations from 'libcpp.vector' */ - -/* Module declarations from 'libc.stdint' */ - -/* Module declarations from 'libc.string' */ - -/* Module declarations from 'libcpp.string' */ - -/* Module declarations from 'libcpp.utility' */ - -/* Module declarations from 'libcpp.map' */ - -/* Module declarations from 'libcpp.set' */ - -/* Module declarations from 'libcpp.queue' */ - -/* Module declarations from 'khmer._oxli._oxli' */ - -/* Module declarations from 'cython' */ - -/* Module declarations from 'libc.limits' */ - -/* Module declarations from 'libc.stdio' */ - -/* Module declarations from 'khmer._oxli.partitioning' */ -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning_Component = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning_StreamingPartitioner = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components = 0; -static PyObject *__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(const std::vector &); /*proto*/ -static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ -#define __Pyx_MODULE_NAME "khmer._oxli.partitioning" -int __pyx_module_is_main_khmer___oxli__partitioning = 0; - -/* Implementation of 'khmer._oxli.partitioning' */ -static PyObject *__pyx_builtin_staticmethod; -static PyObject *__pyx_builtin_NotImplementedError; -static PyObject *__pyx_builtin_ValueError; -static PyObject *__pyx_builtin_MemoryError; -static PyObject *__pyx_builtin_IOError; -static PyObject *__pyx_builtin_range; -static const char __pyx_k_args[] = "args"; -static const char __pyx_k_iter[] = "__iter__"; -static const char __pyx_k_main[] = "__main__"; -static const char __pyx_k_send[] = "send"; -static const char __pyx_k_test[] = "__test__"; -static const char __pyx_k_close[] = "close"; -static const char __pyx_k_graph[] = "graph"; -static const char __pyx_k_khmer[] = "_khmer"; -static const char __pyx_k_other[] = "other"; -static const char __pyx_k_range[] = "range"; -static const char __pyx_k_throw[] = "throw"; -static const char __pyx_k_utf_8[] = "utf-8"; -static const char __pyx_k_encode[] = "encode"; -static const char __pyx_k_import[] = "__import__"; -static const char __pyx_k_IOError[] = "IOError"; -static const char __pyx_k_Nodegraph[] = "Nodegraph"; -static const char __pyx_k_component[] = "component"; -static const char __pyx_k_graph_ptr[] = "graph_ptr"; -static const char __pyx_k_Countgraph[] = "Countgraph"; -static const char __pyx_k_ValueError[] = "ValueError"; -static const char __pyx_k_components[] = "components"; -static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; -static const char __pyx_k_tag_counts[] = "tag_counts"; -static const char __pyx_k_MemoryError[] = "MemoryError"; -static const char __pyx_k_component_id[] = "component_id"; -static const char __pyx_k_staticmethod[] = "staticmethod"; -static const char __pyx_k_tag_components[] = "tag_components"; -static const char __pyx_k_Can_t_open_file[] = "Can't open file."; -static const char __pyx_k_Component___iter[] = "Component.__iter__"; -static const char __pyx_k_NotImplementedError[] = "NotImplementedError"; -static const char __pyx_k_Operator_not_available[] = "Operator not available."; -static const char __pyx_k_khmer__oxli_partitioning[] = "khmer._oxli.partitioning"; -static const char __pyx_k_StreamingPartitioner_components[] = "StreamingPartitioner.components"; -static const char __pyx_k_Users_camille_w_khmer_khmer__ox[] = "/Users/camille/w/khmer/khmer/_oxli/partitioning.pyx"; -static const char __pyx_k_Can_t_locked_underlying_Componen[] = "Can't locked underlying Component set"; -static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; -static const char __pyx_k_StreamingPartitioner_tag_compone[] = "StreamingPartitioner.tag_components"; -static PyObject *__pyx_kp_s_Can_t_locked_underlying_Componen; -static PyObject *__pyx_kp_s_Can_t_open_file; -static PyObject *__pyx_n_s_Component___iter; -static PyObject *__pyx_n_s_Countgraph; -static PyObject *__pyx_n_s_IOError; -static PyObject *__pyx_n_s_MemoryError; -static PyObject *__pyx_kp_s_Must_take_an_object_with_Hashtab; -static PyObject *__pyx_n_s_Nodegraph; -static PyObject *__pyx_n_s_NotImplementedError; -static PyObject *__pyx_kp_s_Operator_not_available; -static PyObject *__pyx_n_s_StreamingPartitioner_components; -static PyObject *__pyx_n_s_StreamingPartitioner_tag_compone; -static PyObject *__pyx_kp_s_Users_camille_w_khmer_khmer__ox; -static PyObject *__pyx_n_s_ValueError; -static PyObject *__pyx_n_s_args; -static PyObject *__pyx_n_s_close; -static PyObject *__pyx_n_s_component; -static PyObject *__pyx_n_s_component_id; -static PyObject *__pyx_n_s_components; -static PyObject *__pyx_n_s_encode; -static PyObject *__pyx_n_s_graph; -static PyObject *__pyx_n_s_graph_ptr; -static PyObject *__pyx_n_s_import; -static PyObject *__pyx_n_s_iter; -static PyObject *__pyx_n_s_khmer; -static PyObject *__pyx_n_s_khmer__oxli_partitioning; -static PyObject *__pyx_n_s_main; -static PyObject *__pyx_n_s_other; -static PyObject *__pyx_n_s_pyx_vtable; -static PyObject *__pyx_n_s_range; -static PyObject *__pyx_n_s_send; -static PyObject *__pyx_n_s_staticmethod; -static PyObject *__pyx_n_s_tag_components; -static PyObject *__pyx_n_s_tag_counts; -static PyObject *__pyx_n_s_test; -static PyObject *__pyx_n_s_throw; -static PyObject *__pyx_kp_s_utf_8; -static int __pyx_pf_5khmer_5_oxli_12partitioning_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_other); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ -static Py_ssize_t __pyx_pf_5khmer_5_oxli_12partitioning_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ -static Py_hash_t __pyx_pf_5khmer_5_oxli_12partitioning_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_11tag_counts(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_component, PyObject *__pyx_v_graph); /* proto */ -static int __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_4consume_fasta(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6get_tag_component(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_8get_nearest_component(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_13tag_components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_16write_components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self); /* proto */ -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning_Component(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_int_2; -static PyObject *__pyx_tuple_; -static PyObject *__pyx_tuple__2; -static PyObject *__pyx_tuple__3; -static PyObject *__pyx_tuple__4; -static PyObject *__pyx_tuple__5; -static PyObject *__pyx_tuple__6; -static PyObject *__pyx_tuple__7; -static PyObject *__pyx_tuple__8; -static PyObject *__pyx_tuple__9; -static PyObject *__pyx_tuple__10; -static PyObject *__pyx_tuple__11; -static PyObject *__pyx_codeobj__12; - -/* "khmer/_oxli/partitioning.pyx":25 - * cdef class Component: - * - * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< - * if other is not None: - * self._this.reset(other._this.get()) - */ - -/* Python wrapper */ -static int __pyx_pw_5khmer_5_oxli_12partitioning_9Component_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5khmer_5_oxli_12partitioning_9Component_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_other = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_other,0}; - PyObject* values[1] = {0}; - values[0] = (PyObject *)((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)Py_None); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_other); - if (value) { values[0] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 25, __pyx_L3_error) - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_other = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)values[0]); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 25, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.partitioning.Component.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5khmer_5_oxli_12partitioning_Component, 1, "other", 0))) __PYX_ERR(0, 25, __pyx_L1_error) - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component___cinit__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self), __pyx_v_other); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5khmer_5_oxli_12partitioning_9Component___cinit__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_other) { - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "khmer/_oxli/partitioning.pyx":26 - * - * def __cinit__(self, Component other=None): - * if other is not None: # <<<<<<<<<<<<<< - * self._this.reset(other._this.get()) - * - */ - __pyx_t_1 = (((PyObject *)__pyx_v_other) != Py_None); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "khmer/_oxli/partitioning.pyx":27 - * def __cinit__(self, Component other=None): - * if other is not None: - * self._this.reset(other._this.get()) # <<<<<<<<<<<<<< - * - * property component_id: - */ - __pyx_v_self->_this.reset(__pyx_v_other->_this.get()); - - /* "khmer/_oxli/partitioning.pyx":26 - * - * def __cinit__(self, Component other=None): - * if other is not None: # <<<<<<<<<<<<<< - * self._this.reset(other._this.get()) - * - */ - } - - /* "khmer/_oxli/partitioning.pyx":25 - * cdef class Component: - * - * def __cinit__(self, Component other=None): # <<<<<<<<<<<<<< - * if other is not None: - * self._this.reset(other._this.get()) - */ - - /* function exit code */ - __pyx_r = 0; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":30 - * - * property component_id: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).component_id - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12component_id_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12component_id_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_12component_id___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_12component_id___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli/partitioning.pyx":31 - * property component_id: - * def __get__(self): - * return deref(self._this).component_id # <<<<<<<<<<<<<< - * - * property _n_created: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 31, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":30 - * - * property component_id: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).component_id - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.partitioning.Component.component_id.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":34 - * - * property _n_created: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_created() - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_10_n_created_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_10_n_created_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_10_n_created___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_10_n_created___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli/partitioning.pyx":35 - * property _n_created: - * def __get__(self): - * return deref(self._this).get_n_created() # <<<<<<<<<<<<<< - * - * property _n_destroyed: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_created()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 35, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":34 - * - * property _n_created: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_created() - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.partitioning.Component._n_created.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":38 - * - * property _n_destroyed: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_destroyed() - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli/partitioning.pyx":39 - * property _n_destroyed: - * def __get__(self): - * return deref(self._this).get_n_destroyed() # <<<<<<<<<<<<<< - * - * def __len__(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_destroyed()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 39, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":38 - * - * property _n_destroyed: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_destroyed() - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.partitioning.Component._n_destroyed.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":41 - * return deref(self._this).get_n_destroyed() - * - * def __len__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_tags() - * - */ - -/* Python wrapper */ -static Py_ssize_t __pyx_pw_5khmer_5_oxli_12partitioning_9Component_3__len__(PyObject *__pyx_v_self); /*proto*/ -static Py_ssize_t __pyx_pw_5khmer_5_oxli_12partitioning_9Component_3__len__(PyObject *__pyx_v_self) { - Py_ssize_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_2__len__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static Py_ssize_t __pyx_pf_5khmer_5_oxli_12partitioning_9Component_2__len__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { - Py_ssize_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__len__", 0); - - /* "khmer/_oxli/partitioning.pyx":42 - * - * def __len__(self): - * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< - * - * def __iter__(self): - */ - __pyx_r = (*__pyx_v_self->_this).get_n_tags(); - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":41 - * return deref(self._this).get_n_destroyed() - * - * def __len__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_tags() - * - */ - - /* function exit code */ - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ - -/* "khmer/_oxli/partitioning.pyx":44 - * return deref(self._this).get_n_tags() - * - * def __iter__(self): # <<<<<<<<<<<<<< - * cdef HashIntoType tag - * for tag in deref(self._this).tags: - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_5__iter__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_5__iter__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_4__iter__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_4__iter__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *__pyx_cur_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__iter__", 0); - __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(0, 44, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_12partitioning_9Component_6generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Component___iter, __pyx_n_s_khmer__oxli_partitioning); if (unlikely(!gen)) __PYX_ERR(0, 44, __pyx_L1_error) - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("khmer._oxli.partitioning.Component.__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_9Component_6generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)__pyx_generator->closure); - PyObject *__pyx_r = NULL; - std::set ::iterator __pyx_t_1; - std::set *__pyx_t_2; - khmer::HashIntoType __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L6_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 44, __pyx_L1_error) - - /* "khmer/_oxli/partitioning.pyx":46 - * def __iter__(self): - * cdef HashIntoType tag - * for tag in deref(self._this).tags: # <<<<<<<<<<<<<< - * yield tag - * - */ - __pyx_t_2 = &(*__pyx_cur_scope->__pyx_v_self->_this).tags; - __pyx_t_1 = __pyx_t_2->begin(); - for (;;) { - if (!(__pyx_t_1 != __pyx_t_2->end())) break; - __pyx_t_3 = *__pyx_t_1; - ++__pyx_t_1; - __pyx_cur_scope->__pyx_v_tag = __pyx_t_3; - - /* "khmer/_oxli/partitioning.pyx":47 - * cdef HashIntoType tag - * for tag in deref(self._this).tags: - * yield tag # <<<<<<<<<<<<<< - * - * def __hash__(self): - */ - __pyx_t_4 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_cur_scope->__pyx_v_tag); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 47, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; - __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_2; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L6_resume_from_yield:; - __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; - __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 47, __pyx_L1_error) - - /* "khmer/_oxli/partitioning.pyx":46 - * def __iter__(self): - * cdef HashIntoType tag - * for tag in deref(self._this).tags: # <<<<<<<<<<<<<< - * yield tag - * - */ - } - if (1); else __pyx_cur_scope = __pyx_cur_scope; - - /* "khmer/_oxli/partitioning.pyx":44 - * return deref(self._this).get_n_tags() - * - * def __iter__(self): # <<<<<<<<<<<<<< - * cdef HashIntoType tag - * for tag in deref(self._this).tags: - */ - - /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_XDECREF(__pyx_r); __pyx_r = 0; - __pyx_generator->resume_label = -1; - __Pyx_Coroutine_clear((PyObject*)__pyx_generator); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":49 - * yield tag - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return self._this.get() - * - */ - -/* Python wrapper */ -static Py_hash_t __pyx_pw_5khmer_5_oxli_12partitioning_9Component_8__hash__(PyObject *__pyx_v_self); /*proto*/ -static Py_hash_t __pyx_pw_5khmer_5_oxli_12partitioning_9Component_8__hash__(PyObject *__pyx_v_self) { - Py_hash_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_7__hash__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static Py_hash_t __pyx_pf_5khmer_5_oxli_12partitioning_9Component_7__hash__(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_self) { - Py_hash_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__", 0); - - /* "khmer/_oxli/partitioning.pyx":50 - * - * def __hash__(self): - * return self._this.get() # <<<<<<<<<<<<<< - * - * def __richcmp__(x, y, op): - */ - __pyx_r = ((uintptr_t)__pyx_v_self->_this.get()); - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":49 - * yield tag - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return self._this.get() - * - */ - - /* function exit code */ - __pyx_L0:; - if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":52 - * return self._this.get() - * - * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< - * if op == 2: - * return x.component_id == y.component_id - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_10__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_arg_op); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_10__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_arg_op) { - PyObject *__pyx_v_op = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); - __pyx_v_op = __Pyx_PyInt_From_int(__pyx_arg_op); if (unlikely(!__pyx_v_op)) __PYX_ERR(0, 52, __pyx_L3_error) - __Pyx_GOTREF(__pyx_v_op); - goto __pyx_L4_argument_unpacking_done; - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.partitioning.Component.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_9__richcmp__(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), ((PyObject *)__pyx_v_op)); - - /* function exit code */ - __Pyx_XDECREF(__pyx_v_op); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_9__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_op) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannySetupContext("__richcmp__", 0); - - /* "khmer/_oxli/partitioning.pyx":53 - * - * def __richcmp__(x, y, op): - * if op == 2: # <<<<<<<<<<<<<< - * return x.component_id == y.component_id - * else: - */ - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_v_op, __pyx_int_2, 2, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 53, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 53, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (__pyx_t_2) { - - /* "khmer/_oxli/partitioning.pyx":54 - * def __richcmp__(x, y, op): - * if op == 2: - * return x.component_id == y.component_id # <<<<<<<<<<<<<< - * else: - * raise NotImplementedError('Operator not available.') - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_x, __pyx_n_s_component_id); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 54, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_y, __pyx_n_s_component_id); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 54, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_RichCompare(__pyx_t_1, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 54, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":53 - * - * def __richcmp__(x, y, op): - * if op == 2: # <<<<<<<<<<<<<< - * return x.component_id == y.component_id - * else: - */ - } - - /* "khmer/_oxli/partitioning.pyx":56 - * return x.component_id == y.component_id - * else: - * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< - * - * @staticmethod - */ - /*else*/ { - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_NotImplementedError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 56, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(0, 56, __pyx_L1_error) - } - - /* "khmer/_oxli/partitioning.pyx":52 - * return self._this.get() - * - * def __richcmp__(x, y, op): # <<<<<<<<<<<<<< - * if op == 2: - * return x.component_id == y.component_id - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("khmer._oxli.partitioning.Component.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":59 - * - * @staticmethod - * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< - * cdef Component comp = Component() - * comp._this = ptr - */ - -static struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(khmer::ComponentPtr __pyx_v_ptr) { - struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_comp = 0; - struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("create", 0); - - /* "khmer/_oxli/partitioning.pyx":60 - * @staticmethod - * cdef Component create(ComponentPtr ptr): - * cdef Component comp = Component() # <<<<<<<<<<<<<< - * comp._this = ptr - * return comp - */ - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5khmer_5_oxli_12partitioning_Component), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 60, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_comp = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)__pyx_t_1); - __pyx_t_1 = 0; - - /* "khmer/_oxli/partitioning.pyx":61 - * cdef Component create(ComponentPtr ptr): - * cdef Component comp = Component() - * comp._this = ptr # <<<<<<<<<<<<<< - * return comp - * - */ - __pyx_v_comp->_this = __pyx_v_ptr; - - /* "khmer/_oxli/partitioning.pyx":62 - * cdef Component comp = Component() - * comp._this = ptr - * return comp # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __Pyx_XDECREF(((PyObject *)__pyx_r)); - __Pyx_INCREF(((PyObject *)__pyx_v_comp)); - __pyx_r = __pyx_v_comp; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":59 - * - * @staticmethod - * cdef Component create(ComponentPtr ptr): # <<<<<<<<<<<<<< - * cdef Component comp = Component() - * comp._this = ptr - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.partitioning.Component.create", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XDECREF((PyObject *)__pyx_v_comp); - __Pyx_XGIVEREF((PyObject *)__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":65 - * - * @staticmethod - * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph): # <<<<<<<<<<<<<< - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef vector[BoundedCounterType] counts - */ - -static std::vector __pyx_f_5khmer_5_oxli_12partitioning_9Component__tag_counts(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph) { - uint64_t __pyx_v_n_tags; - std::vector __pyx_v_counts; - int __pyx_v_idx; - uint64_t __pyx_v_tag; - std::vector __pyx_r; - __Pyx_RefNannyDeclarations - std::vector __pyx_t_1; - std::set ::iterator __pyx_t_2; - std::set *__pyx_t_3; - khmer::HashIntoType __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - PyObject *(*__pyx_t_9)(PyObject *); - int __pyx_t_10; - uint64_t __pyx_t_11; - __Pyx_RefNannySetupContext("_tag_counts", 0); - - /* "khmer/_oxli/partitioning.pyx":66 - * @staticmethod - * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph): - * cdef uint64_t n_tags = deref(comp).get_n_tags() # <<<<<<<<<<<<<< - * cdef vector[BoundedCounterType] counts - * counts = vector[BoundedCounterType](n_tags) - */ - __pyx_v_n_tags = (*__pyx_v_comp).get_n_tags(); - - /* "khmer/_oxli/partitioning.pyx":68 - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef vector[BoundedCounterType] counts - * counts = vector[BoundedCounterType](n_tags) # <<<<<<<<<<<<<< - * cdef int idx - * cdef uint64_t tag - */ - try { - __pyx_t_1 = std::vector (__pyx_v_n_tags); - } catch(...) { - __Pyx_CppExn2PyErr(); - __PYX_ERR(0, 68, __pyx_L1_error) - } - __pyx_v_counts = __pyx_t_1; - - /* "khmer/_oxli/partitioning.pyx":71 - * cdef int idx - * cdef uint64_t tag - * for idx, tag in deref(comp).tags: # <<<<<<<<<<<<<< - * counts[idx] = deref(graph).get_count(tag) - * return counts - */ - __pyx_t_3 = &(*__pyx_v_comp).tags; - __pyx_t_2 = __pyx_t_3->begin(); - for (;;) { - if (!(__pyx_t_2 != __pyx_t_3->end())) break; - __pyx_t_4 = *__pyx_t_2; - ++__pyx_t_2; - __pyx_t_5 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 71, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { - PyObject* sequence = __pyx_t_5; - #if !CYTHON_COMPILING_IN_PYPY - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - __PYX_ERR(0, 71, __pyx_L1_error) - } - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - if (likely(PyTuple_CheckExact(sequence))) { - __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); - __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); - } else { - __pyx_t_6 = PyList_GET_ITEM(sequence, 0); - __pyx_t_7 = PyList_GET_ITEM(sequence, 1); - } - __Pyx_INCREF(__pyx_t_6); - __Pyx_INCREF(__pyx_t_7); - #else - __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 71, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 71, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - #endif - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - } else { - Py_ssize_t index = -1; - __pyx_t_8 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 71, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_9 = Py_TYPE(__pyx_t_8)->tp_iternext; - index = 0; __pyx_t_6 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_6)) goto __pyx_L5_unpacking_failed; - __Pyx_GOTREF(__pyx_t_6); - index = 1; __pyx_t_7 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_7)) goto __pyx_L5_unpacking_failed; - __Pyx_GOTREF(__pyx_t_7); - if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < 0) __PYX_ERR(0, 71, __pyx_L1_error) - __pyx_t_9 = NULL; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - goto __pyx_L6_unpacking_done; - __pyx_L5_unpacking_failed:; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_9 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); - __PYX_ERR(0, 71, __pyx_L1_error) - __pyx_L6_unpacking_done:; - } - __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_6); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_11 = __Pyx_PyInt_As_uint64_t(__pyx_t_7); if (unlikely((__pyx_t_11 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 71, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_v_idx = __pyx_t_10; - __pyx_v_tag = __pyx_t_11; - - /* "khmer/_oxli/partitioning.pyx":72 - * cdef uint64_t tag - * for idx, tag in deref(comp).tags: - * counts[idx] = deref(graph).get_count(tag) # <<<<<<<<<<<<<< - * return counts - * - */ - (__pyx_v_counts[__pyx_v_idx]) = (*__pyx_v_graph).get_count(__pyx_v_tag); - - /* "khmer/_oxli/partitioning.pyx":71 - * cdef int idx - * cdef uint64_t tag - * for idx, tag in deref(comp).tags: # <<<<<<<<<<<<<< - * counts[idx] = deref(graph).get_count(tag) - * return counts - */ - } - - /* "khmer/_oxli/partitioning.pyx":73 - * for idx, tag in deref(comp).tags: - * counts[idx] = deref(graph).get_count(tag) - * return counts # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __pyx_r = __pyx_v_counts; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":65 - * - * @staticmethod - * cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph): # <<<<<<<<<<<<<< - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef vector[BoundedCounterType] counts - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_WriteUnraisable("khmer._oxli.partitioning.Component._tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":76 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CPyHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12tag_counts(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyMethodDef __pyx_mdef_5khmer_5_oxli_12partitioning_9Component_12tag_counts = {"tag_counts", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12tag_counts, METH_VARARGS|METH_KEYWORDS, 0}; -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12tag_counts(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_component = 0; - PyObject *__pyx_v_graph = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("tag_counts (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_component,&__pyx_n_s_graph,0}; - PyObject* values[2] = {0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_component)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("tag_counts", 1, 2, 2, 1); __PYX_ERR(0, 76, __pyx_L3_error) - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "tag_counts") < 0)) __PYX_ERR(0, 76, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - } - __pyx_v_component = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)values[0]); - __pyx_v_graph = values[1]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("tag_counts", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 76, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.partitioning.Component.tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_component), __pyx_ptype_5khmer_5_oxli_12partitioning_Component, 1, "component", 0))) __PYX_ERR(0, 76, __pyx_L1_error) - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_9Component_11tag_counts(__pyx_v_component, __pyx_v_graph); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_9Component_11tag_counts(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *__pyx_v_component, PyObject *__pyx_v_graph) { - khmer::khmer_KHashtable_Object *__pyx_v_graph_ptr; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("tag_counts", 0); - - /* "khmer/_oxli/partitioning.pyx":77 - * @staticmethod - * def tag_counts(Component component, graph): - * cdef CPyHashtable_Object* graph_ptr = graph # <<<<<<<<<<<<<< - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - * - */ - __pyx_v_graph_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - - /* "khmer/_oxli/partitioning.pyx":78 - * def tag_counts(Component component, graph): - * cdef CPyHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(__pyx_f_5khmer_5_oxli_12partitioning_9Component__tag_counts(__pyx_v_component->_this, (*__pyx_v_graph_ptr).hashtable)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 78, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":76 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CPyHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.partitioning.Component.tag_counts", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":81 - * - * @staticmethod - * cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): # <<<<<<<<<<<<<< - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef float acc = 0 - */ - -static float __pyx_f_5khmer_5_oxli_12partitioning_9Component__mean_tag_count(khmer::ComponentPtr __pyx_v_comp, khmer::Hashtable *__pyx_v_graph) { - uint64_t __pyx_v_n_tags; - float __pyx_v_acc; - uint64_t __pyx_v_tag; - float __pyx_r; - __Pyx_RefNannyDeclarations - std::set ::iterator __pyx_t_1; - std::set *__pyx_t_2; - khmer::HashIntoType __pyx_t_3; - __Pyx_RefNannySetupContext("_mean_tag_count", 0); - - /* "khmer/_oxli/partitioning.pyx":82 - * @staticmethod - * cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): - * cdef uint64_t n_tags = deref(comp).get_n_tags() # <<<<<<<<<<<<<< - * cdef float acc = 0 - * cdef uint64_t tag - */ - __pyx_v_n_tags = (*__pyx_v_comp).get_n_tags(); - - /* "khmer/_oxli/partitioning.pyx":83 - * cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef float acc = 0 # <<<<<<<<<<<<<< - * cdef uint64_t tag - * for tag in deref(comp).tags: - */ - __pyx_v_acc = 0.0; - - /* "khmer/_oxli/partitioning.pyx":85 - * cdef float acc = 0 - * cdef uint64_t tag - * for tag in deref(comp).tags: # <<<<<<<<<<<<<< - * acc += deref(graph).get_count(tag) - * return acc / n_tags - */ - __pyx_t_2 = &(*__pyx_v_comp).tags; - __pyx_t_1 = __pyx_t_2->begin(); - for (;;) { - if (!(__pyx_t_1 != __pyx_t_2->end())) break; - __pyx_t_3 = *__pyx_t_1; - ++__pyx_t_1; - __pyx_v_tag = __pyx_t_3; - - /* "khmer/_oxli/partitioning.pyx":86 - * cdef uint64_t tag - * for tag in deref(comp).tags: - * acc += deref(graph).get_count(tag) # <<<<<<<<<<<<<< - * return acc / n_tags - * - */ - __pyx_v_acc = (__pyx_v_acc + ((float)(*__pyx_v_graph).get_count(__pyx_v_tag))); - - /* "khmer/_oxli/partitioning.pyx":85 - * cdef float acc = 0 - * cdef uint64_t tag - * for tag in deref(comp).tags: # <<<<<<<<<<<<<< - * acc += deref(graph).get_count(tag) - * return acc / n_tags - */ - } - - /* "khmer/_oxli/partitioning.pyx":87 - * for tag in deref(comp).tags: - * acc += deref(graph).get_count(tag) - * return acc / n_tags # <<<<<<<<<<<<<< - * - * - */ - if (unlikely(((float)__pyx_v_n_tags) == 0)) { - PyErr_SetString(PyExc_ZeroDivisionError, "float division"); - __PYX_ERR(0, 87, __pyx_L1_error) - } - __pyx_r = (__pyx_v_acc / ((float)__pyx_v_n_tags)); - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":81 - * - * @staticmethod - * cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph): # <<<<<<<<<<<<<< - * cdef uint64_t n_tags = deref(comp).get_n_tags() - * cdef float acc = 0 - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_WriteUnraisable("khmer._oxli.partitioning.Component._mean_tag_count", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); - __pyx_r = 0; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":92 - * cdef class StreamingPartitioner: - * - * def __cinit__(self, graph): # <<<<<<<<<<<<<< - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') - */ - -/* Python wrapper */ -static int __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_graph = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_graph,0}; - PyObject* values[1] = {0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 92, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - } - __pyx_v_graph = values[0]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 92, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner___cinit__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), __pyx_v_graph); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner___cinit__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_graph) { - khmer::khmer_KHashtable_Object *__pyx_v_ptr; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - int __pyx_t_3; - int __pyx_t_4; - khmer::Hashtable *__pyx_t_5; - khmer::StreamingPartitioner *__pyx_t_6; - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "khmer/_oxli/partitioning.pyx":93 - * - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< - * raise ValueError('Must take an object with Hashtable *') - * - */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 93, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 93, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_4 = (__pyx_t_3 != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_1 = __pyx_t_4; - goto __pyx_L4_bool_binop_done; - } - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 93, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 93, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = (__pyx_t_4 != 0); - __pyx_t_1 = __pyx_t_3; - __pyx_L4_bool_binop_done:; - __pyx_t_3 = ((!__pyx_t_1) != 0); - if (__pyx_t_3) { - - /* "khmer/_oxli/partitioning.pyx":94 - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< - * - * cdef CPyHashtable_Object* ptr = graph - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 94, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(0, 94, __pyx_L1_error) - - /* "khmer/_oxli/partitioning.pyx":93 - * - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< - * raise ValueError('Must take an object with Hashtable *') - * - */ - } - - /* "khmer/_oxli/partitioning.pyx":96 - * raise ValueError('Must take an object with Hashtable *') - * - * cdef CPyHashtable_Object* ptr = graph # <<<<<<<<<<<<<< - * self._graph_ptr = deref(ptr).hashtable - * - */ - __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - - /* "khmer/_oxli/partitioning.pyx":97 - * - * cdef CPyHashtable_Object* ptr = graph - * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< - * - * self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) - */ - __pyx_t_5 = (*__pyx_v_ptr).hashtable; - __pyx_v_self->_graph_ptr = __pyx_t_5; - - /* "khmer/_oxli/partitioning.pyx":99 - * self._graph_ptr = deref(ptr).hashtable - * - * self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) # <<<<<<<<<<<<<< - * - * self._tag_component_map = deref(self._this).get_tag_component_map() - */ - try { - __pyx_t_6 = new khmer::StreamingPartitioner(__pyx_v_self->_graph_ptr); - } catch(...) { - try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(0, 99, __pyx_L1_error) - } - __pyx_v_self->_this.reset(__pyx_t_6); - - /* "khmer/_oxli/partitioning.pyx":101 - * self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) - * - * self._tag_component_map = deref(self._this).get_tag_component_map() # <<<<<<<<<<<<<< - * self._components = deref(self._this).get_component_set() - * self.n_consumed = 0 - */ - __pyx_v_self->_tag_component_map = (*__pyx_v_self->_this).get_tag_component_map(); - - /* "khmer/_oxli/partitioning.pyx":102 - * - * self._tag_component_map = deref(self._this).get_tag_component_map() - * self._components = deref(self._this).get_component_set() # <<<<<<<<<<<<<< - * self.n_consumed = 0 - * - */ - __pyx_v_self->_components = (*__pyx_v_self->_this).get_component_set(); - - /* "khmer/_oxli/partitioning.pyx":103 - * self._tag_component_map = deref(self._this).get_tag_component_map() - * self._components = deref(self._this).get_component_set() - * self.n_consumed = 0 # <<<<<<<<<<<<<< - * - * def consume_sequence(self, sequence): - */ - __pyx_v_self->n_consumed = 0; - - /* "khmer/_oxli/partitioning.pyx":92 - * cdef class StreamingPartitioner: - * - * def __cinit__(self, graph): # <<<<<<<<<<<<<< - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":105 - * self.n_consumed = 0 - * - * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< - * deref(self._this).consume_sequence(sequence.encode('utf-8')) - * self.n_consumed += 1 - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_3consume_sequence(PyObject *__pyx_v_self, PyObject *__pyx_v_sequence); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_3consume_sequence(PyObject *__pyx_v_self, PyObject *__pyx_v_sequence) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("consume_sequence (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_2consume_sequence(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_sequence)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_2consume_sequence(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_sequence) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - std::string __pyx_t_3; - __Pyx_RefNannySetupContext("consume_sequence", 0); - - /* "khmer/_oxli/partitioning.pyx":106 - * - * def consume_sequence(self, sequence): - * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< - * self.n_consumed += 1 - * - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_sequence, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 106, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 106, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 106, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - try { - (*__pyx_v_self->_this).consume_sequence(__pyx_t_3); - } catch(...) { - try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(0, 106, __pyx_L1_error) - } - - /* "khmer/_oxli/partitioning.pyx":107 - * def consume_sequence(self, sequence): - * deref(self._this).consume_sequence(sequence.encode('utf-8')) - * self.n_consumed += 1 # <<<<<<<<<<<<<< - * - * def consume_fasta(self, filename): - */ - __pyx_v_self->n_consumed = (__pyx_v_self->n_consumed + 1); - - /* "khmer/_oxli/partitioning.pyx":105 - * self.n_consumed = 0 - * - * def consume_sequence(self, sequence): # <<<<<<<<<<<<<< - * deref(self._this).consume_sequence(sequence.encode('utf-8')) - * self.n_consumed += 1 - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.consume_sequence", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":109 - * self.n_consumed += 1 - * - * def consume_fasta(self, filename): # <<<<<<<<<<<<<< - * return deref(self._this).consume_fasta(filename.encode('utf-8')) - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_5consume_fasta(PyObject *__pyx_v_self, PyObject *__pyx_v_filename); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_5consume_fasta(PyObject *__pyx_v_self, PyObject *__pyx_v_filename) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("consume_fasta (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_4consume_fasta(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_filename)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_4consume_fasta(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - std::string __pyx_t_3; - uint64_t __pyx_t_4; - __Pyx_RefNannySetupContext("consume_fasta", 0); - - /* "khmer/_oxli/partitioning.pyx":110 - * - * def consume_fasta(self, filename): - * return deref(self._this).consume_fasta(filename.encode('utf-8')) # <<<<<<<<<<<<<< - * - * def get_tag_component(self, kmer): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_filename, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 110, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 110, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 110, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - try { - __pyx_t_4 = (*__pyx_v_self->_this).consume_fasta(__pyx_t_3); - } catch(...) { - try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_MemoryError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_MemoryError); } - __PYX_ERR(0, 110, __pyx_L1_error) - } - __pyx_t_2 = __Pyx_PyInt_From_uint64_t(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 110, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":109 - * self.n_consumed += 1 - * - * def consume_fasta(self, filename): # <<<<<<<<<<<<<< - * return deref(self._this).consume_fasta(filename.encode('utf-8')) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.consume_fasta", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":112 - * return deref(self._this).consume_fasta(filename.encode('utf-8')) - * - * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_7get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_7get_tag_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("get_tag_component (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6get_tag_component(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6get_tag_component(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { - khmer::ComponentPtr __pyx_v_compptr; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - std::string __pyx_t_3; - int __pyx_t_4; - __Pyx_RefNannySetupContext("get_tag_component", 0); - - /* "khmer/_oxli/partitioning.pyx":114 - * def get_tag_component(self, kmer): - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if compptr == NULL: - * return None - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 114, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 114, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_compptr = (*__pyx_v_self->_this).get_tag_component(__pyx_t_3); - - /* "khmer/_oxli/partitioning.pyx":115 - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if compptr == NULL: # <<<<<<<<<<<<<< - * return None - * else: - */ - __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); - if (__pyx_t_4) { - - /* "khmer/_oxli/partitioning.pyx":116 - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if compptr == NULL: - * return None # <<<<<<<<<<<<<< - * else: - * return Component.create(compptr) - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(Py_None); - __pyx_r = Py_None; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":115 - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - * if compptr == NULL: # <<<<<<<<<<<<<< - * return None - * else: - */ - } - - /* "khmer/_oxli/partitioning.pyx":118 - * return None - * else: - * return Component.create(compptr) # <<<<<<<<<<<<<< - * - * def get_nearest_component(self, kmer): - */ - /*else*/ { - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 118, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - } - - /* "khmer/_oxli/partitioning.pyx":112 - * return deref(self._this).consume_fasta(filename.encode('utf-8')) - * - * def get_tag_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.get_tag_component", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":120 - * return Component.create(compptr) - * - * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_9get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_9get_nearest_component(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("get_nearest_component (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_8get_nearest_component(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_kmer)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_8get_nearest_component(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_kmer) { - khmer::ComponentPtr __pyx_v_compptr; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - std::string __pyx_t_3; - int __pyx_t_4; - __Pyx_RefNannySetupContext("get_nearest_component", 0); - - /* "khmer/_oxli/partitioning.pyx":122 - * def get_nearest_component(self, kmer): - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if compptr == NULL: - * return None - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kmer, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 122, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 122, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_compptr = (*__pyx_v_self->_this).get_nearest_component(__pyx_t_3); - - /* "khmer/_oxli/partitioning.pyx":123 - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if compptr == NULL: # <<<<<<<<<<<<<< - * return None - * else: - */ - __pyx_t_4 = ((__pyx_v_compptr == NULL) != 0); - if (__pyx_t_4) { - - /* "khmer/_oxli/partitioning.pyx":124 - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if compptr == NULL: - * return None # <<<<<<<<<<<<<< - * else: - * return Component.create(compptr) - */ - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(Py_None); - __pyx_r = Py_None; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":123 - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - * if compptr == NULL: # <<<<<<<<<<<<<< - * return None - * else: - */ - } - - /* "khmer/_oxli/partitioning.pyx":126 - * return None - * else: - * return Component.create(compptr) # <<<<<<<<<<<<<< - * - * def components(self): - */ - /*else*/ { - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = ((PyObject *)__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(__pyx_v_compptr)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 126, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - goto __pyx_L0; - } - - /* "khmer/_oxli/partitioning.pyx":120 - * return Component.create(compptr) - * - * def get_nearest_component(self, kmer): # <<<<<<<<<<<<<< - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.get_nearest_component", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ - -/* "khmer/_oxli/partitioning.pyx":128 - * return Component.create(compptr) - * - * def components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[ComponentPtrSet] locked - * cdef ComponentPtr cmpptr - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_11components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_11components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("components (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10components(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *__pyx_cur_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("components", 0); - __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(0, 128, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12generator1, (PyObject *) __pyx_cur_scope, __pyx_n_s_components, __pyx_n_s_StreamingPartitioner_components, __pyx_n_s_khmer__oxli_partitioning); if (unlikely(!gen)) __PYX_ERR(0, 128, __pyx_L1_error) - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12generator1(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)__pyx_generator->closure); - PyObject *__pyx_r = NULL; - bool __pyx_t_1; - std::set ::iterator __pyx_t_2; - khmer::ComponentPtrSet *__pyx_t_3; - khmer::ComponentPtr __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L7_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 128, __pyx_L1_error) - - /* "khmer/_oxli/partitioning.pyx":131 - * cdef shared_ptr[ComponentPtrSet] locked - * cdef ComponentPtr cmpptr - * lockedptr = self._components.lock() # <<<<<<<<<<<<<< - * if lockedptr: - * for cmpptr in deref(lockedptr): - */ - __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_components.lock(); - - /* "khmer/_oxli/partitioning.pyx":132 - * cdef ComponentPtr cmpptr - * lockedptr = self._components.lock() - * if lockedptr: # <<<<<<<<<<<<<< - * for cmpptr in deref(lockedptr): - * yield Component.create(cmpptr) - */ - __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); - if (__pyx_t_1) { - - /* "khmer/_oxli/partitioning.pyx":133 - * lockedptr = self._components.lock() - * if lockedptr: - * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< - * yield Component.create(cmpptr) - * else: - */ - __pyx_t_3 = &(*__pyx_cur_scope->__pyx_v_lockedptr); - __pyx_t_2 = __pyx_t_3->begin(); - for (;;) { - if (!(__pyx_t_2 != __pyx_t_3->end())) break; - __pyx_t_4 = *__pyx_t_2; - ++__pyx_t_2; - __pyx_cur_scope->__pyx_v_cmpptr = __pyx_t_4; - - /* "khmer/_oxli/partitioning.pyx":134 - * if lockedptr: - * for cmpptr in deref(lockedptr): - * yield Component.create(cmpptr) # <<<<<<<<<<<<<< - * else: - * raise MemoryError("Can't locked underlying Component set") - */ - __pyx_t_5 = ((PyObject *)__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(__pyx_cur_scope->__pyx_v_cmpptr)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 134, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_r = __pyx_t_5; - __pyx_t_5 = 0; - __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L7_resume_from_yield:; - __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; - __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 134, __pyx_L1_error) - - /* "khmer/_oxli/partitioning.pyx":133 - * lockedptr = self._components.lock() - * if lockedptr: - * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< - * yield Component.create(cmpptr) - * else: - */ - } - - /* "khmer/_oxli/partitioning.pyx":132 - * cdef ComponentPtr cmpptr - * lockedptr = self._components.lock() - * if lockedptr: # <<<<<<<<<<<<<< - * for cmpptr in deref(lockedptr): - * yield Component.create(cmpptr) - */ - goto __pyx_L4; - } - - /* "khmer/_oxli/partitioning.pyx":136 - * yield Component.create(cmpptr) - * else: - * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< - * - * def tag_components(self): - */ - /*else*/ { - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 136, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_Raise(__pyx_t_5, 0, 0, 0); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(0, 136, __pyx_L1_error) - } - __pyx_L4:; - if (1); else __pyx_cur_scope = __pyx_cur_scope; - - /* "khmer/_oxli/partitioning.pyx":128 - * return Component.create(compptr) - * - * def components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[ComponentPtrSet] locked - * cdef ComponentPtr cmpptr - */ - - /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_5); - __Pyx_AddTraceback("components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_XDECREF(__pyx_r); __pyx_r = 0; - __pyx_generator->resume_label = -1; - __Pyx_Coroutine_clear((PyObject*)__pyx_generator); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_15generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ - -/* "khmer/_oxli/partitioning.pyx":138 - * raise MemoryError("Can't locked underlying Component set") - * - * def tag_components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[CpGuardedKmerCompMap] locked - * cdef pair[HashIntoType,ComponentPtr] cpair - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_14tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_14tag_components(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("tag_components (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_13tag_components(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_13tag_components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *__pyx_cur_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("tag_components", 0); - __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(__pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(0, 138, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_15generator2, (PyObject *) __pyx_cur_scope, __pyx_n_s_tag_components, __pyx_n_s_StreamingPartitioner_tag_compone, __pyx_n_s_khmer__oxli_partitioning); if (unlikely(!gen)) __PYX_ERR(0, 138, __pyx_L1_error) - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_gb_5khmer_5_oxli_12partitioning_20StreamingPartitioner_15generator2(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)__pyx_generator->closure); - PyObject *__pyx_r = NULL; - bool __pyx_t_1; - std::map ::iterator __pyx_t_2; - std::map *__pyx_t_3; - std::pair __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L7_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 138, __pyx_L1_error) - - /* "khmer/_oxli/partitioning.pyx":141 - * cdef shared_ptr[CpGuardedKmerCompMap] locked - * cdef pair[HashIntoType,ComponentPtr] cpair - * lockedptr = self._tag_component_map.lock() # <<<<<<<<<<<<<< - * if lockedptr: - * for cpair in deref(lockedptr).data: - */ - __pyx_cur_scope->__pyx_v_lockedptr = __pyx_cur_scope->__pyx_v_self->_tag_component_map.lock(); - - /* "khmer/_oxli/partitioning.pyx":142 - * cdef pair[HashIntoType,ComponentPtr] cpair - * lockedptr = self._tag_component_map.lock() - * if lockedptr: # <<<<<<<<<<<<<< - * for cpair in deref(lockedptr).data: - * yield cpair.first, Component.create(cpair.second) - */ - __pyx_t_1 = __pyx_cur_scope->__pyx_v_lockedptr.operator bool(); - if (__pyx_t_1) { - - /* "khmer/_oxli/partitioning.pyx":143 - * lockedptr = self._tag_component_map.lock() - * if lockedptr: - * for cpair in deref(lockedptr).data: # <<<<<<<<<<<<<< - * yield cpair.first, Component.create(cpair.second) - * else: - */ - __pyx_t_3 = &(*__pyx_cur_scope->__pyx_v_lockedptr).data; - __pyx_t_2 = __pyx_t_3->begin(); - for (;;) { - if (!(__pyx_t_2 != __pyx_t_3->end())) break; - __pyx_t_4 = *__pyx_t_2; - ++__pyx_t_2; - __pyx_cur_scope->__pyx_v_cpair = __pyx_t_4; - - /* "khmer/_oxli/partitioning.pyx":144 - * if lockedptr: - * for cpair in deref(lockedptr).data: - * yield cpair.first, Component.create(cpair.second) # <<<<<<<<<<<<<< - * else: - * raise MemoryError("Can't locked underlying Component set") - */ - __pyx_t_5 = __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(__pyx_cur_scope->__pyx_v_cpair.first); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 144, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = ((PyObject *)__pyx_f_5khmer_5_oxli_12partitioning_9Component_create(__pyx_cur_scope->__pyx_v_cpair.second)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 144, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 144, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_GIVEREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6); - __pyx_t_5 = 0; - __pyx_t_6 = 0; - __pyx_r = __pyx_t_7; - __pyx_t_7 = 0; - __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L7_resume_from_yield:; - __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; - __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 144, __pyx_L1_error) - - /* "khmer/_oxli/partitioning.pyx":143 - * lockedptr = self._tag_component_map.lock() - * if lockedptr: - * for cpair in deref(lockedptr).data: # <<<<<<<<<<<<<< - * yield cpair.first, Component.create(cpair.second) - * else: - */ - } - - /* "khmer/_oxli/partitioning.pyx":142 - * cdef pair[HashIntoType,ComponentPtr] cpair - * lockedptr = self._tag_component_map.lock() - * if lockedptr: # <<<<<<<<<<<<<< - * for cpair in deref(lockedptr).data: - * yield cpair.first, Component.create(cpair.second) - */ - goto __pyx_L4; - } - - /* "khmer/_oxli/partitioning.pyx":146 - * yield cpair.first, Component.create(cpair.second) - * else: - * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< - * - * def write_components(self, filename): - */ - /*else*/ { - __pyx_t_7 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 146, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_Raise(__pyx_t_7, 0, 0, 0); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __PYX_ERR(0, 146, __pyx_L1_error) - } - __pyx_L4:; - if (1); else __pyx_cur_scope = __pyx_cur_scope; - - /* "khmer/_oxli/partitioning.pyx":138 - * raise MemoryError("Can't locked underlying Component set") - * - * def tag_components(self): # <<<<<<<<<<<<<< - * cdef shared_ptr[CpGuardedKmerCompMap] locked - * cdef pair[HashIntoType,ComponentPtr] cpair - */ - - /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_AddTraceback("tag_components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_XDECREF(__pyx_r); __pyx_r = 0; - __pyx_generator->resume_label = -1; - __Pyx_Coroutine_clear((PyObject*)__pyx_generator); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":148 - * raise MemoryError("Can't locked underlying Component set") - * - * def write_components(self, filename): # <<<<<<<<<<<<<< - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_17write_components(PyObject *__pyx_v_self, PyObject *__pyx_v_filename); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_17write_components(PyObject *__pyx_v_self, PyObject *__pyx_v_filename) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("write_components (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_16write_components(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self), ((PyObject *)__pyx_v_filename)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_16write_components(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self, PyObject *__pyx_v_filename) { - FILE *__pyx_v_fp; - khmer::ComponentPtr __pyx_v_cmpptr; - std::shared_ptr __pyx_v_lockedptr; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - char const *__pyx_t_3; - int __pyx_t_4; - bool __pyx_t_5; - std::set ::iterator __pyx_t_6; - khmer::ComponentPtrSet *__pyx_t_7; - khmer::ComponentPtr __pyx_t_8; - __Pyx_RefNannySetupContext("write_components", 0); - - /* "khmer/_oxli/partitioning.pyx":150 - * def write_components(self, filename): - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') # <<<<<<<<<<<<<< - * if fp == NULL: - * raise IOError("Can't open file.") - */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_filename, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 150, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 150, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_PyObject_AsString(__pyx_t_2); if (unlikely((!__pyx_t_3) && PyErr_Occurred())) __PYX_ERR(0, 150, __pyx_L1_error) - __pyx_v_fp = fopen(__pyx_t_3, ((char const *)"wb")); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "khmer/_oxli/partitioning.pyx":151 - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') - * if fp == NULL: # <<<<<<<<<<<<<< - * raise IOError("Can't open file.") - * - */ - __pyx_t_4 = ((__pyx_v_fp == NULL) != 0); - if (__pyx_t_4) { - - /* "khmer/_oxli/partitioning.pyx":152 - * fp = fopen(filename.encode('utf-8'), 'wb') - * if fp == NULL: - * raise IOError("Can't open file.") # <<<<<<<<<<<<<< - * - * cdef ComponentPtr cmpptr - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 152, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(0, 152, __pyx_L1_error) - - /* "khmer/_oxli/partitioning.pyx":151 - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') - * if fp == NULL: # <<<<<<<<<<<<<< - * raise IOError("Can't open file.") - * - */ - } - - /* "khmer/_oxli/partitioning.pyx":156 - * cdef ComponentPtr cmpptr - * cdef shared_ptr[ComponentPtrSet] lockedptr - * lockedptr = self._components.lock() # <<<<<<<<<<<<<< - * - * if lockedptr: - */ - __pyx_v_lockedptr = __pyx_v_self->_components.lock(); - - /* "khmer/_oxli/partitioning.pyx":158 - * lockedptr = self._components.lock() - * - * if lockedptr: # <<<<<<<<<<<<<< - * for cmpptr in deref(lockedptr): - * fprintf(fp, "%llu,%llu,%f\n", - */ - __pyx_t_5 = __pyx_v_lockedptr.operator bool(); - if (__pyx_t_5) { - - /* "khmer/_oxli/partitioning.pyx":159 - * - * if lockedptr: - * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< - * fprintf(fp, "%llu,%llu,%f\n", - * deref(cmpptr).component_id, - */ - __pyx_t_7 = &(*__pyx_v_lockedptr); - __pyx_t_6 = __pyx_t_7->begin(); - for (;;) { - if (!(__pyx_t_6 != __pyx_t_7->end())) break; - __pyx_t_8 = *__pyx_t_6; - ++__pyx_t_6; - __pyx_v_cmpptr = __pyx_t_8; - - /* "khmer/_oxli/partitioning.pyx":160 - * if lockedptr: - * for cmpptr in deref(lockedptr): - * fprintf(fp, "%llu,%llu,%f\n", # <<<<<<<<<<<<<< - * deref(cmpptr).component_id, - * deref(cmpptr).get_n_tags(), - */ - fprintf(__pyx_v_fp, ((char const *)"%llu,%llu,%f\n"), (*__pyx_v_cmpptr).component_id, (*__pyx_v_cmpptr).get_n_tags(), __pyx_f_5khmer_5_oxli_12partitioning_9Component__mean_tag_count(__pyx_v_cmpptr, __pyx_v_self->_graph_ptr)); - - /* "khmer/_oxli/partitioning.pyx":159 - * - * if lockedptr: - * for cmpptr in deref(lockedptr): # <<<<<<<<<<<<<< - * fprintf(fp, "%llu,%llu,%f\n", - * deref(cmpptr).component_id, - */ - } - - /* "khmer/_oxli/partitioning.pyx":158 - * lockedptr = self._components.lock() - * - * if lockedptr: # <<<<<<<<<<<<<< - * for cmpptr in deref(lockedptr): - * fprintf(fp, "%llu,%llu,%f\n", - */ - } - - /* "khmer/_oxli/partitioning.pyx":164 - * deref(cmpptr).get_n_tags(), - * Component._mean_tag_count(cmpptr, self._graph_ptr)) - * fclose(fp) # <<<<<<<<<<<<<< - * - * property n_components: - */ - fclose(__pyx_v_fp); - - /* "khmer/_oxli/partitioning.pyx":148 - * raise MemoryError("Can't locked underlying Component set") - * - * def write_components(self, filename): # <<<<<<<<<<<<<< - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.write_components", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":167 - * - * property n_components: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_components() - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli/partitioning.pyx":168 - * property n_components: - * def __get__(self): - * return deref(self._this).get_n_components() # <<<<<<<<<<<<<< - * - * property n_tags: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_components()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 168, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":167 - * - * property n_components: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_components() - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.n_components.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pyx":171 - * - * property n_tags: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_tags() - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "khmer/_oxli/partitioning.pyx":172 - * property n_tags: - * def __get__(self): - * return deref(self._this).get_n_tags() # <<<<<<<<<<<<<< - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t((*__pyx_v_self->_this).get_n_tags()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 172, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "khmer/_oxli/partitioning.pyx":171 - * - * property n_tags: - * def __get__(self): # <<<<<<<<<<<<<< - * return deref(self._this).get_n_tags() - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.n_tags.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "khmer/_oxli/partitioning.pxd":26 - * cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map - * cdef CpHashtable * _graph_ptr - * cdef readonly uint64_t n_consumed # <<<<<<<<<<<<<< - * - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed___get__(((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed___get__(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_uint64_t(__pyx_v_self->n_consumed); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 26, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("khmer._oxli.partitioning.StreamingPartitioner.n_consumed.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "vector.to_py":67 - * - * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") - * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): # <<<<<<<<<<<<<< - * return [X_to_py(v[i]) for i in range(v.size())] - * - */ - -static PyObject *__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(const std::vector &__pyx_v_v) { - size_t __pyx_v_i; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - size_t __pyx_t_2; - size_t __pyx_t_3; - PyObject *__pyx_t_4 = NULL; - __Pyx_RefNannySetupContext("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType", 0); - - /* "vector.to_py":68 - * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") - * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): - * return [X_to_py(v[i]) for i in range(v.size())] # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 68, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_v_v.size(); - for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { - __pyx_v_i = __pyx_t_3; - __pyx_t_4 = __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 68, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(2, 68, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - } - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "vector.to_py":67 - * - * @cname("__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType") - * cdef object __pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType(vector[X]& v): # <<<<<<<<<<<<<< - * return [X_to_py(v[i]) for i in range(v.size())] - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("vector.to_py.__pyx_convert_vector_to_py_khmer_3a__3a_BoundedCounterType", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - -static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { - Py_ssize_t __pyx_v_length; - char *__pyx_v_data; - std::string __pyx_r; - __Pyx_RefNannyDeclarations - char *__pyx_t_1; - __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); - - /* "string.from_py":15 - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< - * return string(data, length) - * - */ - __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(2, 15, __pyx_L1_error) - __pyx_v_data = __pyx_t_1; - - /* "string.from_py":16 - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - * return string(data, length) # <<<<<<<<<<<<<< - * - * - */ - __pyx_r = std::string(__pyx_v_data, __pyx_v_length); - goto __pyx_L0; - - /* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static struct __pyx_vtabstruct_5khmer_5_oxli_12partitioning_Component __pyx_vtable_5khmer_5_oxli_12partitioning_Component; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning_Component(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)o); - p->__pyx_vtab = __pyx_vtabptr_5khmer_5_oxli_12partitioning_Component; - new((void*)&(p->_this)) khmer::ComponentPtr(); - if (unlikely(__pyx_pw_5khmer_5_oxli_12partitioning_9Component_1__cinit__(o, a, k) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning_Component(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - __Pyx_call_destructor(p->_this); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_9Component_component_id(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_12partitioning_9Component_12component_id_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_9Component__n_created(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_12partitioning_9Component_10_n_created_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_9Component__n_destroyed(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_12partitioning_9Component_12_n_destroyed_1__get__(o); -} - -static PyMethodDef __pyx_methods_5khmer_5_oxli_12partitioning_Component[] = { - {"tag_counts", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_9Component_12tag_counts, METH_VARARGS|METH_KEYWORDS, 0}, - {0, 0, 0, 0} -}; - -static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_12partitioning_Component[] = { - {(char *)"component_id", __pyx_getprop_5khmer_5_oxli_12partitioning_9Component_component_id, 0, (char *)0, 0}, - {(char *)"_n_created", __pyx_getprop_5khmer_5_oxli_12partitioning_9Component__n_created, 0, (char *)0, 0}, - {(char *)"_n_destroyed", __pyx_getprop_5khmer_5_oxli_12partitioning_9Component__n_destroyed, 0, (char *)0, 0}, - {0, 0, 0, 0, 0} -}; - -static PySequenceMethods __pyx_tp_as_sequence_Component = { - __pyx_pw_5khmer_5_oxli_12partitioning_9Component_3__len__, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Component = { - __pyx_pw_5khmer_5_oxli_12partitioning_9Component_3__len__, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning_Component = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.partitioning.Component", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning_Component), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_12partitioning_Component, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - &__pyx_tp_as_sequence_Component, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Component, /*tp_as_mapping*/ - __pyx_pw_5khmer_5_oxli_12partitioning_9Component_8__hash__, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - __pyx_pw_5khmer_5_oxli_12partitioning_9Component_10__richcmp__, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - __pyx_pw_5khmer_5_oxli_12partitioning_9Component_5__iter__, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5khmer_5_oxli_12partitioning_Component, /*tp_methods*/ - 0, /*tp_members*/ - __pyx_getsets_5khmer_5_oxli_12partitioning_Component, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_12partitioning_Component, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning_StreamingPartitioner(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)o); - new((void*)&(p->_this)) std::unique_ptr (); - new((void*)&(p->_components)) std::weak_ptr (); - new((void*)&(p->_tag_component_map)) std::weak_ptr (); - if (unlikely(__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_1__cinit__(o, a, k) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning_StreamingPartitioner(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - __Pyx_call_destructor(p->_this); - __Pyx_call_destructor(p->_components); - __Pyx_call_destructor(p->_tag_component_map); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_components(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_12n_components_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_tags(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_6n_tags_1__get__(o); -} - -static PyObject *__pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_consumed(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_10n_consumed_1__get__(o); -} - -static PyMethodDef __pyx_methods_5khmer_5_oxli_12partitioning_StreamingPartitioner[] = { - {"consume_sequence", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_3consume_sequence, METH_O, 0}, - {"consume_fasta", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_5consume_fasta, METH_O, 0}, - {"get_tag_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_7get_tag_component, METH_O, 0}, - {"get_nearest_component", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_9get_nearest_component, METH_O, 0}, - {"components", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_11components, METH_NOARGS, 0}, - {"tag_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_14tag_components, METH_NOARGS, 0}, - {"write_components", (PyCFunction)__pyx_pw_5khmer_5_oxli_12partitioning_20StreamingPartitioner_17write_components, METH_O, 0}, - {0, 0, 0, 0} -}; - -static struct PyGetSetDef __pyx_getsets_5khmer_5_oxli_12partitioning_StreamingPartitioner[] = { - {(char *)"n_components", __pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_components, 0, (char *)0, 0}, - {(char *)"n_tags", __pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_tags, 0, (char *)0, 0}, - {(char *)"n_consumed", __pyx_getprop_5khmer_5_oxli_12partitioning_20StreamingPartitioner_n_consumed, 0, (char *)0, 0}, - {0, 0, 0, 0, 0} -}; - -static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.partitioning.StreamingPartitioner", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_12partitioning_StreamingPartitioner, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5khmer_5_oxli_12partitioning_StreamingPartitioner, /*tp_methods*/ - 0, /*tp_members*/ - __pyx_getsets_5khmer_5_oxli_12partitioning_StreamingPartitioner, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_12partitioning_StreamingPartitioner, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__[8]; -static int __pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ = 0; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *p; - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__)))) { - o = (PyObject*)__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__[--__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__]; - memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__)); - (void) PyObject_INIT(o, t); - PyObject_GC_Track(o); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o); - new((void*)&(p->__pyx_t_0)) std::set ::iterator(); - return o; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o; - PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_t_0); - Py_CLEAR(p->__pyx_v_self); - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__)))) { - __pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__[__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__++] = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static int __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o; - if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ *)o; - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.partitioning.__pyx_scope_struct____iter__", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, /*tp_traverse*/ - __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components[8]; -static int __pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components = 0; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *p; - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components)))) { - o = (PyObject*)__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components[--__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components]; - memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components)); - (void) PyObject_INIT(o, t); - PyObject_GC_Track(o); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o); - new((void*)&(p->__pyx_v_cmpptr)) khmer::ComponentPtr(); - new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); - new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); - new((void*)&(p->__pyx_t_0)) std::set ::iterator(); - return o; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o; - PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_v_cmpptr); - __Pyx_call_destructor(p->__pyx_v_locked); - __Pyx_call_destructor(p->__pyx_v_lockedptr); - __Pyx_call_destructor(p->__pyx_t_0); - Py_CLEAR(p->__pyx_v_self); - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components)))) { - __pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components[__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components++] = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static int __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o; - if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components *)o; - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.partitioning.__pyx_scope_struct_1_components", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, /*tp_traverse*/ - __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components[8]; -static int __pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components = 0; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *p; - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components)))) { - o = (PyObject*)__pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components[--__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components]; - memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components)); - (void) PyObject_INIT(o, t); - PyObject_GC_Track(o); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - p = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o); - new((void*)&(p->__pyx_v_cpair)) std::pair (); - new((void*)&(p->__pyx_v_locked)) std::shared_ptr (); - new((void*)&(p->__pyx_v_lockedptr)) std::shared_ptr (); - new((void*)&(p->__pyx_t_0)) std::map ::iterator(); - return o; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o; - PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_v_cpair); - __Pyx_call_destructor(p->__pyx_v_locked); - __Pyx_call_destructor(p->__pyx_v_lockedptr); - __Pyx_call_destructor(p->__pyx_t_0); - Py_CLEAR(p->__pyx_v_self); - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components)))) { - __pyx_freelist_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components[__pyx_freecount_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components++] = ((struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static int __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o; - if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *p = (struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components *)o; - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_12partitioning_StreamingPartitioner *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyTypeObject __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.partitioning.__pyx_scope_struct_2_tag_components", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - __pyx_tp_traverse_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, /*tp_traverse*/ - __pyx_tp_clear_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyMethodDef __pyx_methods[] = { - {0, 0, 0, 0} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - #if PY_VERSION_HEX < 0x03020000 - { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, - #else - PyModuleDef_HEAD_INIT, - #endif - "partitioning", - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp_s_Can_t_locked_underlying_Componen, __pyx_k_Can_t_locked_underlying_Componen, sizeof(__pyx_k_Can_t_locked_underlying_Componen), 0, 0, 1, 0}, - {&__pyx_kp_s_Can_t_open_file, __pyx_k_Can_t_open_file, sizeof(__pyx_k_Can_t_open_file), 0, 0, 1, 0}, - {&__pyx_n_s_Component___iter, __pyx_k_Component___iter, sizeof(__pyx_k_Component___iter), 0, 0, 1, 1}, - {&__pyx_n_s_Countgraph, __pyx_k_Countgraph, sizeof(__pyx_k_Countgraph), 0, 0, 1, 1}, - {&__pyx_n_s_IOError, __pyx_k_IOError, sizeof(__pyx_k_IOError), 0, 0, 1, 1}, - {&__pyx_n_s_MemoryError, __pyx_k_MemoryError, sizeof(__pyx_k_MemoryError), 0, 0, 1, 1}, - {&__pyx_kp_s_Must_take_an_object_with_Hashtab, __pyx_k_Must_take_an_object_with_Hashtab, sizeof(__pyx_k_Must_take_an_object_with_Hashtab), 0, 0, 1, 0}, - {&__pyx_n_s_Nodegraph, __pyx_k_Nodegraph, sizeof(__pyx_k_Nodegraph), 0, 0, 1, 1}, - {&__pyx_n_s_NotImplementedError, __pyx_k_NotImplementedError, sizeof(__pyx_k_NotImplementedError), 0, 0, 1, 1}, - {&__pyx_kp_s_Operator_not_available, __pyx_k_Operator_not_available, sizeof(__pyx_k_Operator_not_available), 0, 0, 1, 0}, - {&__pyx_n_s_StreamingPartitioner_components, __pyx_k_StreamingPartitioner_components, sizeof(__pyx_k_StreamingPartitioner_components), 0, 0, 1, 1}, - {&__pyx_n_s_StreamingPartitioner_tag_compone, __pyx_k_StreamingPartitioner_tag_compone, sizeof(__pyx_k_StreamingPartitioner_tag_compone), 0, 0, 1, 1}, - {&__pyx_kp_s_Users_camille_w_khmer_khmer__ox, __pyx_k_Users_camille_w_khmer_khmer__ox, sizeof(__pyx_k_Users_camille_w_khmer_khmer__ox), 0, 0, 1, 0}, - {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, - {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, - {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, - {&__pyx_n_s_component, __pyx_k_component, sizeof(__pyx_k_component), 0, 0, 1, 1}, - {&__pyx_n_s_component_id, __pyx_k_component_id, sizeof(__pyx_k_component_id), 0, 0, 1, 1}, - {&__pyx_n_s_components, __pyx_k_components, sizeof(__pyx_k_components), 0, 0, 1, 1}, - {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, - {&__pyx_n_s_graph, __pyx_k_graph, sizeof(__pyx_k_graph), 0, 0, 1, 1}, - {&__pyx_n_s_graph_ptr, __pyx_k_graph_ptr, sizeof(__pyx_k_graph_ptr), 0, 0, 1, 1}, - {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, - {&__pyx_n_s_iter, __pyx_k_iter, sizeof(__pyx_k_iter), 0, 0, 1, 1}, - {&__pyx_n_s_khmer, __pyx_k_khmer, sizeof(__pyx_k_khmer), 0, 0, 1, 1}, - {&__pyx_n_s_khmer__oxli_partitioning, __pyx_k_khmer__oxli_partitioning, sizeof(__pyx_k_khmer__oxli_partitioning), 0, 0, 1, 1}, - {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, - {&__pyx_n_s_other, __pyx_k_other, sizeof(__pyx_k_other), 0, 0, 1, 1}, - {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, - {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, - {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, - {&__pyx_n_s_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 0, 0, 1, 1}, - {&__pyx_n_s_tag_components, __pyx_k_tag_components, sizeof(__pyx_k_tag_components), 0, 0, 1, 1}, - {&__pyx_n_s_tag_counts, __pyx_k_tag_counts, sizeof(__pyx_k_tag_counts), 0, 0, 1, 1}, - {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, - {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, - {0, 0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetBuiltinName(__pyx_n_s_staticmethod); if (!__pyx_builtin_staticmethod) __PYX_ERR(0, 75, __pyx_L1_error) - __pyx_builtin_NotImplementedError = __Pyx_GetBuiltinName(__pyx_n_s_NotImplementedError); if (!__pyx_builtin_NotImplementedError) __PYX_ERR(0, 56, __pyx_L1_error) - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 94, __pyx_L1_error) - __pyx_builtin_MemoryError = __Pyx_GetBuiltinName(__pyx_n_s_MemoryError); if (!__pyx_builtin_MemoryError) __PYX_ERR(0, 136, __pyx_L1_error) - __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s_IOError); if (!__pyx_builtin_IOError) __PYX_ERR(0, 152, __pyx_L1_error) - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) - return 0; - __pyx_L1_error:; - return -1; -} - -static int __Pyx_InitCachedConstants(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - - /* "khmer/_oxli/partitioning.pyx":56 - * return x.component_id == y.component_id - * else: - * raise NotImplementedError('Operator not available.') # <<<<<<<<<<<<<< - * - * @staticmethod - */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Operator_not_available); if (unlikely(!__pyx_tuple_)) __PYX_ERR(0, 56, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple_); - __Pyx_GIVEREF(__pyx_tuple_); - - /* "khmer/_oxli/partitioning.pyx":94 - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< - * - * cdef CPyHashtable_Object* ptr = graph - */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(0, 94, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__2); - __Pyx_GIVEREF(__pyx_tuple__2); - - /* "khmer/_oxli/partitioning.pyx":106 - * - * def consume_sequence(self, sequence): - * deref(self._this).consume_sequence(sequence.encode('utf-8')) # <<<<<<<<<<<<<< - * self.n_consumed += 1 - * - */ - __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(0, 106, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__3); - __Pyx_GIVEREF(__pyx_tuple__3); - - /* "khmer/_oxli/partitioning.pyx":110 - * - * def consume_fasta(self, filename): - * return deref(self._this).consume_fasta(filename.encode('utf-8')) # <<<<<<<<<<<<<< - * - * def get_tag_component(self, kmer): - */ - __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(0, 110, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__4); - __Pyx_GIVEREF(__pyx_tuple__4); - - /* "khmer/_oxli/partitioning.pyx":114 - * def get_tag_component(self, kmer): - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if compptr == NULL: - * return None - */ - __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(0, 114, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__5); - __Pyx_GIVEREF(__pyx_tuple__5); - - /* "khmer/_oxli/partitioning.pyx":122 - * def get_nearest_component(self, kmer): - * cdef ComponentPtr compptr - * compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) # <<<<<<<<<<<<<< - * if compptr == NULL: - * return None - */ - __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(0, 122, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__6); - __Pyx_GIVEREF(__pyx_tuple__6); - - /* "khmer/_oxli/partitioning.pyx":136 - * yield Component.create(cmpptr) - * else: - * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< - * - * def tag_components(self): - */ - __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(0, 136, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__7); - __Pyx_GIVEREF(__pyx_tuple__7); - - /* "khmer/_oxli/partitioning.pyx":146 - * yield cpair.first, Component.create(cpair.second) - * else: - * raise MemoryError("Can't locked underlying Component set") # <<<<<<<<<<<<<< - * - * def write_components(self, filename): - */ - __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_s_Can_t_locked_underlying_Componen); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(0, 146, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__8); - __Pyx_GIVEREF(__pyx_tuple__8); - - /* "khmer/_oxli/partitioning.pyx":150 - * def write_components(self, filename): - * cdef FILE* fp - * fp = fopen(filename.encode('utf-8'), 'wb') # <<<<<<<<<<<<<< - * if fp == NULL: - * raise IOError("Can't open file.") - */ - __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(0, 150, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__9); - __Pyx_GIVEREF(__pyx_tuple__9); - - /* "khmer/_oxli/partitioning.pyx":152 - * fp = fopen(filename.encode('utf-8'), 'wb') - * if fp == NULL: - * raise IOError("Can't open file.") # <<<<<<<<<<<<<< - * - * cdef ComponentPtr cmpptr - */ - __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_s_Can_t_open_file); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(0, 152, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - - /* "khmer/_oxli/partitioning.pyx":76 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CPyHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - __pyx_tuple__11 = PyTuple_Pack(3, __pyx_n_s_component, __pyx_n_s_graph, __pyx_n_s_graph_ptr); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(0, 76, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__11); - __Pyx_GIVEREF(__pyx_tuple__11); - __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_camille_w_khmer_khmer__ox, __pyx_n_s_tag_counts, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(0, 76, __pyx_L1_error) - __Pyx_RefNannyFinishContext(); - return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); - __pyx_int_2 = PyInt_FromLong(2); if (unlikely(!__pyx_int_2)) __PYX_ERR(0, 1, __pyx_L1_error) - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC initpartitioning(void); /*proto*/ -PyMODINIT_FUNC initpartitioning(void) -#else -PyMODINIT_FUNC PyInit_partitioning(void); /*proto*/ -PyMODINIT_FUNC PyInit_partitioning(void) -#endif -{ - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannyDeclarations - #if CYTHON_REFNANNY - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); - if (!__Pyx_RefNanny) { - PyErr_Clear(); - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); - if (!__Pyx_RefNanny) - Py_FatalError("failed to import 'refnanny' module"); - } - #endif - __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_partitioning(void)", 0); - if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) - #ifdef __Pyx_CyFunction_USED - if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_FusedFunction_USED - if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Coroutine_USED - if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Generator_USED - if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_StopAsyncIteration_USED - if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - /*--- Library function declarations ---*/ - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4("partitioning", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) - Py_INCREF(__pyx_d); - __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) - #if CYTHON_COMPILING_IN_PYPY - Py_INCREF(__pyx_b); - #endif - if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); - /*--- Initialize various global constants etc. ---*/ - if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - if (__pyx_module_is_main_khmer___oxli__partitioning) { - if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) - } - #if PY_MAJOR_VERSION >= 3 - { - PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) - if (!PyDict_GetItemString(modules, "khmer._oxli.partitioning")) { - if (unlikely(PyDict_SetItemString(modules, "khmer._oxli.partitioning", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) - } - } - #endif - /*--- Builtin init code ---*/ - if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - /*--- Constants init code ---*/ - if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - /*--- Global init code ---*/ - /*--- Variable export code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - __pyx_vtabptr_5khmer_5_oxli_12partitioning_Component = &__pyx_vtable_5khmer_5_oxli_12partitioning_Component; - __pyx_vtable_5khmer_5_oxli_12partitioning_Component.create = (struct __pyx_obj_5khmer_5_oxli_12partitioning_Component *(*)(khmer::ComponentPtr))__pyx_f_5khmer_5_oxli_12partitioning_9Component_create; - __pyx_vtable_5khmer_5_oxli_12partitioning_Component._tag_counts = (std::vector (*)(khmer::ComponentPtr, khmer::Hashtable *))__pyx_f_5khmer_5_oxli_12partitioning_9Component__tag_counts; - __pyx_vtable_5khmer_5_oxli_12partitioning_Component._mean_tag_count = (float (*)(khmer::ComponentPtr, khmer::Hashtable *))__pyx_f_5khmer_5_oxli_12partitioning_9Component__mean_tag_count; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning_Component) < 0) __PYX_ERR(0, 23, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_12partitioning_Component.tp_print = 0; - if (__Pyx_SetVtable(__pyx_type_5khmer_5_oxli_12partitioning_Component.tp_dict, __pyx_vtabptr_5khmer_5_oxli_12partitioning_Component) < 0) __PYX_ERR(0, 23, __pyx_L1_error) - if (PyObject_SetAttrString(__pyx_m, "Component", (PyObject *)&__pyx_type_5khmer_5_oxli_12partitioning_Component) < 0) __PYX_ERR(0, 23, __pyx_L1_error) - __pyx_ptype_5khmer_5_oxli_12partitioning_Component = &__pyx_type_5khmer_5_oxli_12partitioning_Component; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner) < 0) __PYX_ERR(0, 90, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "StreamingPartitioner", (PyObject *)&__pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner) < 0) __PYX_ERR(0, 90, __pyx_L1_error) - __pyx_ptype_5khmer_5_oxli_12partitioning_StreamingPartitioner = &__pyx_type_5khmer_5_oxli_12partitioning_StreamingPartitioner; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__) < 0) __PYX_ERR(0, 44, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__.tp_print = 0; - __pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__ = &__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct____iter__; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components) < 0) __PYX_ERR(0, 128, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components.tp_print = 0; - __pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components = &__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_1_components; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components) < 0) __PYX_ERR(0, 138, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components.tp_print = 0; - __pyx_ptype_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components = &__pyx_type_5khmer_5_oxli_12partitioning___pyx_scope_struct_2_tag_components; - /*--- Type import code ---*/ - /*--- Variable import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - - /* "khmer/_oxli/partitioning.pyx":20 - * - * from _oxli cimport * - * from .._khmer import Countgraph # <<<<<<<<<<<<<< - * from .._khmer import Nodegraph - * - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 20, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(__pyx_n_s_Countgraph); - __Pyx_GIVEREF(__pyx_n_s_Countgraph); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_Countgraph); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer, __pyx_t_1, 2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 20, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 20, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(0, 20, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "khmer/_oxli/partitioning.pyx":21 - * from _oxli cimport * - * from .._khmer import Countgraph - * from .._khmer import Nodegraph # <<<<<<<<<<<<<< - * - * cdef class Component: - */ - __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_INCREF(__pyx_n_s_Nodegraph); - __Pyx_GIVEREF(__pyx_n_s_Nodegraph); - PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Nodegraph); - __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer, __pyx_t_2, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(0, 21, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "khmer/_oxli/partitioning.pyx":76 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CPyHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5khmer_5_oxli_12partitioning_9Component_12tag_counts, NULL, __pyx_n_s_khmer__oxli_partitioning); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - - /* "khmer/_oxli/partitioning.pyx":75 - * return counts - * - * @staticmethod # <<<<<<<<<<<<<< - * def tag_counts(Component component, graph): - * cdef CPyHashtable_Object* graph_ptr = graph - */ - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 75, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_staticmethod, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 75, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5khmer_5_oxli_12partitioning_Component->tp_dict, __pyx_n_s_tag_counts, __pyx_t_1) < 0) __PYX_ERR(0, 76, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - PyType_Modified(__pyx_ptype_5khmer_5_oxli_12partitioning_Component); - - /* "khmer/_oxli/partitioning.pyx":76 - * - * @staticmethod - * def tag_counts(Component component, graph): # <<<<<<<<<<<<<< - * cdef CPyHashtable_Object* graph_ptr = graph - * return Component._tag_counts(component._this, deref(graph_ptr).hashtable) - */ - __pyx_t_1 = __Pyx_GetNameInClass((PyObject *)__pyx_ptype_5khmer_5_oxli_12partitioning_Component, __pyx_n_s_tag_counts); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - - /* "khmer/_oxli/partitioning.pyx":75 - * return counts - * - * @staticmethod # <<<<<<<<<<<<<< - * def tag_counts(Component component, graph): - * cdef CPyHashtable_Object* graph_ptr = graph - */ - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 75, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_staticmethod, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 75, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5khmer_5_oxli_12partitioning_Component->tp_dict, __pyx_n_s_tag_counts, __pyx_t_1) < 0) __PYX_ERR(0, 76, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - PyType_Modified(__pyx_ptype_5khmer_5_oxli_12partitioning_Component); - - /* "khmer/_oxli/partitioning.pyx":1 - * import cython # <<<<<<<<<<<<<< - * from cython.operator cimport dereference as deref, preincrement as inc - * - */ - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "string.from_py":13 - * - * @cname("__pyx_convert_string_from_py_std__in_string") - * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< - * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) - */ - - /*--- Wrapped vars code ---*/ - - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - if (__pyx_m) { - if (__pyx_d) { - __Pyx_AddTraceback("init khmer._oxli.partitioning", __pyx_clineno, __pyx_lineno, __pyx_filename); - } - Py_DECREF(__pyx_m); __pyx_m = 0; - } else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ImportError, "init khmer._oxli.partitioning"); - } - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -/* --- Runtime support code --- */ -/* Refnanny */ -#if CYTHON_REFNANNY -static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { - PyObject *m = NULL, *p = NULL; - void *r = NULL; - m = PyImport_ImportModule((char *)modname); - if (!m) goto end; - p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); - if (!p) goto end; - r = PyLong_AsVoidPtr(p); -end: - Py_XDECREF(p); - Py_XDECREF(m); - return (__Pyx_RefNannyAPIStruct *)r; -} -#endif - -/* GetBuiltinName */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); - if (unlikely(!result)) { - PyErr_Format(PyExc_NameError, -#if PY_MAJOR_VERSION >= 3 - "name '%U' is not defined", name); -#else - "name '%.200s' is not defined", PyString_AS_STRING(name)); -#endif - } - return result; -} - -/* RaiseDoubleKeywords */ -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AsString(kw_name)); - #endif -} - -/* ParseKeywords */ -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - continue; - } - name = first_kw_arg; - #if PY_MAJOR_VERSION < 3 - if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { - while (*name) { - if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) - && _PyString_Eq(**name, key)) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - if ((**argname == key) || ( - (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) - && _PyString_Eq(**argname, key))) { - goto arg_passed_twice; - } - argname++; - } - } - } else - #endif - if (likely(PyUnicode_Check(key))) { - while (*name) { - int cmp = (**name == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**name, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - int cmp = (**argname == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**argname, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) goto arg_passed_twice; - argname++; - } - } - } else - goto invalid_keyword_type; - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -/* RaiseArgTupleInvalid */ -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *more_or_less; - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", - func_name, more_or_less, num_expected, - (num_expected == 1) ? "" : "s", num_found); -} - -/* ArgTypeTest */ -static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); -} -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - if (none_allowed && obj == Py_None) return 1; - else if (exact) { - if (likely(Py_TYPE(obj) == type)) return 1; - #if PY_MAJOR_VERSION == 2 - else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } - else { - if (likely(PyObject_TypeCheck(obj, type))) return 1; - } - __Pyx_RaiseArgumentTypeInvalid(name, obj, type); - return 0; -} - -/* PyIntBinop */ -#if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { - if (op1 == op2) { - Py_RETURN_TRUE; - } - #if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(op1))) { - const long b = intval; - long a = PyInt_AS_LONG(op1); - if (a == b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } - } - #endif - #if CYTHON_USE_PYLONG_INTERNALS - if (likely(PyLong_CheckExact(op1))) { - const long b = intval; - long a; - const digit* digits = ((PyLongObject*)op1)->ob_digit; - const Py_ssize_t size = Py_SIZE(op1); - if (likely(__Pyx_sst_abs(size) <= 1)) { - a = likely(size) ? digits[0] : 0; - if (size == -1) a = -a; - } else { - switch (size) { - case -2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case -3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case -4: - if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 4: - if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 - default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); - #else - default: Py_RETURN_FALSE; - #endif - } - } - if (a == b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } - } - #endif - if (PyFloat_CheckExact(op1)) { - const long b = intval; - double a = PyFloat_AS_DOUBLE(op1); - if ((double)a == (double)b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } - } - return PyObject_RichCompare(op1, op2, Py_EQ); -} -#endif - -/* PyObjectCall */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyObject *result; - ternaryfunc call = func->ob_type->tp_call; - if (unlikely(!call)) - return PyObject_Call(func, arg, kw); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = (*call)(func, arg, kw); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyErrFetchRestore */ -#if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -} -#endif - -/* RaiseException */ -#if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { - __Pyx_PyThreadState_declare - Py_XINCREF(type); - if (!value || value == Py_None) - value = NULL; - else - Py_INCREF(value); - if (!tb || tb == Py_None) - tb = NULL; - else { - Py_INCREF(tb); - if (!PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - } - if (PyType_Check(type)) { -#if CYTHON_COMPILING_IN_PYPY - if (!value) { - Py_INCREF(Py_None); - value = Py_None; - } -#endif - PyErr_NormalizeException(&type, &value, &tb); - } else { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - value = type; - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - } - __Pyx_PyThreadState_assign - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} -#else -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { - PyObject* owned_instance = NULL; - if (tb == Py_None) { - tb = 0; - } else if (tb && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto bad; - } - if (value == Py_None) - value = 0; - if (PyExceptionInstance_Check(type)) { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto bad; - } - value = type; - type = (PyObject*) Py_TYPE(value); - } else if (PyExceptionClass_Check(type)) { - PyObject *instance_class = NULL; - if (value && PyExceptionInstance_Check(value)) { - instance_class = (PyObject*) Py_TYPE(value); - if (instance_class != type) { - int is_subclass = PyObject_IsSubclass(instance_class, type); - if (!is_subclass) { - instance_class = NULL; - } else if (unlikely(is_subclass == -1)) { - goto bad; - } else { - type = instance_class; - } - } - } - if (!instance_class) { - PyObject *args; - if (!value) - args = PyTuple_New(0); - else if (PyTuple_Check(value)) { - Py_INCREF(value); - args = value; - } else - args = PyTuple_Pack(1, value); - if (!args) - goto bad; - owned_instance = PyObject_Call(type, args, NULL); - Py_DECREF(args); - if (!owned_instance) - goto bad; - value = owned_instance; - if (!PyExceptionInstance_Check(value)) { - PyErr_Format(PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto bad; - } - } - } else { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto bad; - } -#if PY_VERSION_HEX >= 0x03030000 - if (cause) { -#else - if (cause && cause != Py_None) { -#endif - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; - } - PyException_SetCause(value, fixed_cause); - } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = PyThreadState_GET(); - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); - } -#endif - } -bad: - Py_XDECREF(owned_instance); - return; -} -#endif - -/* RaiseTooManyValuesToUnpack */ - static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { - PyErr_Format(PyExc_ValueError, - "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); -} - -/* RaiseNeedMoreValuesToUnpack */ - static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { - PyErr_Format(PyExc_ValueError, - "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", - index, (index == 1) ? "" : "s"); -} - -/* IterFinish */ - static CYTHON_INLINE int __Pyx_IterFinish(void) { -#if CYTHON_FAST_THREAD_STATE - PyThreadState *tstate = PyThreadState_GET(); - PyObject* exc_type = tstate->curexc_type; - if (unlikely(exc_type)) { - if (likely(exc_type == PyExc_StopIteration) || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) { - PyObject *exc_value, *exc_tb; - exc_value = tstate->curexc_value; - exc_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; - Py_DECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_tb); - return 0; - } else { - return -1; - } - } - return 0; -#else - if (unlikely(PyErr_Occurred())) { - if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) { - PyErr_Clear(); - return 0; - } else { - return -1; - } - } - return 0; -#endif -} - -/* UnpackItemEndCheck */ - static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { - if (unlikely(retval)) { - Py_DECREF(retval); - __Pyx_RaiseTooManyValuesError(expected); - return -1; - } else { - return __Pyx_IterFinish(); - } - return 0; -} - -/* WriteUnraisableException */ - static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, - CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, - int full_traceback, CYTHON_UNUSED int nogil) { - PyObject *old_exc, *old_val, *old_tb; - PyObject *ctx; - __Pyx_PyThreadState_declare -#ifdef WITH_THREAD - PyGILState_STATE state; - if (nogil) - state = PyGILState_Ensure(); -#ifdef _MSC_VER - else state = (PyGILState_STATE)-1; -#endif -#endif - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); - if (full_traceback) { - Py_XINCREF(old_exc); - Py_XINCREF(old_val); - Py_XINCREF(old_tb); - __Pyx_ErrRestore(old_exc, old_val, old_tb); - PyErr_PrintEx(1); - } - #if PY_MAJOR_VERSION < 3 - ctx = PyString_FromString(name); - #else - ctx = PyUnicode_FromString(name); - #endif - __Pyx_ErrRestore(old_exc, old_val, old_tb); - if (!ctx) { - PyErr_WriteUnraisable(Py_None); - } else { - PyErr_WriteUnraisable(ctx); - Py_DECREF(ctx); - } -#ifdef WITH_THREAD - if (nogil) - PyGILState_Release(state); -#endif -} - -/* GetModuleGlobalName */ - static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS - result = PyDict_GetItem(__pyx_d, name); - if (likely(result)) { - Py_INCREF(result); - } else { -#else - result = PyObject_GetItem(__pyx_d, name); - if (!result) { - PyErr_Clear(); -#endif - result = __Pyx_GetBuiltinName(name); - } - return result; -} - -/* SetVTable */ - static int __Pyx_SetVtable(PyObject *dict, void *vtable) { -#if PY_VERSION_HEX >= 0x02070000 - PyObject *ob = PyCapsule_New(vtable, 0, 0); -#else - PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); -#endif - if (!ob) - goto bad; - if (PyDict_SetItem(dict, __pyx_n_s_pyx_vtable, ob) < 0) - goto bad; - Py_DECREF(ob); - return 0; -bad: - Py_XDECREF(ob); - return -1; -} - -/* Import */ - static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_import; - py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); - if (!py_import) - goto bad; - #endif - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - { - #if PY_MAJOR_VERSION >= 3 - if (level == -1) { - if (strchr(__Pyx_MODULE_NAME, '.')) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(1); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - #endif - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - goto bad; - PyErr_Clear(); - } - } - level = 0; - } - #endif - if (!module) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(level); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); - #endif - } - } -bad: - #if PY_VERSION_HEX < 0x03030000 - Py_XDECREF(py_import); - #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); - return module; -} - -/* ImportFrom */ - static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { - PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); - if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Format(PyExc_ImportError, - #if PY_MAJOR_VERSION < 3 - "cannot import name %.230s", PyString_AS_STRING(name)); - #else - "cannot import name %S", name); - #endif - } - return value; -} - -/* GetNameInClass */ - static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name) { - PyObject *result; - result = __Pyx_PyObject_GetAttrStr(nmspace, name); - if (!result) - result = __Pyx_GetModuleGlobalName(name); - return result; -} - -/* CodeObjectCache */ - static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { - int start = 0, mid = 0, end = count - 1; - if (end >= 0 && code_line > entries[end].code_line) { - return count; - } - while (start < end) { - mid = start + (end - start) / 2; - if (code_line < entries[mid].code_line) { - end = mid; - } else if (code_line > entries[mid].code_line) { - start = mid + 1; - } else { - return mid; - } - } - if (code_line <= entries[mid].code_line) { - return mid; - } else { - return mid + 1; - } -} -static PyCodeObject *__pyx_find_code_object(int code_line) { - PyCodeObject* code_object; - int pos; - if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { - return NULL; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { - return NULL; - } - code_object = __pyx_code_cache.entries[pos].code_object; - Py_INCREF(code_object); - return code_object; -} -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { - int pos, i; - __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; - if (unlikely(!code_line)) { - return; - } - if (unlikely(!entries)) { - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); - if (likely(entries)) { - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = 64; - __pyx_code_cache.count = 1; - entries[0].code_line = code_line; - entries[0].code_object = code_object; - Py_INCREF(code_object); - } - return; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { - PyCodeObject* tmp = entries[pos].code_object; - entries[pos].code_object = code_object; - Py_DECREF(tmp); - return; - } - if (__pyx_code_cache.count == __pyx_code_cache.max_count) { - int new_max = __pyx_code_cache.max_count + 64; - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( - __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); - if (unlikely(!entries)) { - return; - } - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = new_max; - } - for (i=__pyx_code_cache.count; i>pos; i--) { - entries[i] = entries[i-1]; - } - entries[pos].code_line = code_line; - entries[pos].code_object = code_object; - __pyx_code_cache.count++; - Py_INCREF(code_object); -} - -/* AddTraceback */ - #include "compile.h" -#include "frameobject.h" -#include "traceback.h" -static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( - const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(filename); - #else - py_srcfile = PyUnicode_FromString(filename); - #endif - if (!py_srcfile) goto bad; - if (c_line) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_code = __Pyx_PyCode_New( - 0, - 0, - 0, - 0, - 0, - __pyx_empty_bytes, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - py_line, - __pyx_empty_bytes /*PyObject *lnotab*/ - ); - Py_DECREF(py_srcfile); - Py_DECREF(py_funcname); - return py_code; -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - return NULL; -} -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - py_code = __pyx_find_code_object(c_line ? c_line : py_line); - if (!py_code) { - py_code = __Pyx_CreateCodeObjectForTraceback( - funcname, c_line, py_line, filename); - if (!py_code) goto bad; - __pyx_insert_code_object(c_line ? c_line : py_line, py_code); - } - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - __pyx_d, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - __Pyx_PyFrame_SetLineNumber(py_frame, py_line); - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { - const int neg_one = (int) -1, const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(int) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(int) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(int) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(int), - little, !is_unsigned); - } -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value) { - const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(uint64_t) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(uint64_t) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(uint64_t) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(uint64_t), - little, !is_unsigned); - } -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_HashIntoType(khmer::HashIntoType value) { - const khmer::HashIntoType neg_one = (khmer::HashIntoType) -1, const_zero = (khmer::HashIntoType) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(khmer::HashIntoType) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::HashIntoType) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(khmer::HashIntoType) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::HashIntoType) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(khmer::HashIntoType), - little, !is_unsigned); - } -} - -/* CIntFromPyVerify */ - #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) -#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) -#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ - {\ - func_type value = func_value;\ - if (sizeof(target_type) < sizeof(func_type)) {\ - if (unlikely(value != (func_type) (target_type) value)) {\ - func_type zero = 0;\ - if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ - return (target_type) -1;\ - if (is_unsigned && unlikely(value < zero))\ - goto raise_neg_overflow;\ - else\ - goto raise_overflow;\ - }\ - }\ - return (target_type) value;\ - } - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_khmer_3a__3a_BoundedCounterType(khmer::BoundedCounterType value) { - const khmer::BoundedCounterType neg_one = (khmer::BoundedCounterType) -1, const_zero = (khmer::BoundedCounterType) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(khmer::BoundedCounterType) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(khmer::BoundedCounterType) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::BoundedCounterType) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(khmer::BoundedCounterType) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(khmer::BoundedCounterType) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(khmer::BoundedCounterType), - little, !is_unsigned); - } -} - -/* CIntFromPy */ - static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) -1, const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (int) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { - return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { - return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { - return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (int) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) - case -2: - if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - } -#endif - if (sizeof(int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - int val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (int) -1; - } - } else { - int val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (int) -1; - val = __Pyx_PyInt_As_int(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to int"); - return (int) -1; -} - -/* CIntFromPy */ - static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *x) { - const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(uint64_t) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(uint64_t, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (uint64_t) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (uint64_t) 0; - case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, digits[0]) - case 2: - if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) >= 2 * PyLong_SHIFT) { - return (uint64_t) (((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) >= 3 * PyLong_SHIFT) { - return (uint64_t) (((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) >= 4 * PyLong_SHIFT) { - return (uint64_t) (((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (uint64_t) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(uint64_t) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (uint64_t) 0; - case -1: __PYX_VERIFY_RETURN_INT(uint64_t, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, +digits[0]) - case -2: - if (8 * sizeof(uint64_t) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { - return (uint64_t) (((uint64_t)-1)*(((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { - return (uint64_t) ((((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { - return (uint64_t) (((uint64_t)-1)*(((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { - return (uint64_t) ((((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { - return (uint64_t) (((uint64_t)-1)*(((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { - return (uint64_t) ((((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); - } - } - break; - } -#endif - if (sizeof(uint64_t) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(uint64_t, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(uint64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - uint64_t val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (uint64_t) -1; - } - } else { - uint64_t val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (uint64_t) -1; - val = __Pyx_PyInt_As_uint64_t(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to uint64_t"); - return (uint64_t) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to uint64_t"); - return (uint64_t) -1; -} - -/* CIntFromPy */ - static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { - const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(size_t) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (size_t) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (size_t) 0; - case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) - case 2: - if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { - return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { - return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { - return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (size_t) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(size_t) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (size_t) 0; - case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) - case -2: - if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { - return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { - return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); - } - } - break; - } -#endif - if (sizeof(size_t) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - size_t val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (size_t) -1; - } - } else { - size_t val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (size_t) -1; - val = __Pyx_PyInt_As_size_t(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to size_t"); - return (size_t) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to size_t"); - return (size_t) -1; -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - -/* CIntFromPy */ - static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(long) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (long) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { - return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { - return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { - return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (long) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(long) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) - case -2: - if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - } -#endif - if (sizeof(long) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - long val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (long) -1; - } - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (long) -1; - val = __Pyx_PyInt_As_long(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to long"); - return (long) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to long"); - return (long) -1; -} - -/* FetchCommonType */ - static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } -done: - Py_DECREF(fake_module); - return cached_type; -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; -} - -/* SwapException */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = *type; - tstate->exc_value = *value; - tstate->exc_traceback = *tb; - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} -#else -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); - PyErr_SetExcInfo(*type, *value, *tb); - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} -#endif - -/* PyCFunctionFastCall */ - #if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - PyObject *result; - int flags; - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); -} -#endif // CYTHON_FAST_PYCCALL - -/* PyFunctionFastCall */ - #if CYTHON_FAST_PYCALL -#include "frameobject.h" -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = PyThreadState_GET(); - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = f->f_localsplus; - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; -} -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && -#endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); -#endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); - } - else { - d = NULL; - nd = 0; - } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); -#else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); -#endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; -} -#endif // CPython < 3.6 -#endif // CYTHON_FAST_PYCALL - -/* PyObjectCallMethO */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyObjectCallOneArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif - -/* PyObjectCallMethod1 */ - static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { - PyObject *method, *result = NULL; - method = __Pyx_PyObject_GetAttrStr(obj, method_name); - if (unlikely(!method)) goto done; -#if CYTHON_UNPACK_METHODS - if (likely(PyMethod_Check(method))) { - PyObject *self = PyMethod_GET_SELF(method); - if (likely(self)) { - PyObject *args; - PyObject *function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyFunction_FastCall(function, args, 2); - goto done; - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyCFunction_FastCall(function, args, 2); - goto done; - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; - } - } -#endif - result = __Pyx_PyObject_CallOneArg(method, arg); -done: - Py_XDECREF(method); - return result; -} - -/* CoroutineBase */ - #include -#include -static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); -static PyObject *__Pyx_Coroutine_Close(PyObject *self); -static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); -#define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { - PyObject *et, *ev, *tb; - PyObject *value = NULL; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&et, &ev, &tb); - if (!et) { - Py_XDECREF(tb); - Py_XDECREF(ev); - Py_INCREF(Py_None); - *pvalue = Py_None; - return 0; - } - if (likely(et == PyExc_StopIteration)) { - if (!ev) { - Py_INCREF(Py_None); - value = Py_None; - } -#if PY_VERSION_HEX >= 0x030300A0 - else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { - value = ((PyStopIterationObject *)ev)->value; - Py_INCREF(value); - Py_DECREF(ev); - } -#endif - else if (unlikely(PyTuple_Check(ev))) { - if (PyTuple_GET_SIZE(ev) >= 1) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - value = PyTuple_GET_ITEM(ev, 0); - Py_INCREF(value); -#else - value = PySequence_ITEM(ev, 0); -#endif - } else { - Py_INCREF(Py_None); - value = Py_None; - } - Py_DECREF(ev); - } - else if (!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { - value = ev; - } - if (likely(value)) { - Py_XDECREF(tb); - Py_DECREF(et); - *pvalue = value; - return 0; - } - } else if (!PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - PyErr_NormalizeException(&et, &ev, &tb); - if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - Py_XDECREF(tb); - Py_DECREF(et); -#if PY_VERSION_HEX >= 0x030300A0 - value = ((PyStopIterationObject *)ev)->value; - Py_INCREF(value); - Py_DECREF(ev); -#else - { - PyObject* args = __Pyx_PyObject_GetAttrStr(ev, __pyx_n_s_args); - Py_DECREF(ev); - if (likely(args)) { - value = PySequence_GetItem(args, 0); - Py_DECREF(args); - } - if (unlikely(!value)) { - __Pyx_ErrRestore(NULL, NULL, NULL); - Py_INCREF(Py_None); - value = Py_None; - } - } -#endif - *pvalue = value; - return 0; -} -#endif -static CYTHON_INLINE -void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { - PyObject *exc_type = self->exc_type; - PyObject *exc_value = self->exc_value; - PyObject *exc_traceback = self->exc_traceback; - self->exc_type = NULL; - self->exc_value = NULL; - self->exc_traceback = NULL; - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_traceback); -} -static CYTHON_INLINE -int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) { - if (unlikely(gen->is_running)) { - PyErr_SetString(PyExc_ValueError, - "generator already executing"); - return 1; - } - return 0; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) { - PyObject *retval; - __Pyx_PyThreadState_declare - assert(!self->is_running); - if (unlikely(self->resume_label == 0)) { - if (unlikely(value && value != Py_None)) { - PyErr_SetString(PyExc_TypeError, - "can't send non-None value to a " - "just-started generator"); - return NULL; - } - } - if (unlikely(self->resume_label == -1)) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - __Pyx_PyThreadState_assign - if (value) { -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON -#else - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_XINCREF(__pyx_tstate->frame); - assert(f->f_back == NULL); - f->f_back = __pyx_tstate->frame; - } -#endif - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); - } else { - __Pyx_Coroutine_ExceptionClear(self); - } - self->is_running = 1; - retval = self->body((PyObject *) self, value); - self->is_running = 0; - if (retval) { - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON -#else - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_CLEAR(f->f_back); - } -#endif - } else { - __Pyx_Coroutine_ExceptionClear(self); - } - return retval; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) { - if (unlikely(!retval && !PyErr_Occurred())) { - PyErr_SetNone(PyExc_StopIteration); - } - return retval; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { - PyObject *ret; - PyObject *val = NULL; - __Pyx_Coroutine_Undelegate(gen); - __Pyx_PyGen_FetchStopIterationValue(&val); - ret = __Pyx_Coroutine_SendEx(gen, val); - Py_XDECREF(val); - return ret; -} -static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { - PyObject *retval; - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Coroutine_Send(yf, value); - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - ret = __Pyx_Coroutine_Send(yf, value); - } else - #endif - { - if (value == Py_None) - ret = Py_TYPE(yf)->tp_iternext(yf); - else - ret = __Pyx_PyObject_CallMethod1(yf, __pyx_n_s_send, value); - } - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - retval = __Pyx_Coroutine_FinishDelegation(gen); - } else { - retval = __Pyx_Coroutine_SendEx(gen, value); - } - return __Pyx_Coroutine_MethodReturn(retval); -} -static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { - PyObject *retval = NULL; - int err = 0; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - retval = __Pyx_Coroutine_Close(yf); - if (!retval) - return -1; - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - retval = __Pyx_Coroutine_Close(yf); - if (!retval) - return -1; - } else - #endif - { - PyObject *meth; - gen->is_running = 1; - meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_close); - if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_WriteUnraisable(yf); - } - PyErr_Clear(); - } else { - retval = PyObject_CallFunction(meth, NULL); - Py_DECREF(meth); - if (!retval) - err = -1; - } - gen->is_running = 0; - } - Py_XDECREF(retval); - return err; -} -static PyObject *__Pyx_Generator_Next(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Generator_Next(yf); - } else - #endif - ret = Py_TYPE(yf)->tp_iternext(yf); - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - return __Pyx_Coroutine_FinishDelegation(gen); - } - return __Pyx_Coroutine_SendEx(gen, Py_None); -} -static PyObject *__Pyx_Coroutine_Close(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject *retval, *raised_exception; - PyObject *yf = gen->yieldfrom; - int err = 0; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - Py_INCREF(yf); - err = __Pyx_Coroutine_CloseIter(gen, yf); - __Pyx_Coroutine_Undelegate(gen); - Py_DECREF(yf); - } - if (err == 0) - PyErr_SetNone(PyExc_GeneratorExit); - retval = __Pyx_Coroutine_SendEx(gen, NULL); - if (retval) { - Py_DECREF(retval); - PyErr_SetString(PyExc_RuntimeError, - "generator ignored GeneratorExit"); - return NULL; - } - raised_exception = PyErr_Occurred(); - if (!raised_exception - || raised_exception == PyExc_StopIteration - || raised_exception == PyExc_GeneratorExit - || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) - || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) - { - if (raised_exception) PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - return NULL; -} -static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; - PyObject *yf = gen->yieldfrom; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) - return NULL; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - Py_INCREF(yf); - if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { - int err = __Pyx_Coroutine_CloseIter(gen, yf); - Py_DECREF(yf); - __Pyx_Coroutine_Undelegate(gen); - if (err < 0) - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); - goto throw_here; - } - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else - #endif - { - PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_throw); - if (unlikely(!meth)) { - Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - gen->is_running = 0; - return NULL; - } - PyErr_Clear(); - __Pyx_Coroutine_Undelegate(gen); - gen->is_running = 0; - goto throw_here; - } - ret = PyObject_CallObject(meth, args); - Py_DECREF(meth); - } - gen->is_running = 0; - Py_DECREF(yf); - if (!ret) { - ret = __Pyx_Coroutine_FinishDelegation(gen); - } - return __Pyx_Coroutine_MethodReturn(ret); - } -throw_here: - __Pyx_Raise(typ, val, tb, NULL); - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); -} -static int __Pyx_Coroutine_traverse(PyObject *self, visitproc visit, void *arg) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - Py_VISIT(gen->closure); - Py_VISIT(gen->classobj); - Py_VISIT(gen->yieldfrom); - Py_VISIT(gen->exc_type); - Py_VISIT(gen->exc_value); - Py_VISIT(gen->exc_traceback); - return 0; -} -static int __Pyx_Coroutine_clear(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - Py_CLEAR(gen->closure); - Py_CLEAR(gen->classobj); - Py_CLEAR(gen->yieldfrom); - Py_CLEAR(gen->exc_type); - Py_CLEAR(gen->exc_value); - Py_CLEAR(gen->exc_traceback); - Py_CLEAR(gen->gi_name); - Py_CLEAR(gen->gi_qualname); - return 0; -} -static void __Pyx_Coroutine_dealloc(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject_GC_UnTrack(gen); - if (gen->gi_weakreflist != NULL) - PyObject_ClearWeakRefs(self); - if (gen->resume_label > 0) { - PyObject_GC_Track(self); -#if PY_VERSION_HEX >= 0x030400a1 - if (PyObject_CallFinalizerFromDealloc(self)) -#else - Py_TYPE(gen)->tp_del(self); - if (self->ob_refcnt > 0) -#endif - { - return; - } - PyObject_GC_UnTrack(self); - } - __Pyx_Coroutine_clear(self); - PyObject_GC_Del(gen); -} -static void __Pyx_Coroutine_del(PyObject *self) { - PyObject *res; - PyObject *error_type, *error_value, *error_traceback; - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - __Pyx_PyThreadState_declare - if (gen->resume_label <= 0) - return ; -#if PY_VERSION_HEX < 0x030400a1 - assert(self->ob_refcnt == 0); - self->ob_refcnt = 1; -#endif - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); - res = __Pyx_Coroutine_Close(self); - if (res == NULL) - PyErr_WriteUnraisable(self); - else - Py_DECREF(res); - __Pyx_ErrRestore(error_type, error_value, error_traceback); -#if PY_VERSION_HEX < 0x030400a1 - assert(self->ob_refcnt > 0); - if (--self->ob_refcnt == 0) { - return; - } - { - Py_ssize_t refcnt = self->ob_refcnt; - _Py_NewReference(self); - self->ob_refcnt = refcnt; - } -#if CYTHON_COMPILING_IN_CPYTHON - assert(PyType_IS_GC(self->ob_type) && - _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); - _Py_DEC_REFTOTAL; -#endif -#ifdef COUNT_ALLOCS - --Py_TYPE(self)->tp_frees; - --Py_TYPE(self)->tp_allocs; -#endif -#endif -} -static PyObject * -__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self) -{ - PyObject *name = self->gi_name; - if (unlikely(!name)) name = Py_None; - Py_INCREF(name); - return name; -} -static int -__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { -#else - if (unlikely(value == NULL || !PyString_Check(value))) { -#endif - PyErr_SetString(PyExc_TypeError, - "__name__ must be set to a string object"); - return -1; - } - tmp = self->gi_name; - Py_INCREF(value); - self->gi_name = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self) -{ - PyObject *name = self->gi_qualname; - if (unlikely(!name)) name = Py_None; - Py_INCREF(name); - return name; -} -static int -__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { -#else - if (unlikely(value == NULL || !PyString_Check(value))) { -#endif - PyErr_SetString(PyExc_TypeError, - "__qualname__ must be set to a string object"); - return -1; - } - tmp = self->gi_qualname; - Py_INCREF(value); - self->gi_qualname = value; - Py_XDECREF(tmp); - return 0; -} -static __pyx_CoroutineObject *__Pyx__Coroutine_New( - PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *closure, - PyObject *name, PyObject *qualname, PyObject *module_name) { - __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); - if (gen == NULL) - return NULL; - gen->body = body; - gen->closure = closure; - Py_XINCREF(closure); - gen->is_running = 0; - gen->resume_label = 0; - gen->classobj = NULL; - gen->yieldfrom = NULL; - gen->exc_type = NULL; - gen->exc_value = NULL; - gen->exc_traceback = NULL; - gen->gi_weakreflist = NULL; - Py_XINCREF(qualname); - gen->gi_qualname = qualname; - Py_XINCREF(name); - gen->gi_name = name; - Py_XINCREF(module_name); - gen->gi_modulename = module_name; - PyObject_GC_Track(gen); - return gen; -} - -/* PatchModuleWithCoroutine */ - static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code) { -#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - int result; - PyObject *globals, *result_obj; - globals = PyDict_New(); if (unlikely(!globals)) goto ignore; - result = PyDict_SetItemString(globals, "_cython_coroutine_type", - #ifdef __Pyx_Coroutine_USED - (PyObject*)__pyx_CoroutineType); - #else - Py_None); - #endif - if (unlikely(result < 0)) goto ignore; - result = PyDict_SetItemString(globals, "_cython_generator_type", - #ifdef __Pyx_Generator_USED - (PyObject*)__pyx_GeneratorType); - #else - Py_None); - #endif - if (unlikely(result < 0)) goto ignore; - if (unlikely(PyDict_SetItemString(globals, "_module", module) < 0)) goto ignore; - if (unlikely(PyDict_SetItemString(globals, "__builtins__", __pyx_b) < 0)) goto ignore; - result_obj = PyRun_String(py_code, Py_file_input, globals, globals); - if (unlikely(!result_obj)) goto ignore; - Py_DECREF(result_obj); - Py_DECREF(globals); - return module; -ignore: - Py_XDECREF(globals); - PyErr_WriteUnraisable(module); - if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, "Cython module failed to patch module with custom type", 1) < 0)) { - Py_DECREF(module); - module = NULL; - } -#else - py_code++; -#endif - return module; -} - -/* PatchGeneratorABC */ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) -static PyObject* __Pyx_patch_abc_module(PyObject *module); -static PyObject* __Pyx_patch_abc_module(PyObject *module) { - module = __Pyx_Coroutine_patch_module( - module, "" -"if _cython_generator_type is not None:\n" -" try: Generator = _module.Generator\n" -" except AttributeError: pass\n" -" else: Generator.register(_cython_generator_type)\n" -"if _cython_coroutine_type is not None:\n" -" try: Coroutine = _module.Coroutine\n" -" except AttributeError: pass\n" -" else: Coroutine.register(_cython_coroutine_type)\n" - ); - return module; -} -#endif -static int __Pyx_patch_abc(void) { -#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - static int abc_patched = 0; - if (!abc_patched) { - PyObject *module; - module = PyImport_ImportModule((PY_VERSION_HEX >= 0x03030000) ? "collections.abc" : "collections"); - if (!module) { - PyErr_WriteUnraisable(NULL); - if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, - ((PY_VERSION_HEX >= 0x03030000) ? - "Cython module failed to register with collections.abc module" : - "Cython module failed to register with collections module"), 1) < 0)) { - return -1; - } - } else { - module = __Pyx_patch_abc_module(module); - abc_patched = 1; - if (unlikely(!module)) - return -1; - Py_DECREF(module); - } - module = PyImport_ImportModule("backports_abc"); - if (module) { - module = __Pyx_patch_abc_module(module); - Py_XDECREF(module); - } - if (!module) { - PyErr_Clear(); - } - } -#else - if (0) __Pyx_Coroutine_patch_module(NULL, NULL); -#endif - return 0; -} - -/* Generator */ - static PyMethodDef __pyx_Generator_methods[] = { - {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, - (char*) PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, - {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, - (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, - {"close", (PyCFunction) __Pyx_Coroutine_Close, METH_NOARGS, - (char*) PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, - {0, 0, 0, 0} -}; -static PyMemberDef __pyx_Generator_memberlist[] = { - {(char *) "gi_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, - {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, - (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, - {0, 0, 0, 0, 0} -}; -static PyGetSetDef __pyx_Generator_getsets[] = { - {(char *) "__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, - (char*) PyDoc_STR("name of the generator"), 0}, - {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, - (char*) PyDoc_STR("qualified name of the generator"), 0}, - {0, 0, 0, 0, 0} -}; -static PyTypeObject __pyx_GeneratorType_type = { - PyVarObject_HEAD_INIT(0, 0) - "generator", - sizeof(__pyx_CoroutineObject), - 0, - (destructor) __Pyx_Coroutine_dealloc, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, - 0, - (traverseproc) __Pyx_Coroutine_traverse, - 0, - 0, - offsetof(__pyx_CoroutineObject, gi_weakreflist), - 0, - (iternextfunc) __Pyx_Generator_Next, - __pyx_Generator_methods, - __pyx_Generator_memberlist, - __pyx_Generator_getsets, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, -#if PY_VERSION_HEX >= 0x030400a1 - 0, -#else - __Pyx_Coroutine_del, -#endif - 0, -#if PY_VERSION_HEX >= 0x030400a1 - __Pyx_Coroutine_del, -#endif -}; -static int __pyx_Generator_init(void) { - __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; - __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; - __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); - if (unlikely(!__pyx_GeneratorType)) { - return -1; - } - return 0; -} - -/* CheckBinaryVersion */ - static int __Pyx_check_binary_version(void) { - char ctversion[4], rtversion[4]; - PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); - PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); - if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { - char message[200]; - PyOS_snprintf(message, sizeof(message), - "compiletime version %s of module '%.100s' " - "does not match runtime version %s", - ctversion, __Pyx_MODULE_NAME, rtversion); - return PyErr_WarnEx(NULL, message, 1); - } - return 0; -} - -/* InitStrings */ - static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { - return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { - Py_ssize_t ignore; - return __Pyx_PyObject_AsStringAndSize(o, &ignore); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { -#if PY_VERSION_HEX < 0x03030000 - char* defenc_c; - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - PyUnicode_AsASCIIString(o); - return NULL; - } - } - } -#endif - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; -#else - if (__Pyx_PyUnicode_READY(o) == -1) return NULL; -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (PyUnicode_IS_ASCII(o)) { - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - PyUnicode_AsASCIIString(o); - return NULL; - } -#else - return PyUnicode_AsUTF8AndSize(o, length); -#endif -#endif - } else -#endif -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) - if (PyByteArray_Check(o)) { - *length = PyByteArray_GET_SIZE(o); - return PyByteArray_AS_STRING(o); - } else -#endif - { - char* result; - int r = PyBytes_AsStringAndSize(o, &result, length); - if (unlikely(r < 0)) { - return NULL; - } else { - return result; - } - } -} -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - int is_true = x == Py_True; - if (is_true | (x == Py_False) | (x == Py_None)) return is_true; - else return PyObject_IsTrue(x); -} -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { -#if CYTHON_USE_TYPE_SLOTS - PyNumberMethods *m; -#endif - const char *name = NULL; - PyObject *res = NULL; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return __Pyx_NewRef(x); -#if CYTHON_USE_TYPE_SLOTS - m = Py_TYPE(x)->tp_as_number; - #if PY_MAJOR_VERSION < 3 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } - #else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } - #endif -#else - res = PyNumber_Int(x); -#endif - if (res) { -#if PY_MAJOR_VERSION < 3 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject *x; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(b))) { - if (sizeof(Py_ssize_t) >= sizeof(long)) - return PyInt_AS_LONG(b); - else - return PyInt_AsSsize_t(x); - } -#endif - if (likely(PyLong_CheckExact(b))) { - #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)b)->ob_digit; - const Py_ssize_t size = Py_SIZE(b); - if (likely(__Pyx_sst_abs(size) <= 1)) { - ival = likely(size) ? digits[0] : 0; - if (size == -1) ival = -ival; - return ival; - } else { - switch (size) { - case 2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - } - } - #endif - return PyLong_AsSsize_t(b); - } - x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { - return PyInt_FromSize_t(ival); -} - - -#endif /* Py_PYTHON_H */ diff --git a/khmer/_oxli/traversal.cpp b/khmer/_oxli/traversal.cpp deleted file mode 100644 index b859552da1..0000000000 --- a/khmer/_oxli/traversal.cpp +++ /dev/null @@ -1,4360 +0,0 @@ -/* Generated by Cython 0.25.1 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) - #error Cython requires Python 2.6+ or Python 3.2+. -#else -#define CYTHON_ABI "0_25_1" -#include -#ifndef offsetof - #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#endif -#ifndef DL_IMPORT - #define DL_IMPORT(t) t -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#ifndef HAVE_LONG_LONG - #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) - #define HAVE_LONG_LONG - #endif -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef Py_HUGE_VAL - #define Py_HUGE_VAL HUGE_VAL -#endif -#ifdef PYPY_VERSION - #define CYTHON_COMPILING_IN_PYPY 1 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #undef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 0 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #undef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #undef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 1 - #undef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 0 - #undef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 0 - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 -#elif defined(PYSTON_VERSION) - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 1 - #define CYTHON_COMPILING_IN_CPYTHON 0 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #undef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 0 - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #undef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 0 -#else - #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 - #define CYTHON_COMPILING_IN_CPYTHON 1 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif - #if PY_MAJOR_VERSION < 3 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 - #elif !defined(CYTHON_USE_ASYNC_SLOTS) - #define CYTHON_USE_ASYNC_SLOTS 1 - #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) - #define CYTHON_USE_PYLONG_INTERNALS 1 - #endif - #ifndef CYTHON_USE_PYLIST_INTERNALS - #define CYTHON_USE_PYLIST_INTERNALS 1 - #endif - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 - #endif - #if PY_VERSION_HEX < 0x030300F0 - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 - #elif !defined(CYTHON_USE_UNICODE_WRITER) - #define CYTHON_USE_UNICODE_WRITER 1 - #endif - #ifndef CYTHON_AVOID_BORROWED_REFS - #define CYTHON_AVOID_BORROWED_REFS 0 - #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif - #ifndef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 1 - #endif - #ifndef CYTHON_FAST_PYCALL - #define CYTHON_FAST_PYCALL 1 - #endif -#endif -#if !defined(CYTHON_FAST_PYCCALL) -#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) -#endif -#if CYTHON_USE_PYLONG_INTERNALS - #include "longintrepr.h" - #undef SHIFT - #undef BASE - #undef MASK -#endif -#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) - #define Py_OptimizeFlag 0 -#endif -#define __PYX_BUILD_PY_SSIZE_T "n" -#define CYTHON_FORMAT_SSIZE_T "z" -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyClass_Type -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ - PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyType_Type -#endif -#ifndef Py_TPFLAGS_CHECKTYPES - #define Py_TPFLAGS_CHECKTYPES 0 -#endif -#ifndef Py_TPFLAGS_HAVE_INDEX - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#ifndef Py_TPFLAGS_HAVE_NEWBUFFER - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#ifndef Py_TPFLAGS_HAVE_FINALIZE - #define Py_TPFLAGS_HAVE_FINALIZE 0 -#endif -#ifndef METH_FASTCALL - #define METH_FASTCALL 0x80 - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); -#else - #define __Pyx_PyCFunctionFast _PyCFunctionFast -#endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) -#else -#define __Pyx_PyFastCFunction_Check(func) 0 -#endif -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) - #define CYTHON_PEP393_ENABLED 1 - #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ - 0 : _PyUnicode_Ready((PyObject *)(op))) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) - #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) - #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) -#else - #define CYTHON_PEP393_ENABLED 0 - #define PyUnicode_1BYTE_KIND 1 - #define PyUnicode_2BYTE_KIND 2 - #define PyUnicode_4BYTE_KIND 4 - #define __Pyx_PyUnicode_READY(op) (0) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) - #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) - #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) - #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) -#endif -#if CYTHON_COMPILING_IN_PYPY - #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) -#else - #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ - PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) - #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) - #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) - #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) -#endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) - #define PyObject_Malloc(s) PyMem_Malloc(s) - #define PyObject_Free(p) PyMem_Free(p) - #define PyObject_Realloc(p) PyMem_Realloc(p) -#endif -#if CYTHON_COMPILING_IN_PYSTON - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) -#else - #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) -#endif -#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) -#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) -#else - #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) -#endif -#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) - #define PyObject_ASCII(o) PyObject_Repr(o) -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyStringObject PyUnicodeObject - #define PyString_Type PyUnicode_Type - #define PyString_Check PyUnicode_Check - #define PyString_CheckExact PyUnicode_CheckExact -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) - #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) -#else - #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) - #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) -#endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) -#endif -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) -#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) -#if PY_MAJOR_VERSION >= 3 - #define PyIntObject PyLongObject - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define PyNumber_Int PyNumber_Long -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBoolObject PyLongObject -#endif -#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY - #ifndef PyUnicode_InternFromString - #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) - #endif -#endif -#if PY_VERSION_HEX < 0x030200A4 - typedef long Py_hash_t; - #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong -#else - #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) -#endif -#if CYTHON_USE_ASYNC_SLOTS - #if PY_VERSION_HEX >= 0x030500B1 - #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods - #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) - #else - typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; - } __Pyx_PyAsyncMethodsStruct; - #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) - #endif -#else - #define __Pyx_PyType_AsAsync(obj) NULL -#endif -#ifndef CYTHON_RESTRICT - #if defined(__GNUC__) - #define CYTHON_RESTRICT __restrict__ - #elif defined(_MSC_VER) && _MSC_VER >= 1400 - #define CYTHON_RESTRICT __restrict - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_RESTRICT restrict - #else - #define CYTHON_RESTRICT - #endif -#endif -#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) - -#ifndef __cplusplus - #error "Cython files generated with the C++ option must be compiled with a C++ compiler." -#endif -#ifndef CYTHON_INLINE - #define CYTHON_INLINE inline -#endif -template -void __Pyx_call_destructor(T& x) { - x.~T(); -} -template -class __Pyx_FakeReference { - public: - __Pyx_FakeReference() : ptr(NULL) { } - __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } - T *operator->() { return ptr; } - operator T&() { return *ptr; } - template bool operator ==(U other) { return *ptr == other; }; - template bool operator !=(U other) { return *ptr != other; }; - private: - T *ptr; -}; - -#if defined(WIN32) || defined(MS_WINDOWS) - #define _USE_MATH_DEFINES -#endif -#include -#ifdef NAN -#define __PYX_NAN() ((float) NAN) -#else -static CYTHON_INLINE float __PYX_NAN() { - float value; - memset(&value, 0xFF, sizeof(value)); - return value; -} -#endif -#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) -#define __Pyx_truncl trunc -#else -#define __Pyx_truncl truncl -#endif - - -#define __PYX_ERR(f_index, lineno, Ln_error) \ -{ \ - __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ -} - -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) -#endif - -#ifndef __PYX_EXTERN_C - #ifdef __cplusplus - #define __PYX_EXTERN_C extern "C" - #else - #define __PYX_EXTERN_C extern - #endif -#endif - -#define __PYX_HAVE__khmer___oxli__traversal -#define __PYX_HAVE_API__khmer___oxli__traversal -#include -#include "ios" -#include "new" -#include "stdexcept" -#include "typeinfo" -#include -#include -#include -#include -#include -#include -#include -#include -#include "khmer.hh" -#include "kmer_hash.hh" -#include "_khmer.hh" -#include "hashtable.hh" -#include "traversal.hh" -#include "partitioning.hh" -#ifdef _OPENMP -#include -#endif /* _OPENMP */ - -#ifdef PYREX_WITHOUT_ASSERTIONS -#define CYTHON_WITHOUT_ASSERTIONS -#endif - -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif -typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; - const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; - -#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 -#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 -#define __PYX_DEFAULT_STRING_ENCODING "" -#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString -#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#define __Pyx_uchar_cast(c) ((unsigned char)c) -#define __Pyx_long_cast(x) ((long)x) -#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ - (sizeof(type) < sizeof(Py_ssize_t)) ||\ - (sizeof(type) > sizeof(Py_ssize_t) &&\ - likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX) &&\ - (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ - v == (type)PY_SSIZE_T_MIN))) ||\ - (sizeof(type) == sizeof(Py_ssize_t) &&\ - (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ - v == (type)PY_SSIZE_T_MAX))) ) -#if defined (__cplusplus) && __cplusplus >= 201103L - #include - #define __Pyx_sst_abs(value) std::abs(value) -#elif SIZEOF_INT >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) abs(value) -#elif SIZEOF_LONG >= SIZEOF_SIZE_T - #define __Pyx_sst_abs(value) labs(value) -#elif defined (_MSC_VER) && defined (_M_X64) - #define __Pyx_sst_abs(value) _abs64(value) -#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define __Pyx_sst_abs(value) llabs(value) -#elif defined (__GNUC__) - #define __Pyx_sst_abs(value) __builtin_llabs(value) -#else - #define __Pyx_sst_abs(value) ((value<0) ? -value : value) -#endif -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); -#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) -#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); -#if PY_MAJOR_VERSION < 3 - #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#else - #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize -#endif -#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) -#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) -#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) -#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) -#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -#if PY_MAJOR_VERSION < 3 -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) -{ - const Py_UNICODE *u_end = u; - while (*u_end++) ; - return (size_t)(u_end - u - 1); -} -#else -#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen -#endif -#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) -#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode -#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode -#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) -#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) -#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -#if CYTHON_ASSUME_SAFE_MACROS -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) -#else -#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) -#endif -#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) -#if PY_MAJOR_VERSION >= 3 -#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) -#else -#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) -#endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII -static int __Pyx_sys_getdefaultencoding_not_ascii; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - PyObject* ascii_chars_u = NULL; - PyObject* ascii_chars_b = NULL; - const char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - if (strcmp(default_encoding_c, "ascii") == 0) { - __Pyx_sys_getdefaultencoding_not_ascii = 0; - } else { - char ascii_chars[128]; - int c; - for (c = 0; c < 128; c++) { - ascii_chars[c] = c; - } - __Pyx_sys_getdefaultencoding_not_ascii = 1; - ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); - if (!ascii_chars_u) goto bad; - ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); - if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { - PyErr_Format( - PyExc_ValueError, - "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", - default_encoding_c); - goto bad; - } - Py_DECREF(ascii_chars_u); - Py_DECREF(ascii_chars_b); - } - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - Py_XDECREF(ascii_chars_u); - Py_XDECREF(ascii_chars_b); - return -1; -} -#endif -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) -#else -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT -static char* __PYX_DEFAULT_STRING_ENCODING; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); - if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; - strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - return -1; -} -#endif -#endif - - -/* Test for GCC > 2.95 */ -#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) - #define likely(x) __builtin_expect(!!(x), 1) - #define unlikely(x) __builtin_expect(!!(x), 0) -#else /* !__GNUC__ or GCC < 2.95 */ - #define likely(x) (x) - #define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_d; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static PyObject *__pyx_empty_bytes; -static PyObject *__pyx_empty_unicode; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; - - -static const char *__pyx_f[] = { - "khmer/_oxli/traversal.pyx", - "khmer/_oxli/hashing.pxd", -}; - -/*--- Type declarations ---*/ -struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer; -struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser; -struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors; - -/* "hashing.pxd":4 - * from _oxli cimport CpKmer, HashIntoType, WordLength - * - * cdef class Kmer: # <<<<<<<<<<<<<< - * cdef shared_ptr[CpKmer] _this - * cdef readonly str kmer - */ -struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer { - PyObject_HEAD - struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer *__pyx_vtab; - std::shared_ptr _this; - PyObject *kmer; -}; - - -/* "khmer/_oxli/traversal.pxd":5 - * from _oxli cimport CpTraverser, CpHashtable - * - * cdef class Traverser: # <<<<<<<<<<<<<< - * cdef unique_ptr[CpTraverser] _this - * cdef CpHashtable * _graph_ptr - */ -struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser { - PyObject_HEAD - std::unique_ptr _this; - khmer::Hashtable *_graph_ptr; -}; - - -/* "khmer/_oxli/traversal.pyx":30 - * ''' - * - * def right_neighbors(self, hashing.Kmer kmer): # <<<<<<<<<<<<<< - * cdef KmerQueue kmer_q - * cdef CpKmer neighbor - */ -struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors { - PyObject_HEAD - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer; - khmer::KmerQueue __pyx_v_kmer_q; - khmer::Kmer __pyx_v_neighbor; - khmer::Kmer *__pyx_v_ptr; - struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self; -}; - - - -/* "hashing.pxd":4 - * from _oxli cimport CpKmer, HashIntoType, WordLength - * - * cdef class Kmer: # <<<<<<<<<<<<<< - * cdef shared_ptr[CpKmer] _this - * cdef readonly str kmer - */ - -struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer { - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*wrap)(khmer::Kmer *, khmer::WordLength); - struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *(*create)(khmer::HashIntoType, khmer::WordLength); -}; -static struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer *__pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer; - -/* --- Runtime support code (head) --- */ -/* Refnanny.proto */ -#ifndef CYTHON_REFNANNY - #define CYTHON_REFNANNY 0 -#endif -#if CYTHON_REFNANNY - typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); - void (*FinishContext)(void**); - } __Pyx_RefNannyAPIStruct; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); - #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; -#ifdef WITH_THREAD - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - if (acquire_gil) {\ - PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - PyGILState_Release(__pyx_gilstate_save);\ - } else {\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ - } -#else - #define __Pyx_RefNannySetupContext(name, acquire_gil)\ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) -#endif - #define __Pyx_RefNannyFinishContext()\ - __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) -#else - #define __Pyx_RefNannyDeclarations - #define __Pyx_RefNannySetupContext(name, acquire_gil) - #define __Pyx_RefNannyFinishContext() - #define __Pyx_INCREF(r) Py_INCREF(r) - #define __Pyx_DECREF(r) Py_DECREF(r) - #define __Pyx_GOTREF(r) - #define __Pyx_GIVEREF(r) - #define __Pyx_XINCREF(r) Py_XINCREF(r) - #define __Pyx_XDECREF(r) Py_XDECREF(r) - #define __Pyx_XGOTREF(r) - #define __Pyx_XGIVEREF(r) -#endif -#define __Pyx_XDECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_XDECREF(tmp);\ - } while (0) -#define __Pyx_DECREF_SET(r, v) do {\ - PyObject *tmp = (PyObject *) r;\ - r = v; __Pyx_DECREF(tmp);\ - } while (0) -#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) -#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) - -/* PyObjectGetAttrStr.proto */ -#if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { - PyTypeObject* tp = Py_TYPE(obj); - if (likely(tp->tp_getattro)) - return tp->tp_getattro(obj, attr_name); -#if PY_MAJOR_VERSION < 3 - if (likely(tp->tp_getattr)) - return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); -#endif - return PyObject_GetAttr(obj, attr_name); -} -#else -#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) -#endif - -/* GetBuiltinName.proto */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name); - -/* RaiseDoubleKeywords.proto */ -static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); - -/* ParseKeywords.proto */ -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ - const char* function_name); - -/* RaiseArgTupleInvalid.proto */ -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); - -/* GetModuleGlobalName.proto */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); - -/* PyObjectCall.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); -#else -#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) -#endif - -/* PyThreadStateGet.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; -#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); -#else -#define __Pyx_PyThreadState_declare -#define __Pyx_PyThreadState_assign -#endif - -/* PyErrFetchRestore.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) -#endif - -/* RaiseException.proto */ -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); - -/* ArgTypeTest.proto */ -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact); - -/* IncludeStringH.proto */ -#include - -/* GetVTable.proto */ -static void* __Pyx_GetVtable(PyObject *dict); - -/* Import.proto */ -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); - -/* ImportFrom.proto */ -static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); - -/* CodeObjectCache.proto */ -typedef struct { - PyCodeObject* code_object; - int code_line; -} __Pyx_CodeObjectCacheEntry; -struct __Pyx_CodeObjectCache { - int count; - int max_count; - __Pyx_CodeObjectCacheEntry* entries; -}; -static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); -static PyCodeObject *__pyx_find_code_object(int code_line); -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); - -/* AddTraceback.proto */ -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename); - -/* None.proto */ -#include - -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - -/* CIntFromPy.proto */ -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); - -/* CIntFromPy.proto */ -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); - -/* FetchCommonType.proto */ -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); - -/* SwapException.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); -#endif - -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/* PyFunctionFastCall.proto */ -#if CYTHON_FAST_PYCALL -#define __Pyx_PyFunction_FastCall(func, args, nargs)\ - __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) -#endif -#endif - -/* PyObjectCallMethO.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - -/* PyObjectCallMethod1.proto */ -static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); - -/* CoroutineBase.proto */ -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); -typedef struct { - PyObject_HEAD - __pyx_coroutine_body_t body; - PyObject *closure; - PyObject *exc_type; - PyObject *exc_value; - PyObject *exc_traceback; - PyObject *gi_weakreflist; - PyObject *classobj; - PyObject *yieldfrom; - PyObject *gi_name; - PyObject *gi_qualname; - PyObject *gi_modulename; - int resume_label; - char is_running; -} __pyx_CoroutineObject; -static __pyx_CoroutineObject *__Pyx__Coroutine_New( - PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *closure, - PyObject *name, PyObject *qualname, PyObject *module_name); -static int __Pyx_Coroutine_clear(PyObject *self); -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); -#else -#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) -#endif - -/* PatchModuleWithCoroutine.proto */ -static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code); - -/* PatchGeneratorABC.proto */ -static int __Pyx_patch_abc(void); - -/* Generator.proto */ -#define __Pyx_Generator_USED -static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) -#define __Pyx_Generator_New(body, closure, name, qualname, module_name)\ - __Pyx__Coroutine_New(__pyx_GeneratorType, body, closure, name, qualname, module_name) -static PyObject *__Pyx_Generator_Next(PyObject *self); -static int __pyx_Generator_init(void); - -/* CheckBinaryVersion.proto */ -static int __Pyx_check_binary_version(void); - -/* PyIdentifierFromString.proto */ -#if !defined(__Pyx_PyIdentifier_FromString) -#if PY_MAJOR_VERSION < 3 - #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s) -#else - #define __Pyx_PyIdentifier_FromString(s) PyUnicode_FromString(s) -#endif -#endif - -/* ModuleImport.proto */ -static PyObject *__Pyx_ImportModule(const char *name); - -/* TypeImport.proto */ -static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict); - -/* InitStrings.proto */ -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); - - -/* Module declarations from 'libcpp' */ - -/* Module declarations from 'libcpp.memory' */ - -/* Module declarations from 'libc.string' */ - -/* Module declarations from 'libcpp.string' */ - -/* Module declarations from 'libcpp.vector' */ - -/* Module declarations from 'libcpp.utility' */ - -/* Module declarations from 'libcpp.map' */ - -/* Module declarations from 'libcpp.set' */ - -/* Module declarations from 'libcpp.queue' */ - -/* Module declarations from 'libc.stdint' */ - -/* Module declarations from 'khmer._oxli._oxli' */ - -/* Module declarations from 'khmer._oxli.hashing' */ -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_7hashing_Kmer = 0; - -/* Module declarations from 'khmer._oxli.traversal' */ -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_9traversal_Traverser = 0; -static PyTypeObject *__pyx_ptype_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors = 0; -#define __Pyx_MODULE_NAME "khmer._oxli.traversal" -int __pyx_module_is_main_khmer___oxli__traversal = 0; - -/* Implementation of 'khmer._oxli.traversal' */ -static PyObject *__pyx_builtin_ValueError; -static const char __pyx_k_args[] = "args"; -static const char __pyx_k_main[] = "__main__"; -static const char __pyx_k_send[] = "send"; -static const char __pyx_k_test[] = "__test__"; -static const char __pyx_k_close[] = "close"; -static const char __pyx_k_graph[] = "graph"; -static const char __pyx_k_khmer[] = "_khmer"; -static const char __pyx_k_throw[] = "throw"; -static const char __pyx_k_import[] = "__import__"; -static const char __pyx_k_hashing[] = "hashing"; -static const char __pyx_k_Nodegraph[] = "Nodegraph"; -static const char __pyx_k_Countgraph[] = "Countgraph"; -static const char __pyx_k_ValueError[] = "ValueError"; -static const char __pyx_k_pyx_vtable[] = "__pyx_vtable__"; -static const char __pyx_k_right_neighbors[] = "right_neighbors"; -static const char __pyx_k_khmer__oxli_traversal[] = "khmer._oxli.traversal"; -static const char __pyx_k_Traverser_right_neighbors[] = "Traverser.right_neighbors"; -static const char __pyx_k_Must_take_an_object_with_Hashtab[] = "Must take an object with Hashtable *"; -static PyObject *__pyx_n_s_Countgraph; -static PyObject *__pyx_kp_s_Must_take_an_object_with_Hashtab; -static PyObject *__pyx_n_s_Nodegraph; -static PyObject *__pyx_n_s_Traverser_right_neighbors; -static PyObject *__pyx_n_s_ValueError; -static PyObject *__pyx_n_s_args; -static PyObject *__pyx_n_s_close; -static PyObject *__pyx_n_s_graph; -static PyObject *__pyx_n_s_hashing; -static PyObject *__pyx_n_s_import; -static PyObject *__pyx_n_s_khmer; -static PyObject *__pyx_n_s_khmer__oxli_traversal; -static PyObject *__pyx_n_s_main; -static PyObject *__pyx_n_s_pyx_vtable; -static PyObject *__pyx_n_s_right_neighbors; -static PyObject *__pyx_n_s_send; -static PyObject *__pyx_n_s_test; -static PyObject *__pyx_n_s_throw; -static int __pyx_pf_5khmer_5_oxli_9traversal_9Traverser___cinit__(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self, PyObject *__pyx_v_graph); /* proto */ -static PyObject *__pyx_pf_5khmer_5_oxli_9traversal_9Traverser_2right_neighbors(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer); /* proto */ -static PyObject *__pyx_tp_new_5khmer_5_oxli_9traversal_Traverser(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tuple_; - -/* "khmer/_oxli/traversal.pyx":13 - * cdef class Traverser: - * - * def __cinit__(self, graph): # <<<<<<<<<<<<<< - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') - */ - -/* Python wrapper */ -static int __pyx_pw_5khmer_5_oxli_9traversal_9Traverser_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5khmer_5_oxli_9traversal_9Traverser_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_graph = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_graph,0}; - PyObject* values[1] = {0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_graph)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 13, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - } - __pyx_v_graph = values[0]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 13, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("khmer._oxli.traversal.Traverser.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5khmer_5_oxli_9traversal_9Traverser___cinit__(((struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)__pyx_v_self), __pyx_v_graph); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5khmer_5_oxli_9traversal_9Traverser___cinit__(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self, PyObject *__pyx_v_graph) { - khmer::khmer_KHashtable_Object *__pyx_v_ptr; - int __pyx_r; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - int __pyx_t_3; - int __pyx_t_4; - khmer::Hashtable *__pyx_t_5; - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "khmer/_oxli/traversal.pyx":14 - * - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< - * raise ValueError('Must take an object with Hashtable *') - * - */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Countgraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 14, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_3 == -1)) __PYX_ERR(0, 14, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_4 = (__pyx_t_3 != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_1 = __pyx_t_4; - goto __pyx_L4_bool_binop_done; - } - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 14, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = PyObject_IsInstance(__pyx_v_graph, __pyx_t_2); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 14, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = (__pyx_t_4 != 0); - __pyx_t_1 = __pyx_t_3; - __pyx_L4_bool_binop_done:; - __pyx_t_3 = ((!__pyx_t_1) != 0); - if (__pyx_t_3) { - - /* "khmer/_oxli/traversal.pyx":15 - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< - * - * cdef CPyHashtable_Object* ptr = graph - */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(0, 15, __pyx_L1_error) - - /* "khmer/_oxli/traversal.pyx":14 - * - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): # <<<<<<<<<<<<<< - * raise ValueError('Must take an object with Hashtable *') - * - */ - } - - /* "khmer/_oxli/traversal.pyx":17 - * raise ValueError('Must take an object with Hashtable *') - * - * cdef CPyHashtable_Object* ptr = graph # <<<<<<<<<<<<<< - * self._graph_ptr = deref(ptr).hashtable - * - */ - __pyx_v_ptr = ((khmer::khmer_KHashtable_Object *)__pyx_v_graph); - - /* "khmer/_oxli/traversal.pyx":18 - * - * cdef CPyHashtable_Object* ptr = graph - * self._graph_ptr = deref(ptr).hashtable # <<<<<<<<<<<<<< - * - * self._this.reset(new CpTraverser(self._graph_ptr)) - */ - __pyx_t_5 = (*__pyx_v_ptr).hashtable; - __pyx_v_self->_graph_ptr = __pyx_t_5; - - /* "khmer/_oxli/traversal.pyx":20 - * self._graph_ptr = deref(ptr).hashtable - * - * self._this.reset(new CpTraverser(self._graph_ptr)) # <<<<<<<<<<<<<< - * ''' - * def left_neighbors(self, Kmer kmer): - */ - __pyx_v_self->_this.reset(new khmer::Traverser(__pyx_v_self->_graph_ptr)); - - /* "khmer/_oxli/traversal.pyx":13 - * cdef class Traverser: - * - * def __cinit__(self, graph): # <<<<<<<<<<<<<< - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("khmer._oxli.traversal.Traverser.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5khmer_5_oxli_9traversal_9Traverser_4generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ - -/* "khmer/_oxli/traversal.pyx":30 - * ''' - * - * def right_neighbors(self, hashing.Kmer kmer): # <<<<<<<<<<<<<< - * cdef KmerQueue kmer_q - * cdef CpKmer neighbor - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5khmer_5_oxli_9traversal_9Traverser_3right_neighbors(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer); /*proto*/ -static PyObject *__pyx_pw_5khmer_5_oxli_9traversal_9Traverser_3right_neighbors(PyObject *__pyx_v_self, PyObject *__pyx_v_kmer) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("right_neighbors (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_kmer), __pyx_ptype_5khmer_5_oxli_7hashing_Kmer, 1, "kmer", 0))) __PYX_ERR(0, 30, __pyx_L1_error) - __pyx_r = __pyx_pf_5khmer_5_oxli_9traversal_9Traverser_2right_neighbors(((struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)__pyx_v_self), ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)__pyx_v_kmer)); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5khmer_5_oxli_9traversal_9Traverser_2right_neighbors(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *__pyx_v_self, struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *__pyx_v_kmer) { - struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *__pyx_cur_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("right_neighbors", 0); - __pyx_cur_scope = (struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)__pyx_tp_new_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(__pyx_ptype_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(0, 30, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __pyx_cur_scope->__pyx_v_kmer = __pyx_v_kmer; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_kmer); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_kmer); - { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5khmer_5_oxli_9traversal_9Traverser_4generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_right_neighbors, __pyx_n_s_Traverser_right_neighbors, __pyx_n_s_khmer__oxli_traversal); if (unlikely(!gen)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("khmer._oxli.traversal.Traverser.right_neighbors", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_gb_5khmer_5_oxli_9traversal_9Traverser_4generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *__pyx_cur_scope = ((struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)__pyx_generator->closure); - PyObject *__pyx_r = NULL; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L6_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 30, __pyx_L1_error) - - /* "khmer/_oxli/traversal.pyx":34 - * cdef CpKmer neighbor - * cdef CpKmer* ptr - * deref(self._this).traverse_right(deref(kmer._this), kmer_q) # <<<<<<<<<<<<<< - * while(kmer_q.empty() == 0): - * neighbor = kmer_q.back() - */ - (*__pyx_cur_scope->__pyx_v_self->_this).traverse_right((*__pyx_cur_scope->__pyx_v_kmer->_this), __pyx_cur_scope->__pyx_v_kmer_q); - - /* "khmer/_oxli/traversal.pyx":35 - * cdef CpKmer* ptr - * deref(self._this).traverse_right(deref(kmer._this), kmer_q) - * while(kmer_q.empty() == 0): # <<<<<<<<<<<<<< - * neighbor = kmer_q.back() - * ptr = new CpKmer(neighbor) - */ - while (1) { - __pyx_t_1 = ((__pyx_cur_scope->__pyx_v_kmer_q.empty() == 0) != 0); - if (!__pyx_t_1) break; - - /* "khmer/_oxli/traversal.pyx":36 - * deref(self._this).traverse_right(deref(kmer._this), kmer_q) - * while(kmer_q.empty() == 0): - * neighbor = kmer_q.back() # <<<<<<<<<<<<<< - * ptr = new CpKmer(neighbor) - * yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) - */ - __pyx_cur_scope->__pyx_v_neighbor = __pyx_cur_scope->__pyx_v_kmer_q.back(); - - /* "khmer/_oxli/traversal.pyx":37 - * while(kmer_q.empty() == 0): - * neighbor = kmer_q.back() - * ptr = new CpKmer(neighbor) # <<<<<<<<<<<<<< - * yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) - * kmer_q.pop() - */ - __pyx_cur_scope->__pyx_v_ptr = new khmer::Kmer(__pyx_cur_scope->__pyx_v_neighbor); - - /* "khmer/_oxli/traversal.pyx":38 - * neighbor = kmer_q.back() - * ptr = new CpKmer(neighbor) - * yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) # <<<<<<<<<<<<<< - * kmer_q.pop() - */ - __pyx_t_2 = ((PyObject *)__pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer->wrap(__pyx_cur_scope->__pyx_v_ptr, (*__pyx_cur_scope->__pyx_v_self->_graph_ptr).ksize())); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 38, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L6_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 38, __pyx_L1_error) - - /* "khmer/_oxli/traversal.pyx":39 - * ptr = new CpKmer(neighbor) - * yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) - * kmer_q.pop() # <<<<<<<<<<<<<< - */ - __pyx_cur_scope->__pyx_v_kmer_q.pop(); - } - if (1); else __pyx_cur_scope = __pyx_cur_scope; - - /* "khmer/_oxli/traversal.pyx":30 - * ''' - * - * def right_neighbors(self, hashing.Kmer kmer): # <<<<<<<<<<<<<< - * cdef KmerQueue kmer_q - * cdef CpKmer neighbor - */ - - /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("right_neighbors", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_XDECREF(__pyx_r); __pyx_r = 0; - __pyx_generator->resume_label = -1; - __Pyx_Coroutine_clear((PyObject*)__pyx_generator); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_tp_new_5khmer_5_oxli_9traversal_Traverser(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)o); - new((void*)&(p->_this)) std::unique_ptr (); - if (unlikely(__pyx_pw_5khmer_5_oxli_9traversal_9Traverser_1__cinit__(o, a, k) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_9traversal_Traverser(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *p = (struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - __Pyx_call_destructor(p->_this); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyMethodDef __pyx_methods_5khmer_5_oxli_9traversal_Traverser[] = { - {"right_neighbors", (PyCFunction)__pyx_pw_5khmer_5_oxli_9traversal_9Traverser_3right_neighbors, METH_O, 0}, - {0, 0, 0, 0} -}; - -static PyTypeObject __pyx_type_5khmer_5_oxli_9traversal_Traverser = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.traversal.Traverser", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_9traversal_Traverser, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5khmer_5_oxli_9traversal_Traverser, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_9traversal_Traverser, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *__pyx_freelist_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors[8]; -static int __pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors = 0; - -static PyObject *__pyx_tp_new_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *p; - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors)))) { - o = (PyObject*)__pyx_freelist_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors[--__pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors]; - memset(o, 0, sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors)); - (void) PyObject_INIT(o, t); - PyObject_GC_Track(o); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - p = ((struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o); - new((void*)&(p->__pyx_v_kmer_q)) khmer::KmerQueue(); - new((void*)&(p->__pyx_v_neighbor)) khmer::Kmer(); - return o; -} - -static void __pyx_tp_dealloc_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyObject *o) { - struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *p = (struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o; - PyObject_GC_UnTrack(o); - __Pyx_call_destructor(p->__pyx_v_kmer_q); - __Pyx_call_destructor(p->__pyx_v_neighbor); - Py_CLEAR(p->__pyx_v_kmer); - Py_CLEAR(p->__pyx_v_self); - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors)))) { - __pyx_freelist_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors[__pyx_freecount_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors++] = ((struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static int __pyx_tp_traverse_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *p = (struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o; - if (p->__pyx_v_kmer) { - e = (*v)(((PyObject*)p->__pyx_v_kmer), a); if (e) return e; - } - if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *p = (struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors *)o; - tmp = ((PyObject*)p->__pyx_v_kmer); - p->__pyx_v_kmer = ((struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5khmer_5_oxli_9traversal_Traverser *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyTypeObject __pyx_type_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors = { - PyVarObject_HEAD_INIT(0, 0) - "khmer._oxli.traversal.__pyx_scope_struct__right_neighbors", /*tp_name*/ - sizeof(struct __pyx_obj_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #endif - #if PY_MAJOR_VERSION >= 3 - 0, /*tp_as_async*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - __pyx_tp_traverse_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, /*tp_traverse*/ - __pyx_tp_clear_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyMethodDef __pyx_methods[] = { - {0, 0, 0, 0} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - #if PY_VERSION_HEX < 0x03020000 - { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, - #else - PyModuleDef_HEAD_INIT, - #endif - "traversal", - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_n_s_Countgraph, __pyx_k_Countgraph, sizeof(__pyx_k_Countgraph), 0, 0, 1, 1}, - {&__pyx_kp_s_Must_take_an_object_with_Hashtab, __pyx_k_Must_take_an_object_with_Hashtab, sizeof(__pyx_k_Must_take_an_object_with_Hashtab), 0, 0, 1, 0}, - {&__pyx_n_s_Nodegraph, __pyx_k_Nodegraph, sizeof(__pyx_k_Nodegraph), 0, 0, 1, 1}, - {&__pyx_n_s_Traverser_right_neighbors, __pyx_k_Traverser_right_neighbors, sizeof(__pyx_k_Traverser_right_neighbors), 0, 0, 1, 1}, - {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, - {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, - {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, - {&__pyx_n_s_graph, __pyx_k_graph, sizeof(__pyx_k_graph), 0, 0, 1, 1}, - {&__pyx_n_s_hashing, __pyx_k_hashing, sizeof(__pyx_k_hashing), 0, 0, 1, 1}, - {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, - {&__pyx_n_s_khmer, __pyx_k_khmer, sizeof(__pyx_k_khmer), 0, 0, 1, 1}, - {&__pyx_n_s_khmer__oxli_traversal, __pyx_k_khmer__oxli_traversal, sizeof(__pyx_k_khmer__oxli_traversal), 0, 0, 1, 1}, - {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, - {&__pyx_n_s_pyx_vtable, __pyx_k_pyx_vtable, sizeof(__pyx_k_pyx_vtable), 0, 0, 1, 1}, - {&__pyx_n_s_right_neighbors, __pyx_k_right_neighbors, sizeof(__pyx_k_right_neighbors), 0, 0, 1, 1}, - {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, - {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, - {0, 0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 15, __pyx_L1_error) - return 0; - __pyx_L1_error:; - return -1; -} - -static int __Pyx_InitCachedConstants(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - - /* "khmer/_oxli/traversal.pyx":15 - * def __cinit__(self, graph): - * if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): - * raise ValueError('Must take an object with Hashtable *') # <<<<<<<<<<<<<< - * - * cdef CPyHashtable_Object* ptr = graph - */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_Must_take_an_object_with_Hashtab); if (unlikely(!__pyx_tuple_)) __PYX_ERR(0, 15, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple_); - __Pyx_GIVEREF(__pyx_tuple_); - __Pyx_RefNannyFinishContext(); - return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC inittraversal(void); /*proto*/ -PyMODINIT_FUNC inittraversal(void) -#else -PyMODINIT_FUNC PyInit_traversal(void); /*proto*/ -PyMODINIT_FUNC PyInit_traversal(void) -#endif -{ - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannyDeclarations - #if CYTHON_REFNANNY - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); - if (!__Pyx_RefNanny) { - PyErr_Clear(); - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); - if (!__Pyx_RefNanny) - Py_FatalError("failed to import 'refnanny' module"); - } - #endif - __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_traversal(void)", 0); - if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) - #ifdef __Pyx_CyFunction_USED - if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_FusedFunction_USED - if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Coroutine_USED - if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_Generator_USED - if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - #ifdef __Pyx_StopAsyncIteration_USED - if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - /*--- Library function declarations ---*/ - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4("traversal", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) - __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) - Py_INCREF(__pyx_d); - __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) - #if CYTHON_COMPILING_IN_PYPY - Py_INCREF(__pyx_b); - #endif - if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); - /*--- Initialize various global constants etc. ---*/ - if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - if (__pyx_module_is_main_khmer___oxli__traversal) { - if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) - } - #if PY_MAJOR_VERSION >= 3 - { - PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) - if (!PyDict_GetItemString(modules, "khmer._oxli.traversal")) { - if (unlikely(PyDict_SetItemString(modules, "khmer._oxli.traversal", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) - } - } - #endif - /*--- Builtin init code ---*/ - if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - /*--- Constants init code ---*/ - if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - /*--- Global init code ---*/ - /*--- Variable export code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_9traversal_Traverser) < 0) __PYX_ERR(0, 11, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_9traversal_Traverser.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Traverser", (PyObject *)&__pyx_type_5khmer_5_oxli_9traversal_Traverser) < 0) __PYX_ERR(0, 11, __pyx_L1_error) - __pyx_ptype_5khmer_5_oxli_9traversal_Traverser = &__pyx_type_5khmer_5_oxli_9traversal_Traverser; - if (PyType_Ready(&__pyx_type_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors) < 0) __PYX_ERR(0, 30, __pyx_L1_error) - __pyx_type_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors.tp_print = 0; - __pyx_ptype_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors = &__pyx_type_5khmer_5_oxli_9traversal___pyx_scope_struct__right_neighbors; - /*--- Type import code ---*/ - __pyx_ptype_5khmer_5_oxli_7hashing_Kmer = __Pyx_ImportType("khmer._oxli.hashing", "Kmer", sizeof(struct __pyx_obj_5khmer_5_oxli_7hashing_Kmer), 1); if (unlikely(!__pyx_ptype_5khmer_5_oxli_7hashing_Kmer)) __PYX_ERR(1, 4, __pyx_L1_error) - __pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer = (struct __pyx_vtabstruct_5khmer_5_oxli_7hashing_Kmer*)__Pyx_GetVtable(__pyx_ptype_5khmer_5_oxli_7hashing_Kmer->tp_dict); if (unlikely(!__pyx_vtabptr_5khmer_5_oxli_7hashing_Kmer)) __PYX_ERR(1, 4, __pyx_L1_error) - /*--- Variable import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) - #endif - - /* "khmer/_oxli/traversal.pyx":4 - * from cython.operator cimport dereference as deref - * - * from .._khmer import Countgraph # <<<<<<<<<<<<<< - * from .._khmer import Nodegraph - * - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(__pyx_n_s_Countgraph); - __Pyx_GIVEREF(__pyx_n_s_Countgraph); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_Countgraph); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_khmer, __pyx_t_1, 2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_Countgraph); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Countgraph, __pyx_t_1) < 0) __PYX_ERR(0, 4, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "khmer/_oxli/traversal.pyx":5 - * - * from .._khmer import Countgraph - * from .._khmer import Nodegraph # <<<<<<<<<<<<<< - * - * from _oxli cimport * - */ - __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 5, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_INCREF(__pyx_n_s_Nodegraph); - __Pyx_GIVEREF(__pyx_n_s_Nodegraph); - PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_Nodegraph); - __pyx_t_1 = __Pyx_Import(__pyx_n_s_khmer, __pyx_t_2, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 5, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_Nodegraph); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 5, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_Nodegraph, __pyx_t_2) < 0) __PYX_ERR(0, 5, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "khmer/_oxli/traversal.pyx":9 - * from _oxli cimport * - * cimport hashing - * import hashing # <<<<<<<<<<<<<< - * - * cdef class Traverser: - */ - __pyx_t_1 = __Pyx_Import(__pyx_n_s_hashing, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 9, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_hashing, __pyx_t_1) < 0) __PYX_ERR(0, 9, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "khmer/_oxli/traversal.pyx":1 - * from libcpp.memory cimport unique_ptr # <<<<<<<<<<<<<< - * from cython.operator cimport dereference as deref - * - */ - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /*--- Wrapped vars code ---*/ - - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - if (__pyx_m) { - if (__pyx_d) { - __Pyx_AddTraceback("init khmer._oxli.traversal", __pyx_clineno, __pyx_lineno, __pyx_filename); - } - Py_DECREF(__pyx_m); __pyx_m = 0; - } else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ImportError, "init khmer._oxli.traversal"); - } - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -/* --- Runtime support code --- */ -/* Refnanny */ -#if CYTHON_REFNANNY -static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { - PyObject *m = NULL, *p = NULL; - void *r = NULL; - m = PyImport_ImportModule((char *)modname); - if (!m) goto end; - p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); - if (!p) goto end; - r = PyLong_AsVoidPtr(p); -end: - Py_XDECREF(p); - Py_XDECREF(m); - return (__Pyx_RefNannyAPIStruct *)r; -} -#endif - -/* GetBuiltinName */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); - if (unlikely(!result)) { - PyErr_Format(PyExc_NameError, -#if PY_MAJOR_VERSION >= 3 - "name '%U' is not defined", name); -#else - "name '%.200s' is not defined", PyString_AS_STRING(name)); -#endif - } - return result; -} - -/* RaiseDoubleKeywords */ -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AsString(kw_name)); - #endif -} - -/* ParseKeywords */ -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - continue; - } - name = first_kw_arg; - #if PY_MAJOR_VERSION < 3 - if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { - while (*name) { - if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) - && _PyString_Eq(**name, key)) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - if ((**argname == key) || ( - (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) - && _PyString_Eq(**argname, key))) { - goto arg_passed_twice; - } - argname++; - } - } - } else - #endif - if (likely(PyUnicode_Check(key))) { - while (*name) { - int cmp = (**name == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**name, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - int cmp = (**argname == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**argname, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) goto arg_passed_twice; - argname++; - } - } - } else - goto invalid_keyword_type; - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -/* RaiseArgTupleInvalid */ -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *more_or_less; - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", - func_name, more_or_less, num_expected, - (num_expected == 1) ? "" : "s", num_found); -} - -/* GetModuleGlobalName */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS - result = PyDict_GetItem(__pyx_d, name); - if (likely(result)) { - Py_INCREF(result); - } else { -#else - result = PyObject_GetItem(__pyx_d, name); - if (!result) { - PyErr_Clear(); -#endif - result = __Pyx_GetBuiltinName(name); - } - return result; -} - -/* PyObjectCall */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyObject *result; - ternaryfunc call = func->ob_type->tp_call; - if (unlikely(!call)) - return PyObject_Call(func, arg, kw); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = (*call)(func, arg, kw); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyErrFetchRestore */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -} -#endif - -/* RaiseException */ - #if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { - __Pyx_PyThreadState_declare - Py_XINCREF(type); - if (!value || value == Py_None) - value = NULL; - else - Py_INCREF(value); - if (!tb || tb == Py_None) - tb = NULL; - else { - Py_INCREF(tb); - if (!PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - } - if (PyType_Check(type)) { -#if CYTHON_COMPILING_IN_PYPY - if (!value) { - Py_INCREF(Py_None); - value = Py_None; - } -#endif - PyErr_NormalizeException(&type, &value, &tb); - } else { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - value = type; - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - } - __Pyx_PyThreadState_assign - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} -#else -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { - PyObject* owned_instance = NULL; - if (tb == Py_None) { - tb = 0; - } else if (tb && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto bad; - } - if (value == Py_None) - value = 0; - if (PyExceptionInstance_Check(type)) { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto bad; - } - value = type; - type = (PyObject*) Py_TYPE(value); - } else if (PyExceptionClass_Check(type)) { - PyObject *instance_class = NULL; - if (value && PyExceptionInstance_Check(value)) { - instance_class = (PyObject*) Py_TYPE(value); - if (instance_class != type) { - int is_subclass = PyObject_IsSubclass(instance_class, type); - if (!is_subclass) { - instance_class = NULL; - } else if (unlikely(is_subclass == -1)) { - goto bad; - } else { - type = instance_class; - } - } - } - if (!instance_class) { - PyObject *args; - if (!value) - args = PyTuple_New(0); - else if (PyTuple_Check(value)) { - Py_INCREF(value); - args = value; - } else - args = PyTuple_Pack(1, value); - if (!args) - goto bad; - owned_instance = PyObject_Call(type, args, NULL); - Py_DECREF(args); - if (!owned_instance) - goto bad; - value = owned_instance; - if (!PyExceptionInstance_Check(value)) { - PyErr_Format(PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto bad; - } - } - } else { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto bad; - } -#if PY_VERSION_HEX >= 0x03030000 - if (cause) { -#else - if (cause && cause != Py_None) { -#endif - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; - } - PyException_SetCause(value, fixed_cause); - } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = PyThreadState_GET(); - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); - } -#endif - } -bad: - Py_XDECREF(owned_instance); - return; -} -#endif - -/* ArgTypeTest */ - static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); -} -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - if (none_allowed && obj == Py_None) return 1; - else if (exact) { - if (likely(Py_TYPE(obj) == type)) return 1; - #if PY_MAJOR_VERSION == 2 - else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } - else { - if (likely(PyObject_TypeCheck(obj, type))) return 1; - } - __Pyx_RaiseArgumentTypeInvalid(name, obj, type); - return 0; -} - -/* GetVTable */ - static void* __Pyx_GetVtable(PyObject *dict) { - void* ptr; - PyObject *ob = PyObject_GetItem(dict, __pyx_n_s_pyx_vtable); - if (!ob) - goto bad; -#if PY_VERSION_HEX >= 0x02070000 - ptr = PyCapsule_GetPointer(ob, 0); -#else - ptr = PyCObject_AsVoidPtr(ob); -#endif - if (!ptr && !PyErr_Occurred()) - PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); - Py_DECREF(ob); - return ptr; -bad: - Py_XDECREF(ob); - return NULL; -} - -/* Import */ - static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_import; - py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); - if (!py_import) - goto bad; - #endif - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - { - #if PY_MAJOR_VERSION >= 3 - if (level == -1) { - if (strchr(__Pyx_MODULE_NAME, '.')) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(1); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - #endif - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - goto bad; - PyErr_Clear(); - } - } - level = 0; - } - #endif - if (!module) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(level); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); - #endif - } - } -bad: - #if PY_VERSION_HEX < 0x03030000 - Py_XDECREF(py_import); - #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); - return module; -} - -/* ImportFrom */ - static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { - PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); - if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Format(PyExc_ImportError, - #if PY_MAJOR_VERSION < 3 - "cannot import name %.230s", PyString_AS_STRING(name)); - #else - "cannot import name %S", name); - #endif - } - return value; -} - -/* CodeObjectCache */ - static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { - int start = 0, mid = 0, end = count - 1; - if (end >= 0 && code_line > entries[end].code_line) { - return count; - } - while (start < end) { - mid = start + (end - start) / 2; - if (code_line < entries[mid].code_line) { - end = mid; - } else if (code_line > entries[mid].code_line) { - start = mid + 1; - } else { - return mid; - } - } - if (code_line <= entries[mid].code_line) { - return mid; - } else { - return mid + 1; - } -} -static PyCodeObject *__pyx_find_code_object(int code_line) { - PyCodeObject* code_object; - int pos; - if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { - return NULL; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { - return NULL; - } - code_object = __pyx_code_cache.entries[pos].code_object; - Py_INCREF(code_object); - return code_object; -} -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { - int pos, i; - __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; - if (unlikely(!code_line)) { - return; - } - if (unlikely(!entries)) { - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); - if (likely(entries)) { - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = 64; - __pyx_code_cache.count = 1; - entries[0].code_line = code_line; - entries[0].code_object = code_object; - Py_INCREF(code_object); - } - return; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { - PyCodeObject* tmp = entries[pos].code_object; - entries[pos].code_object = code_object; - Py_DECREF(tmp); - return; - } - if (__pyx_code_cache.count == __pyx_code_cache.max_count) { - int new_max = __pyx_code_cache.max_count + 64; - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( - __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); - if (unlikely(!entries)) { - return; - } - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = new_max; - } - for (i=__pyx_code_cache.count; i>pos; i--) { - entries[i] = entries[i-1]; - } - entries[pos].code_line = code_line; - entries[pos].code_object = code_object; - __pyx_code_cache.count++; - Py_INCREF(code_object); -} - -/* AddTraceback */ - #include "compile.h" -#include "frameobject.h" -#include "traceback.h" -static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( - const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(filename); - #else - py_srcfile = PyUnicode_FromString(filename); - #endif - if (!py_srcfile) goto bad; - if (c_line) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_code = __Pyx_PyCode_New( - 0, - 0, - 0, - 0, - 0, - __pyx_empty_bytes, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - py_line, - __pyx_empty_bytes /*PyObject *lnotab*/ - ); - Py_DECREF(py_srcfile); - Py_DECREF(py_funcname); - return py_code; -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - return NULL; -} -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - py_code = __pyx_find_code_object(c_line ? c_line : py_line); - if (!py_code) { - py_code = __Pyx_CreateCodeObjectForTraceback( - funcname, c_line, py_line, filename); - if (!py_code) goto bad; - __pyx_insert_code_object(c_line ? c_line : py_line, py_code); - } - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - __pyx_d, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - __Pyx_PyFrame_SetLineNumber(py_frame, py_line); - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - -/* CIntFromPyVerify */ - #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) -#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) -#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ - {\ - func_type value = func_value;\ - if (sizeof(target_type) < sizeof(func_type)) {\ - if (unlikely(value != (func_type) (target_type) value)) {\ - func_type zero = 0;\ - if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ - return (target_type) -1;\ - if (is_unsigned && unlikely(value < zero))\ - goto raise_neg_overflow;\ - else\ - goto raise_overflow;\ - }\ - }\ - return (target_type) value;\ - } - -/* CIntFromPy */ - static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { - const long neg_one = (long) -1, const_zero = (long) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(long) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (long) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { - return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { - return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { - return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (long) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(long) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (long) 0; - case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) - case -2: - if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(long) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(long) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(long) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); - } - } - break; - } -#endif - if (sizeof(long) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - long val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (long) -1; - } - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (long) -1; - val = __Pyx_PyInt_As_long(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to long"); - return (long) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to long"); - return (long) -1; -} - -/* CIntFromPy */ - static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) -1, const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (int) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { - return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { - return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { - return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); - } - } - break; - } -#endif -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } -#else - { - int result = PyObject_RichCompareBool(x, Py_False, Py_LT); - if (unlikely(result < 0)) - return (int) -1; - if (unlikely(result == 1)) - goto raise_neg_overflow; - } -#endif - if (sizeof(int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) -#endif - } - } else { -#if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return (int) 0; - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) - case -2: - if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -3: - if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case -4: - if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); - } - } - break; - } -#endif - if (sizeof(int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) -#endif - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - int val; - PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (int) -1; - } - } else { - int val; - PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (int) -1; - val = __Pyx_PyInt_As_int(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to int"); - return (int) -1; -} - -/* FetchCommonType */ - static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } -done: - Py_DECREF(fake_module); - return cached_type; -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; -} - -/* SwapException */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = *type; - tstate->exc_value = *value; - tstate->exc_traceback = *tb; - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} -#else -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); - PyErr_SetExcInfo(*type, *value, *tb); - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} -#endif - -/* PyCFunctionFastCall */ - #if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - PyObject *result; - int flags; - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); -} -#endif // CYTHON_FAST_PYCCALL - -/* PyFunctionFastCall */ - #if CYTHON_FAST_PYCALL -#include "frameobject.h" -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = PyThreadState_GET(); - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = f->f_localsplus; - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; -} -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && -#endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); -#endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); - } - else { - d = NULL; - nd = 0; - } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); -#else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); -#endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; -} -#endif // CPython < 3.6 -#endif // CYTHON_FAST_PYCALL - -/* PyObjectCallMethO */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyObjectCallOneArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif - -/* PyObjectCallMethod1 */ - static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { - PyObject *method, *result = NULL; - method = __Pyx_PyObject_GetAttrStr(obj, method_name); - if (unlikely(!method)) goto done; -#if CYTHON_UNPACK_METHODS - if (likely(PyMethod_Check(method))) { - PyObject *self = PyMethod_GET_SELF(method); - if (likely(self)) { - PyObject *args; - PyObject *function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyFunction_FastCall(function, args, 2); - goto done; - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyCFunction_FastCall(function, args, 2); - goto done; - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; - } - } -#endif - result = __Pyx_PyObject_CallOneArg(method, arg); -done: - Py_XDECREF(method); - return result; -} - -/* CoroutineBase */ - #include -#include -static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); -static PyObject *__Pyx_Coroutine_Close(PyObject *self); -static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); -#define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { - PyObject *et, *ev, *tb; - PyObject *value = NULL; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&et, &ev, &tb); - if (!et) { - Py_XDECREF(tb); - Py_XDECREF(ev); - Py_INCREF(Py_None); - *pvalue = Py_None; - return 0; - } - if (likely(et == PyExc_StopIteration)) { - if (!ev) { - Py_INCREF(Py_None); - value = Py_None; - } -#if PY_VERSION_HEX >= 0x030300A0 - else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { - value = ((PyStopIterationObject *)ev)->value; - Py_INCREF(value); - Py_DECREF(ev); - } -#endif - else if (unlikely(PyTuple_Check(ev))) { - if (PyTuple_GET_SIZE(ev) >= 1) { -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - value = PyTuple_GET_ITEM(ev, 0); - Py_INCREF(value); -#else - value = PySequence_ITEM(ev, 0); -#endif - } else { - Py_INCREF(Py_None); - value = Py_None; - } - Py_DECREF(ev); - } - else if (!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { - value = ev; - } - if (likely(value)) { - Py_XDECREF(tb); - Py_DECREF(et); - *pvalue = value; - return 0; - } - } else if (!PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - PyErr_NormalizeException(&et, &ev, &tb); - if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - Py_XDECREF(tb); - Py_DECREF(et); -#if PY_VERSION_HEX >= 0x030300A0 - value = ((PyStopIterationObject *)ev)->value; - Py_INCREF(value); - Py_DECREF(ev); -#else - { - PyObject* args = __Pyx_PyObject_GetAttrStr(ev, __pyx_n_s_args); - Py_DECREF(ev); - if (likely(args)) { - value = PySequence_GetItem(args, 0); - Py_DECREF(args); - } - if (unlikely(!value)) { - __Pyx_ErrRestore(NULL, NULL, NULL); - Py_INCREF(Py_None); - value = Py_None; - } - } -#endif - *pvalue = value; - return 0; -} -#endif -static CYTHON_INLINE -void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { - PyObject *exc_type = self->exc_type; - PyObject *exc_value = self->exc_value; - PyObject *exc_traceback = self->exc_traceback; - self->exc_type = NULL; - self->exc_value = NULL; - self->exc_traceback = NULL; - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_traceback); -} -static CYTHON_INLINE -int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) { - if (unlikely(gen->is_running)) { - PyErr_SetString(PyExc_ValueError, - "generator already executing"); - return 1; - } - return 0; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) { - PyObject *retval; - __Pyx_PyThreadState_declare - assert(!self->is_running); - if (unlikely(self->resume_label == 0)) { - if (unlikely(value && value != Py_None)) { - PyErr_SetString(PyExc_TypeError, - "can't send non-None value to a " - "just-started generator"); - return NULL; - } - } - if (unlikely(self->resume_label == -1)) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - __Pyx_PyThreadState_assign - if (value) { -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON -#else - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_XINCREF(__pyx_tstate->frame); - assert(f->f_back == NULL); - f->f_back = __pyx_tstate->frame; - } -#endif - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); - } else { - __Pyx_Coroutine_ExceptionClear(self); - } - self->is_running = 1; - retval = self->body((PyObject *) self, value); - self->is_running = 0; - if (retval) { - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON -#else - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_CLEAR(f->f_back); - } -#endif - } else { - __Pyx_Coroutine_ExceptionClear(self); - } - return retval; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) { - if (unlikely(!retval && !PyErr_Occurred())) { - PyErr_SetNone(PyExc_StopIteration); - } - return retval; -} -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { - PyObject *ret; - PyObject *val = NULL; - __Pyx_Coroutine_Undelegate(gen); - __Pyx_PyGen_FetchStopIterationValue(&val); - ret = __Pyx_Coroutine_SendEx(gen, val); - Py_XDECREF(val); - return ret; -} -static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { - PyObject *retval; - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Coroutine_Send(yf, value); - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - ret = __Pyx_Coroutine_Send(yf, value); - } else - #endif - { - if (value == Py_None) - ret = Py_TYPE(yf)->tp_iternext(yf); - else - ret = __Pyx_PyObject_CallMethod1(yf, __pyx_n_s_send, value); - } - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - retval = __Pyx_Coroutine_FinishDelegation(gen); - } else { - retval = __Pyx_Coroutine_SendEx(gen, value); - } - return __Pyx_Coroutine_MethodReturn(retval); -} -static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { - PyObject *retval = NULL; - int err = 0; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - retval = __Pyx_Coroutine_Close(yf); - if (!retval) - return -1; - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - retval = __Pyx_Coroutine_Close(yf); - if (!retval) - return -1; - } else - #endif - { - PyObject *meth; - gen->is_running = 1; - meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_close); - if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_WriteUnraisable(yf); - } - PyErr_Clear(); - } else { - retval = PyObject_CallFunction(meth, NULL); - Py_DECREF(meth); - if (!retval) - err = -1; - } - gen->is_running = 0; - } - Py_XDECREF(retval); - return err; -} -static PyObject *__Pyx_Generator_Next(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Generator_Next(yf); - } else - #endif - ret = Py_TYPE(yf)->tp_iternext(yf); - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - return __Pyx_Coroutine_FinishDelegation(gen); - } - return __Pyx_Coroutine_SendEx(gen, Py_None); -} -static PyObject *__Pyx_Coroutine_Close(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject *retval, *raised_exception; - PyObject *yf = gen->yieldfrom; - int err = 0; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - Py_INCREF(yf); - err = __Pyx_Coroutine_CloseIter(gen, yf); - __Pyx_Coroutine_Undelegate(gen); - Py_DECREF(yf); - } - if (err == 0) - PyErr_SetNone(PyExc_GeneratorExit); - retval = __Pyx_Coroutine_SendEx(gen, NULL); - if (retval) { - Py_DECREF(retval); - PyErr_SetString(PyExc_RuntimeError, - "generator ignored GeneratorExit"); - return NULL; - } - raised_exception = PyErr_Occurred(); - if (!raised_exception - || raised_exception == PyExc_StopIteration - || raised_exception == PyExc_GeneratorExit - || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) - || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) - { - if (raised_exception) PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - return NULL; -} -static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; - PyObject *yf = gen->yieldfrom; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) - return NULL; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - Py_INCREF(yf); - if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { - int err = __Pyx_Coroutine_CloseIter(gen, yf); - Py_DECREF(yf); - __Pyx_Coroutine_Undelegate(gen); - if (err < 0) - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); - goto throw_here; - } - gen->is_running = 1; - #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else - #endif - #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else - #endif - { - PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_throw); - if (unlikely(!meth)) { - Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - gen->is_running = 0; - return NULL; - } - PyErr_Clear(); - __Pyx_Coroutine_Undelegate(gen); - gen->is_running = 0; - goto throw_here; - } - ret = PyObject_CallObject(meth, args); - Py_DECREF(meth); - } - gen->is_running = 0; - Py_DECREF(yf); - if (!ret) { - ret = __Pyx_Coroutine_FinishDelegation(gen); - } - return __Pyx_Coroutine_MethodReturn(ret); - } -throw_here: - __Pyx_Raise(typ, val, tb, NULL); - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); -} -static int __Pyx_Coroutine_traverse(PyObject *self, visitproc visit, void *arg) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - Py_VISIT(gen->closure); - Py_VISIT(gen->classobj); - Py_VISIT(gen->yieldfrom); - Py_VISIT(gen->exc_type); - Py_VISIT(gen->exc_value); - Py_VISIT(gen->exc_traceback); - return 0; -} -static int __Pyx_Coroutine_clear(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - Py_CLEAR(gen->closure); - Py_CLEAR(gen->classobj); - Py_CLEAR(gen->yieldfrom); - Py_CLEAR(gen->exc_type); - Py_CLEAR(gen->exc_value); - Py_CLEAR(gen->exc_traceback); - Py_CLEAR(gen->gi_name); - Py_CLEAR(gen->gi_qualname); - return 0; -} -static void __Pyx_Coroutine_dealloc(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject_GC_UnTrack(gen); - if (gen->gi_weakreflist != NULL) - PyObject_ClearWeakRefs(self); - if (gen->resume_label > 0) { - PyObject_GC_Track(self); -#if PY_VERSION_HEX >= 0x030400a1 - if (PyObject_CallFinalizerFromDealloc(self)) -#else - Py_TYPE(gen)->tp_del(self); - if (self->ob_refcnt > 0) -#endif - { - return; - } - PyObject_GC_UnTrack(self); - } - __Pyx_Coroutine_clear(self); - PyObject_GC_Del(gen); -} -static void __Pyx_Coroutine_del(PyObject *self) { - PyObject *res; - PyObject *error_type, *error_value, *error_traceback; - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - __Pyx_PyThreadState_declare - if (gen->resume_label <= 0) - return ; -#if PY_VERSION_HEX < 0x030400a1 - assert(self->ob_refcnt == 0); - self->ob_refcnt = 1; -#endif - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); - res = __Pyx_Coroutine_Close(self); - if (res == NULL) - PyErr_WriteUnraisable(self); - else - Py_DECREF(res); - __Pyx_ErrRestore(error_type, error_value, error_traceback); -#if PY_VERSION_HEX < 0x030400a1 - assert(self->ob_refcnt > 0); - if (--self->ob_refcnt == 0) { - return; - } - { - Py_ssize_t refcnt = self->ob_refcnt; - _Py_NewReference(self); - self->ob_refcnt = refcnt; - } -#if CYTHON_COMPILING_IN_CPYTHON - assert(PyType_IS_GC(self->ob_type) && - _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); - _Py_DEC_REFTOTAL; -#endif -#ifdef COUNT_ALLOCS - --Py_TYPE(self)->tp_frees; - --Py_TYPE(self)->tp_allocs; -#endif -#endif -} -static PyObject * -__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self) -{ - PyObject *name = self->gi_name; - if (unlikely(!name)) name = Py_None; - Py_INCREF(name); - return name; -} -static int -__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { -#else - if (unlikely(value == NULL || !PyString_Check(value))) { -#endif - PyErr_SetString(PyExc_TypeError, - "__name__ must be set to a string object"); - return -1; - } - tmp = self->gi_name; - Py_INCREF(value); - self->gi_name = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self) -{ - PyObject *name = self->gi_qualname; - if (unlikely(!name)) name = Py_None; - Py_INCREF(name); - return name; -} -static int -__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { -#else - if (unlikely(value == NULL || !PyString_Check(value))) { -#endif - PyErr_SetString(PyExc_TypeError, - "__qualname__ must be set to a string object"); - return -1; - } - tmp = self->gi_qualname; - Py_INCREF(value); - self->gi_qualname = value; - Py_XDECREF(tmp); - return 0; -} -static __pyx_CoroutineObject *__Pyx__Coroutine_New( - PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *closure, - PyObject *name, PyObject *qualname, PyObject *module_name) { - __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); - if (gen == NULL) - return NULL; - gen->body = body; - gen->closure = closure; - Py_XINCREF(closure); - gen->is_running = 0; - gen->resume_label = 0; - gen->classobj = NULL; - gen->yieldfrom = NULL; - gen->exc_type = NULL; - gen->exc_value = NULL; - gen->exc_traceback = NULL; - gen->gi_weakreflist = NULL; - Py_XINCREF(qualname); - gen->gi_qualname = qualname; - Py_XINCREF(name); - gen->gi_name = name; - Py_XINCREF(module_name); - gen->gi_modulename = module_name; - PyObject_GC_Track(gen); - return gen; -} - -/* PatchModuleWithCoroutine */ - static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code) { -#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - int result; - PyObject *globals, *result_obj; - globals = PyDict_New(); if (unlikely(!globals)) goto ignore; - result = PyDict_SetItemString(globals, "_cython_coroutine_type", - #ifdef __Pyx_Coroutine_USED - (PyObject*)__pyx_CoroutineType); - #else - Py_None); - #endif - if (unlikely(result < 0)) goto ignore; - result = PyDict_SetItemString(globals, "_cython_generator_type", - #ifdef __Pyx_Generator_USED - (PyObject*)__pyx_GeneratorType); - #else - Py_None); - #endif - if (unlikely(result < 0)) goto ignore; - if (unlikely(PyDict_SetItemString(globals, "_module", module) < 0)) goto ignore; - if (unlikely(PyDict_SetItemString(globals, "__builtins__", __pyx_b) < 0)) goto ignore; - result_obj = PyRun_String(py_code, Py_file_input, globals, globals); - if (unlikely(!result_obj)) goto ignore; - Py_DECREF(result_obj); - Py_DECREF(globals); - return module; -ignore: - Py_XDECREF(globals); - PyErr_WriteUnraisable(module); - if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, "Cython module failed to patch module with custom type", 1) < 0)) { - Py_DECREF(module); - module = NULL; - } -#else - py_code++; -#endif - return module; -} - -/* PatchGeneratorABC */ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) -static PyObject* __Pyx_patch_abc_module(PyObject *module); -static PyObject* __Pyx_patch_abc_module(PyObject *module) { - module = __Pyx_Coroutine_patch_module( - module, "" -"if _cython_generator_type is not None:\n" -" try: Generator = _module.Generator\n" -" except AttributeError: pass\n" -" else: Generator.register(_cython_generator_type)\n" -"if _cython_coroutine_type is not None:\n" -" try: Coroutine = _module.Coroutine\n" -" except AttributeError: pass\n" -" else: Coroutine.register(_cython_coroutine_type)\n" - ); - return module; -} -#endif -static int __Pyx_patch_abc(void) { -#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - static int abc_patched = 0; - if (!abc_patched) { - PyObject *module; - module = PyImport_ImportModule((PY_VERSION_HEX >= 0x03030000) ? "collections.abc" : "collections"); - if (!module) { - PyErr_WriteUnraisable(NULL); - if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, - ((PY_VERSION_HEX >= 0x03030000) ? - "Cython module failed to register with collections.abc module" : - "Cython module failed to register with collections module"), 1) < 0)) { - return -1; - } - } else { - module = __Pyx_patch_abc_module(module); - abc_patched = 1; - if (unlikely(!module)) - return -1; - Py_DECREF(module); - } - module = PyImport_ImportModule("backports_abc"); - if (module) { - module = __Pyx_patch_abc_module(module); - Py_XDECREF(module); - } - if (!module) { - PyErr_Clear(); - } - } -#else - if (0) __Pyx_Coroutine_patch_module(NULL, NULL); -#endif - return 0; -} - -/* Generator */ - static PyMethodDef __pyx_Generator_methods[] = { - {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, - (char*) PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, - {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, - (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, - {"close", (PyCFunction) __Pyx_Coroutine_Close, METH_NOARGS, - (char*) PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, - {0, 0, 0, 0} -}; -static PyMemberDef __pyx_Generator_memberlist[] = { - {(char *) "gi_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, - {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, - (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, - {0, 0, 0, 0, 0} -}; -static PyGetSetDef __pyx_Generator_getsets[] = { - {(char *) "__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, - (char*) PyDoc_STR("name of the generator"), 0}, - {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, - (char*) PyDoc_STR("qualified name of the generator"), 0}, - {0, 0, 0, 0, 0} -}; -static PyTypeObject __pyx_GeneratorType_type = { - PyVarObject_HEAD_INIT(0, 0) - "generator", - sizeof(__pyx_CoroutineObject), - 0, - (destructor) __Pyx_Coroutine_dealloc, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, - 0, - (traverseproc) __Pyx_Coroutine_traverse, - 0, - 0, - offsetof(__pyx_CoroutineObject, gi_weakreflist), - 0, - (iternextfunc) __Pyx_Generator_Next, - __pyx_Generator_methods, - __pyx_Generator_memberlist, - __pyx_Generator_getsets, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, -#if PY_VERSION_HEX >= 0x030400a1 - 0, -#else - __Pyx_Coroutine_del, -#endif - 0, -#if PY_VERSION_HEX >= 0x030400a1 - __Pyx_Coroutine_del, -#endif -}; -static int __pyx_Generator_init(void) { - __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; - __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; - __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); - if (unlikely(!__pyx_GeneratorType)) { - return -1; - } - return 0; -} - -/* CheckBinaryVersion */ - static int __Pyx_check_binary_version(void) { - char ctversion[4], rtversion[4]; - PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); - PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); - if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { - char message[200]; - PyOS_snprintf(message, sizeof(message), - "compiletime version %s of module '%.100s' " - "does not match runtime version %s", - ctversion, __Pyx_MODULE_NAME, rtversion); - return PyErr_WarnEx(NULL, message, 1); - } - return 0; -} - -/* ModuleImport */ - #ifndef __PYX_HAVE_RT_ImportModule -#define __PYX_HAVE_RT_ImportModule -static PyObject *__Pyx_ImportModule(const char *name) { - PyObject *py_name = 0; - PyObject *py_module = 0; - py_name = __Pyx_PyIdentifier_FromString(name); - if (!py_name) - goto bad; - py_module = PyImport_Import(py_name); - Py_DECREF(py_name); - return py_module; -bad: - Py_XDECREF(py_name); - return 0; -} -#endif - -/* TypeImport */ - #ifndef __PYX_HAVE_RT_ImportType -#define __PYX_HAVE_RT_ImportType -static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, - size_t size, int strict) -{ - PyObject *py_module = 0; - PyObject *result = 0; - PyObject *py_name = 0; - char warning[200]; - Py_ssize_t basicsize; -#ifdef Py_LIMITED_API - PyObject *py_basicsize; -#endif - py_module = __Pyx_ImportModule(module_name); - if (!py_module) - goto bad; - py_name = __Pyx_PyIdentifier_FromString(class_name); - if (!py_name) - goto bad; - result = PyObject_GetAttr(py_module, py_name); - Py_DECREF(py_name); - py_name = 0; - Py_DECREF(py_module); - py_module = 0; - if (!result) - goto bad; - if (!PyType_Check(result)) { - PyErr_Format(PyExc_TypeError, - "%.200s.%.200s is not a type object", - module_name, class_name); - goto bad; - } -#ifndef Py_LIMITED_API - basicsize = ((PyTypeObject *)result)->tp_basicsize; -#else - py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); - if (!py_basicsize) - goto bad; - basicsize = PyLong_AsSsize_t(py_basicsize); - Py_DECREF(py_basicsize); - py_basicsize = 0; - if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) - goto bad; -#endif - if (!strict && (size_t)basicsize > size) { - PyOS_snprintf(warning, sizeof(warning), - "%s.%s size changed, may indicate binary incompatibility. Expected %zd, got %zd", - module_name, class_name, basicsize, size); - if (PyErr_WarnEx(NULL, warning, 0) < 0) goto bad; - } - else if ((size_t)basicsize != size) { - PyErr_Format(PyExc_ValueError, - "%.200s.%.200s has the wrong size, try recompiling. Expected %zd, got %zd", - module_name, class_name, basicsize, size); - goto bad; - } - return (PyTypeObject *)result; -bad: - Py_XDECREF(py_module); - Py_XDECREF(result); - return NULL; -} -#endif - -/* InitStrings */ - static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { - return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { - Py_ssize_t ignore; - return __Pyx_PyObject_AsStringAndSize(o, &ignore); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { -#if PY_VERSION_HEX < 0x03030000 - char* defenc_c; - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - PyUnicode_AsASCIIString(o); - return NULL; - } - } - } -#endif - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; -#else - if (__Pyx_PyUnicode_READY(o) == -1) return NULL; -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (PyUnicode_IS_ASCII(o)) { - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - PyUnicode_AsASCIIString(o); - return NULL; - } -#else - return PyUnicode_AsUTF8AndSize(o, length); -#endif -#endif - } else -#endif -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) - if (PyByteArray_Check(o)) { - *length = PyByteArray_GET_SIZE(o); - return PyByteArray_AS_STRING(o); - } else -#endif - { - char* result; - int r = PyBytes_AsStringAndSize(o, &result, length); - if (unlikely(r < 0)) { - return NULL; - } else { - return result; - } - } -} -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - int is_true = x == Py_True; - if (is_true | (x == Py_False) | (x == Py_None)) return is_true; - else return PyObject_IsTrue(x); -} -static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { -#if CYTHON_USE_TYPE_SLOTS - PyNumberMethods *m; -#endif - const char *name = NULL; - PyObject *res = NULL; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return __Pyx_NewRef(x); -#if CYTHON_USE_TYPE_SLOTS - m = Py_TYPE(x)->tp_as_number; - #if PY_MAJOR_VERSION < 3 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } - #else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } - #endif -#else - res = PyNumber_Int(x); -#endif - if (res) { -#if PY_MAJOR_VERSION < 3 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject *x; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(b))) { - if (sizeof(Py_ssize_t) >= sizeof(long)) - return PyInt_AS_LONG(b); - else - return PyInt_AsSsize_t(x); - } -#endif - if (likely(PyLong_CheckExact(b))) { - #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)b)->ob_digit; - const Py_ssize_t size = Py_SIZE(b); - if (likely(__Pyx_sst_abs(size) <= 1)) { - ival = likely(size) ? digits[0] : 0; - if (size == -1) ival = -ival; - return ival; - } else { - switch (size) { - case 2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -2: - if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -3: - if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case 4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - case -4: - if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { - return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); - } - break; - } - } - #endif - return PyLong_AsSsize_t(b); - } - x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { - return PyInt_FromSize_t(ival); -} - - -#endif /* Py_PYTHON_H */ From 9131a051b68e78962cc3af07a65a27ecef2fc1f0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 16 Nov 2016 18:40:03 -0800 Subject: [PATCH 041/185] Update tests for new cython Extension organization --- tests/test_cython_oxli.py | 74 +++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/test_cython_oxli.py b/tests/test_cython_oxli.py index c66cebe11e..6b116c6f52 100644 --- a/tests/test_cython_oxli.py +++ b/tests/test_cython_oxli.py @@ -6,7 +6,7 @@ import random import khmer -from khmer import _oxli +from khmer._oxli.partitioning import StreamingPartitioner, Component from khmer.khmer_args import estimate_optimal_with_K_and_f as optimal_fp from khmer import reverse_complement as revcomp from khmer import reverse_hash as revhash @@ -23,7 +23,7 @@ def teardown(): @pytest.fixture def partitioner(graph): - sp = _oxli.StreamingPartitioner(graph) + sp = StreamingPartitioner(graph) return graph, sp @@ -31,7 +31,7 @@ def partitioner(graph): def single_component(partitioner, random_sequence): graph, partitioner = partitioner sequence = random_sequence() - partitioner.consume_sequence(sequence) + partitioner.consume(sequence) return graph, partitioner, sequence @@ -50,8 +50,8 @@ def test_one_component(self, known_sequence): inpath = utils.get_test_data('random-20-a.fa') cg = khmer.Countgraph(K, 1e5, 4) - sp = _oxli.StreamingPartitioner(cg) - sp.consume_sequence(known_sequence) + sp = StreamingPartitioner(cg) + sp.consume(known_sequence) assert sp.n_components == 1 @@ -60,12 +60,12 @@ def test_two_components(self, random_sequence): comp2 = random_sequence(exclude=comp1) cg = khmer.Nodegraph(K, 1e5, 4) - sp = _oxli.StreamingPartitioner(cg) + sp = StreamingPartitioner(cg) - sp.consume_sequence(comp1) + sp.consume(comp1) assert sp.n_components == 1 - sp.consume_sequence(comp2) + sp.consume(comp2) assert sp.n_components == 2 def test_components_iter(self, random_sequence): @@ -73,10 +73,10 @@ def test_components_iter(self, random_sequence): comp2 = random_sequence(exclude=comp1) cg = khmer.Nodegraph(K, 1e5, 4) - sp = _oxli.StreamingPartitioner(cg) + sp = StreamingPartitioner(cg) - sp.consume_sequence(comp1) - sp.consume_sequence(comp2) + sp.consume(comp1) + sp.consume(comp2) assert sp.n_components == 2 comps = list(sp.components()) @@ -86,8 +86,8 @@ def test_component_n_tags(self, random_sequence): seq = random_sequence() cg = khmer.Nodegraph(K, 1e5, 4) - sp = _oxli.StreamingPartitioner(cg) - sp.consume_sequence(seq) + sp = StreamingPartitioner(cg) + sp.consume(seq) tags = [t for t,c in sp.tag_components()] comp = sp.get_nearest_component(seq[:K]) @@ -98,10 +98,10 @@ def test_tag_components_iter(self, random_sequence): comp2 = random_sequence(exclude=comp1) cg = khmer.Nodegraph(K, 1e5, 4) - sp = _oxli.StreamingPartitioner(cg) + sp = StreamingPartitioner(cg) - sp.consume_sequence(comp1) - sp.consume_sequence(comp2) + sp.consume(comp1) + sp.consume(comp2) assert sp.n_components == 2 tags = [] @@ -119,10 +119,10 @@ def test_get_nearest_component(self, random_sequence): seq2 = random_sequence(exclude=seq1) cg = khmer.Nodegraph(K, 1e5, 4) - sp = _oxli.StreamingPartitioner(cg) + sp = StreamingPartitioner(cg) - sp.consume_sequence(seq1) - sp.consume_sequence(seq2) + sp.consume(seq1) + sp.consume(seq2) c1 = sp.get_nearest_component(seq1[:K]) c2 = sp.get_nearest_component(seq2[:K]) @@ -141,13 +141,13 @@ def test_merge_components(self, random_sequence): seq2 = random_sequence(exclude=seq1) cg = khmer.Nodegraph(K, 1e5, 4) - sp = _oxli.StreamingPartitioner(cg) + sp = StreamingPartitioner(cg) - sp.consume_sequence(seq1) - sp.consume_sequence(seq2) + sp.consume(seq1) + sp.consume(seq2) assert sp.n_components == 2 - sp.consume_sequence(seq1 + seq2) + sp.consume(seq1 + seq2) assert sp.n_components == 1 comps = list(sp.components()) @@ -160,14 +160,14 @@ def test_multi_merge_components(self, random_sequence): seq3 = random_sequence(exclude=seq1+seq2) cg = khmer.Nodegraph(K, 1e5, 4) - sp = _oxli.StreamingPartitioner(cg) + sp = StreamingPartitioner(cg) - sp.consume_sequence(seq1) - sp.consume_sequence(seq2) - sp.consume_sequence(seq3) + sp.consume(seq1) + sp.consume(seq2) + sp.consume(seq3) assert sp.n_components == 3 - sp.consume_sequence(seq1 + seq2 + seq3) + sp.consume(seq1 + seq2 + seq3) assert sp.n_components == 1 def test_nomerge_k_minus_2_overlap(self, single_component, random_sequence): @@ -179,7 +179,7 @@ def test_nomerge_k_minus_2_overlap(self, single_component, random_sequence): neighbor = random_sequence(exclude=seq) + first assert partitioner.n_components == 1 - partitioner.consume_sequence(neighbor) + partitioner.consume(neighbor) print(seq, neighbor, graph.assemble_linear_path(seq[:K]), sep='\n') assert partitioner.n_components == 2 @@ -198,7 +198,7 @@ def test_merge_k_minus_1_overlap(self, single_component, random_sequence, neighbor = overlap + random_sequence(exclude=seq) assert partitioner.n_components == 1 - partitioner.consume_sequence(neighbor) + partitioner.consume(neighbor) path = graph.assemble_linear_path(seq[:K]) assert partitioner.n_components == 1 @@ -211,7 +211,7 @@ def test_merge_k_overlap(self, single_component, random_sequence): neighbor = random_sequence(exclude=seq) + first assert partitioner.n_components == 1 - partitioner.consume_sequence(neighbor) + partitioner.consume(neighbor) print(seq, neighbor, graph.assemble_linear_path(seq[:K]), sep='\n') assert partitioner.n_components == 1 @@ -222,9 +222,9 @@ def test_one_component_from_reads(self, random_sequence, n_reads): seq_reads = list(reads(seq, dbg_cover=True, N=n_reads)) G = khmer.Nodegraph(K, 1e6, 4) - sp = _oxli.StreamingPartitioner(G) + sp = StreamingPartitioner(G) for read in seq_reads: - sp.consume_sequence(read) + sp.consume(read) assert sp.n_components == 1 @@ -241,11 +241,11 @@ def test_streaming_multicomponents(self, random_sequence, n_components): random.shuffle(seq_reads) G = khmer.Nodegraph(K, 1e6, 4) - sp = _oxli.StreamingPartitioner(G) + sp = StreamingPartitioner(G) for read in seq_reads: assert len(read) >= K - sp.consume_sequence(read) + sp.consume(read) assert sp.n_components == n_components comps = list(sp.components()) @@ -262,11 +262,11 @@ def test_write_components(self, random_sequence, cov, n_components, tmpdir): for _ in range(n_components): seqs.append(random_sequence(exclude=''.join(seqs))) G = khmer.Countgraph(K, 1e6, 4) - sp = _oxli.StreamingPartitioner(G) + sp = StreamingPartitioner(G) for seq in seqs: for _ in range(cov): - sp.consume_sequence(seq) + sp.consume(seq) for seq in seqs: (med, _, _) = G.get_median_count(seq) assert med == cov From 35e0d5a3b0aa29f5a995ca4c34562db96c4dd3b9 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 16 Nov 2016 20:53:51 -0800 Subject: [PATCH 042/185] Add paired-read consume function, split consume into tagging and component finding stages --- khmer/_oxli/_oxli.pxd | 4 +++- khmer/_oxli/partitioning.pyx | 9 +++++++-- lib/partitioning.cc | 35 +++++++++++++++++++++++++++++------ lib/partitioning.hh | 8 +++++++- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index 9c7876f9e6..6202a2a12e 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -150,8 +150,10 @@ cdef extern from "partitioning.hh" namespace "khmer": cdef cppclass CpStreamingPartitioner "khmer::StreamingPartitioner": CpStreamingPartitioner(CpHashtable * ) except +MemoryError - void consume_sequence(string&) except +MemoryError + void consume(string&) except +MemoryError + void consume_pair(string&, string&) except +MemoryError uint64_t consume_fasta(string&) except +MemoryError + void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) void find_connected_tags(queue[CpKmer]&, set[HashIntoType]&, diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 78a43b06da..7bc5e5230e 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -102,10 +102,15 @@ cdef class StreamingPartitioner: self._components = deref(self._this).get_component_set() self.n_consumed = 0 - def consume_sequence(self, sequence): - deref(self._this).consume_sequence(sequence.encode('utf-8')) + def consume(self, sequence): + deref(self._this).consume(sequence.encode('utf-8')) self.n_consumed += 1 + def consume_pair(self, first, second): + deref(self._this).consume_pair(first.encode('utf-8'), + second.encode('utf-8')) + self.n_consumed += 2 + def consume_fasta(self, filename): return deref(self._this).consume_fasta(filename.encode('utf-8')) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 1930594c5e..1928016611 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -83,7 +83,7 @@ uint64_t StreamingPartitioner::consume_fasta(const std::string& filename) bool is_valid = graph->check_and_normalize_read(read.sequence); if (is_valid) { - consume_sequence(read.sequence); + consume(read.sequence); } else { n_invalid++; } @@ -94,7 +94,26 @@ uint64_t StreamingPartitioner::consume_fasta(const std::string& filename) } -void StreamingPartitioner::consume_sequence(const std::string& seq) +void StreamingPartitioner::consume(const std::string& seq) +{ + std::set tags; + consume_and_connect_tags(seq, tags); + create_and_connect_components(tags); +} + + +void StreamingPartitioner::consume_pair(const std::string& first, + const std::string& second) +{ + std::set tags; + consume_and_connect_tags(first, tags); + consume_and_connect_tags(second, tags); + create_and_connect_components(tags); +} + + +void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, + std::set& tags) { /* For the following comments, let G be the set of k-mers * known in the graph before inserting the k-mers R from @@ -181,6 +200,14 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) #endif // Now search for tagged nodes connected to U. find_connected_tags(search_from, tags, seen, false); + } else { + throw khmer_ptr_exception("Hashtable has been deleted."); + } +} + + +void StreamingPartitioner::create_and_connect_components(std::set &tags) +{ // Now resolve components. First, get components from existing tags. ComponentPtrSet found_comps; @@ -227,10 +254,6 @@ void StreamingPartitioner::consume_sequence(const std::string& seq) merge_components(root_comp, found_comps); } } - - } else { - throw khmer_ptr_exception("Hashtable has been deleted."); - } } diff --git a/lib/partitioning.hh b/lib/partitioning.hh index 99f6702941..5a37e084b5 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -196,7 +196,13 @@ class StreamingPartitioner { explicit StreamingPartitioner(Hashtable * graph); - void consume_sequence(const std::string& seq); + void consume(const std::string& seq); + void consume_pair(const std::string& first, + const std::string& second); + void consume_and_connect_tags(const std::string& seq, + std::set& tags); + void create_and_connect_components(std::set& tags); + uint64_t consume_fasta(std::string const &filename); void map_tags_to_component(std::set& tags, ComponentPtr& comp); void find_connected_tags(KmerQueue& node_q, From 3fd044d2dd591c77e2217a448895399cfc952c80 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 16 Nov 2016 20:56:08 -0800 Subject: [PATCH 043/185] Revert to always tagging the start and end k-mers in a read --- lib/partitioning.cc | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 1928016611..63e791fb7e 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -128,22 +128,16 @@ void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, KmerIterator kmers(seq.c_str(), graph->ksize()); unsigned int since = _tag_density / 2 + 1; - std::set tags; std::set seen; std::set intersection; KmerQueue search_from; bool in_known_territory = false; bool found_tag_in_territory = false; - - // First check if we overlap any tags - //Kmer kmer = kmers.next(); - //tags.insert(kmer); //always tag the first k-mer - //bool is_new_kmer = graph->test_and_set_bits(kmer); - //search_from.push(kmer); - + + Kmer kmer; do { - Kmer kmer = kmers.next(); + kmer = kmers.next(); bool is_new_kmer = graph->test_and_set_bits(kmer); bool kmer_tagged = false; @@ -183,16 +177,14 @@ void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, } } while (!kmers.done()); - if (tags.size() == 0) { - // if no tags are found or seeded, add the middle k-mer - uint32_t start = (seq.length() - graph->ksize()) / 2; - std::string kmer = seq.substr(start, graph->ksize()); - tags.insert(graph->hash_dna(kmer.c_str())); - } + // always tag the last k-mer + tags.insert(kmer); + + // now go back and make sure to tag the first k-mer + kmer = kmers.first(); + tags.insert(kmer); - //tags.insert(kmer); // always tag the last k-mer - //search_from.push(kmer); - //intersection.clear(); + intersection.clear(); #if(DEBUG_SP) std::cout << "Done iterating k-mers" << std::endl; From 6e9f909161b7439f4e504225ed8c3fd3b11ac2ef Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 17 Nov 2016 09:22:48 -0800 Subject: [PATCH 044/185] Add tests for paired component merging --- ...on_oxli.py => test_cython_partitioning.py} | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) rename tests/{test_cython_oxli.py => test_cython_partitioning.py} (83%) diff --git a/tests/test_cython_oxli.py b/tests/test_cython_partitioning.py similarity index 83% rename from tests/test_cython_oxli.py rename to tests/test_cython_partitioning.py index 6b116c6f52..ec8d639c3e 100644 --- a/tests/test_cython_oxli.py +++ b/tests/test_cython_partitioning.py @@ -277,3 +277,61 @@ def test_write_components(self, random_sequence, cov, n_components, tmpdir): assert len(results) == n_components for row in results: assert abs(float(row[2])-float(cov)) < 2 + + +class TestStreamingPartitionerPaired: + + def teardown_method(self, method): + # Force garbage to collect. When Python component objects exist and + # their underlying c++ Component objects are destroyed, the Python + # wrapper becomes the sole owner of the pointer. By manually collecting + # garbage between tests we assure that these objects are freed, and we + # can properly test the _n_destroyed property to make sure there are no + # real memory leaks. + gc.collect() + + def test_one_paired_component(self, random_sequence): + first = random_sequence() + second = random_sequence(exclude=first) + + cg = khmer.Countgraph(K, 1e5, 4) + sp = StreamingPartitioner(cg) + sp.consume_pair(first, second) + + assert sp.n_components == 1 + + def test_two_paired_components_merge(self, random_sequence): + comp1 = random_sequence() + comp2 = random_sequence(exclude=comp1) + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = StreamingPartitioner(cg) + + sp.consume(comp1) + assert sp.n_components == 1 + + sp.consume(comp2) + assert sp.n_components == 2 + + sp.consume_pair(comp1, comp2) + assert sp.n_components == 1 + + def test_multi_paired_components_merge(self, random_sequence): + seq1 = random_sequence() + seq2 = random_sequence(exclude=seq1) + seq3 = random_sequence(exclude=seq1+seq2) + + cg = khmer.Nodegraph(K, 1e5, 4) + sp = StreamingPartitioner(cg) + + sp.consume(seq1) + sp.consume(seq2) + sp.consume(seq3) + assert sp.n_components == 3 + + sp.consume_pair(seq1, seq2) + assert sp.n_components == 2 + + sp.consume_pair(seq2, seq3) + assert sp.n_components == 1 + From b58f3d6194e9a6c6a142e31ba14183359802431c Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 17 Nov 2016 15:58:11 -0800 Subject: [PATCH 045/185] Add SplitPairedReader --- khmer/_oxli/streaming.pxd | 9 +++++ khmer/_oxli/streaming.pyx | 70 +++++++++++++++++++++++++++++++++++---- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/khmer/_oxli/streaming.pxd b/khmer/_oxli/streaming.pxd index ec1176af36..762a7b6d76 100644 --- a/khmer/_oxli/streaming.pxd +++ b/khmer/_oxli/streaming.pxd @@ -32,6 +32,15 @@ cdef class FastxParser: cdef Sequence _next(self) +cdef class SplitPairedReader: + + cdef FastxParser left_parser + cdef FastxParser right_parser + cdef readonly int min_length + + cdef tuple _next(self) + + cdef class BrokenPairedReader: cdef FastxParser parser diff --git a/khmer/_oxli/streaming.pyx b/khmer/_oxli/streaming.pyx index 68063f0e61..892d3e831d 100644 --- a/khmer/_oxli/streaming.pyx +++ b/khmer/_oxli/streaming.pyx @@ -37,17 +37,19 @@ cdef class Sequence: def name(self): return deref(self._this).name - @property - def annotations(self): - return deref(self._this).annotations - @property def sequence(self): return deref(self._this).sequence + @property + def annotations(self): + cdef str annotations = deref(self._this).annotations + return annotations if annotations else None + @property def quality(self): - return deref(self._this).quality + cdef str quality = deref(self._this).quality + return quality if quality else None @staticmethod def create(str name, str sequence, str annotations=None, str quality=None): @@ -168,6 +170,60 @@ cdef class FastxParser: seq = self._next() +cdef class SplitPairedReader: + + def __cinit__(self, FastxParser left_parser, + FastxParser right_parser, + int min_length=-1): + + self.left_parser = left_parser + self.right_parser = right_parser + self.min_length = min_length + + def __iter__(self): + cdef Sequence first, second + cdef object err + cdef read_num = 0 + cdef int found + + found, first, second, err = self._next() + while found != 0: + if err is not None: + raise err + + if self.min_length > 0: + if len(first) >= self.min_length or \ + len(second) >= self.min_length: + + yield read_num, True, first, second + else: + yield read_num, True, first, second + + read_num += 2 + found, first, second, err = self._next() + + cdef tuple _next(self): + cdef Sequence first = Sequence(create=True) + cdef Sequence second = Sequence(create=True) + cdef bool left_has_next, right_has_next + + left_has_next = self.left_parser._imprint_next(first) + right_has_next = self.right_parser._imprint_next(second) + + if left_has_next != right_has_next: + err = UnpairedReadsError('Differing lengths of left '\ + 'and right files!') + return -1, None, None, err + + if left_has_next == False: + return 0, None, None, None + + if _check_is_pair(first, second): + return 2, first, second, None + else: + err = UnpairedReadsError(first, second) + return -1, None, None, err + cdef class BrokenPairedReader: def __cinit__(self, FastxParser parser, @@ -242,7 +298,7 @@ cdef class BrokenPairedReader: err = UnpairedReadsError( "Unpaired reads when require_paired is set!", first, second) - return 0, None, None, err + return -1, None, None, err self.record = second return 1, first, None, None else: @@ -250,7 +306,7 @@ cdef class BrokenPairedReader: if self.require_paired: err = UnpairedReadsError("Unpaired reads when require_paired " "is set!", first, None) - return 0, None, None, err + return -1, None, None, err self.record = None return 1, first, None, None From f7d7f9028183889b2c5e66d0f8e006d5d7b00b01 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 17 Nov 2016 15:58:21 -0800 Subject: [PATCH 046/185] Remove extraneous tag clear --- lib/partitioning.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 63e791fb7e..e1610f9e58 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -184,8 +184,6 @@ void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, kmer = kmers.first(); tags.insert(kmer); - intersection.clear(); - #if(DEBUG_SP) std::cout << "Done iterating k-mers" << std::endl; std::cout << tags.size() << " tags in sequence" << std::endl; From cc0e4b1782514f544178fcbcde9894f14b360d49 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 17 Nov 2016 15:59:33 -0800 Subject: [PATCH 047/185] Change name of Component.create to Component.wrap to clarify that it is wrapping an existing pointer --- khmer/_oxli/partitioning.pxd | 5 ++--- khmer/_oxli/partitioning.pyx | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index ca0f67230f..14e40b7a9a 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -5,11 +5,12 @@ from libc.stdint cimport uint32_t, uint8_t, uint64_t from _oxli cimport ComponentPtr, ComponentPtrSet, CpGuardedKmerCompMap from _oxli cimport CpHashtable, CpStreamingPartitioner, BoundedCounterType + cdef class Component: cdef ComponentPtr _this @staticmethod - cdef Component create(ComponentPtr ptr) + cdef Component wrap(ComponentPtr ptr) @staticmethod cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph) @@ -24,5 +25,3 @@ cdef class StreamingPartitioner: cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map cdef CpHashtable * _graph_ptr cdef readonly uint64_t n_consumed - - diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 7bc5e5230e..5796165031 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -56,7 +56,7 @@ cdef class Component: raise NotImplementedError('Operator not available.') @staticmethod - cdef Component create(ComponentPtr ptr): + cdef Component wrap(ComponentPtr ptr): cdef Component comp = Component() comp._this = ptr return comp @@ -120,7 +120,7 @@ cdef class StreamingPartitioner: if compptr == NULL: return None else: - return Component.create(compptr) + return Component.wrap(compptr) def get_nearest_component(self, kmer): cdef ComponentPtr compptr @@ -128,7 +128,7 @@ cdef class StreamingPartitioner: if compptr == NULL: return None else: - return Component.create(compptr) + return Component.wrap(compptr) def components(self): cdef shared_ptr[ComponentPtrSet] locked @@ -136,7 +136,7 @@ cdef class StreamingPartitioner: lockedptr = self._components.lock() if lockedptr: for cmpptr in deref(lockedptr): - yield Component.create(cmpptr) + yield Component.wrap(cmpptr) else: raise MemoryError("Can't locked underlying Component set") @@ -146,7 +146,7 @@ cdef class StreamingPartitioner: lockedptr = self._tag_component_map.lock() if lockedptr: for cpair in deref(lockedptr).data: - yield cpair.first, Component.create(cpair.second) + yield cpair.first, Component.wrap(cpair.second) else: raise MemoryError("Can't locked underlying Component set") From dc0d0a6aaa67e793cd52331562d2aeee4024add9 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 17 Nov 2016 16:00:55 -0800 Subject: [PATCH 048/185] Isolated the partitioner into its own class --- khmer/_oxli/app.pxd | 7 +++ khmer/_oxli/app.pyx | 92 ++++++++++++++++++++++++++++++++++ scripts/partition-streaming.py | 62 ++--------------------- 3 files changed, 102 insertions(+), 59 deletions(-) create mode 100644 khmer/_oxli/app.pxd create mode 100644 khmer/_oxli/app.pyx diff --git a/khmer/_oxli/app.pxd b/khmer/_oxli/app.pxd new file mode 100644 index 0000000000..eb04a818db --- /dev/null +++ b/khmer/_oxli/app.pxd @@ -0,0 +1,7 @@ +from partitioning cimport StreamingPartitioner + +cdef class PartitioningApp: + + cdef object args + cdef object graph + cdef StreamingPartitioner partitioner diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx new file mode 100644 index 0000000000..6b6010e72f --- /dev/null +++ b/khmer/_oxli/app.pyx @@ -0,0 +1,92 @@ +from __future__ import print_function +import argparse +import itertools +import os +import sys + +from .._khmer import Countgraph +from .._khmer import Nodegraph +from khmer.khmer_args import build_counting_args, create_countgraph + +from partitioning cimport StreamingPartitioner, Component +from partitioning import StreamingPartitioner, Component + +from streaming cimport BrokenPairedReader, SplitPairedReader, FastxParser, Sequence +from streaming import BrokenPairedReader, SplitPairedReader, FastxParser, Sequence + +def grouper(n, iterable): + iterable = iter(iterable) + return iter(lambda: list(itertools.islice(iterable, n)), []) + +cdef class PartitioningApp: + + def __init__(self, args=sys.argv[1:]): + self.args = self.parse_args(args) + self.args.write_stats = self.stats_interval > 0 + + self.graph = create_countgraph(self) + self.partitioner = StreamingPartitioner(self.graph) + + def parse_args(self, args): + parser = build_counting_args() + parser.add_argument('--stats-dir', default='component-stats') + parser.add_argument('samples', nargs='+') + parser.add_argument('--pairing-mode', + choices=['split', 'interleaved', 'single'], + default='split') + parser.add_argument('-Z', dest='norm', default=10) + parser.add_argument('--stats-interval', default=0) + + return parser.parse_args(args) + + def write_components(self, folder, n, sample): + sample = os.path.basename(sample) + filename = os.path.join(folder, + '{0}-{1}.stats.csv'.format(sample, n)) + print('# {0}: {1} tags, {2} components.'.format(n, self.partitioner.n_tags, + self.partitioner.n_components)) + print(' writing results to file -> {0}'.format(filename)) + self.partitioner.write_components(filename) + + def run(self): + + if self.args.write_stats > 0: + try: + os.mkdir(self.args.stats_dir) + except OSError as e: + pass + + if self.args.pairing_mode == 'split': + samples = list(grouper(2, self.args.samples)) + for pair in samples: + if len(pair) != 2: + raise ValueError('Must have even number of samples!') + else: + samples = self.args.samples + for group in samples: + if self.args.pairing_mode == 'split': + sample_name = '{0}.{1}'.format(group[0], group[1]) + print('== Starting ({0}) =='.format(sample_name)) + reader = SplitPairedReader(FastxParser(group[0]), + FastxParser(group[1]), + min_length=self.args.ksize) + else: + sample_name = group + print('== Starting {0} =='.format(sample_name)) + reader = BrokenPairedReader(FastxParser(group), min_length=self.args.ksize) + for n, paired, first, second in reader: + + if n % 1000 == 0: + print (n, '...', sep='') + if self.args.write_stats and n > 0 and n % self.args.stats_interval == 0: + self.write_components(self.args.stats_dir, n, sample_name) + + if paired: + self.partitioner.consume_pair(first.sequence, + second.sequence) + else: + self.partitioner.consume(first.sequence) + + self.write_components(self.args.stats_dir, n, sample_name) + + diff --git a/scripts/partition-streaming.py b/scripts/partition-streaming.py index d9b3bdabaa..eb08626979 100644 --- a/scripts/partition-streaming.py +++ b/scripts/partition-streaming.py @@ -1,61 +1,5 @@ -from __future__ import print_function - -import sys -import screed -import os -import khmer -import textwrap -from khmer import khmer_args -from khmer.khmer_args import (build_counting_args, add_loadgraph_args, - report_on_config, info, calculate_graphsize, - sanitize_help, check_argument_range) -from khmer._oxli.partitioning import StreamingPartitioner -import argparse -from khmer.utils import (write_record, broken_paired_reader, ReadBundle) - - -def write_stats(partitioner, folder, n, sample): - sample = os.path.basename(sample) - filename = os.path.join(folder, - '{0}-{1}.stats.csv'.format(sample, n)) - print('# {0}: {1} tags, {2} components.'.format(n, partitioner.n_tags, - partitioner.n_components)) - print(' writing results to file -> {0}'.format(filename)) - partitioner.write_components(filename) - -def main(): - parser = build_counting_args() - parser.add_argument('-stats-dir', default='component_stats') - parser.add_argument('samples', nargs='+') - parser.add_argument('-Z', dest='norm', default=10) - parser.add_argument('--stats-interval', default=50000) - args = parser.parse_args() - - graph = khmer_args.create_countgraph(args) - partitioner = StreamingPartitioner(graph) - - try: - os.mkdir(args.stats_dir) - except OSError as e: - pass - - for sample in args.samples: - print('== Starting {0} =='.format(sample)) - for n, read in enumerate(screed.open(sample)): - if n % 1000 == 0: - print (n, '...', sep='') - if args.stats_interval > 0 and n > 0 and n % args.stats_interval == 0: - write_stats(partitioner, args.stats_dir, n, sample) - cov, _, _ = graph.get_median_count(read.sequence) - if cov < args.norm: - graph.consume(read.sequence) - else: - seq, pos = graph.trim_on_abundance(read.sequence, 2) - if len(seq) < args.ksize: - continue - partitioner.consume_sequence(read.sequence) - - write_stats(partitioner, args.stats_dir, n, sample) +#!/usr/bin/env python +from khmer._oxli.app import PartitioningApp if __name__ == '__main__': - main() + PartitioningApp().run() From 245921df5a805f63f6151aa8d6f9d625f7bb814b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 27 Nov 2016 22:37:20 -0800 Subject: [PATCH 049/185] Update clean to remove __pycache__ folders and cython .so modulefiles --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 463e479fce..d75752747c 100644 --- a/Makefile +++ b/Makefile @@ -132,9 +132,10 @@ clean: FORCE cd lib && $(MAKE) clean || true cd tests && rm -rf khmertest_* || true rm -f $(EXTENSION_MODULE) - rm -f $(CYTHON_MODULE) + @find ./ -type d -name __pycache__ -exec rm -rf {} + + @find ./khmer/ -type f -name *$(MODEXT) -exec rm -f {} + rm -f khmer/*.pyc lib/*.pyc scripts/*.pyc tests/*.pyc oxli/*.pyc \ - sandbox/*.pyc khmer/__pycache__/* sandbox/__pycache__/* khmer/_oxli/*.cpp + sandbox/*.pyc khmer/_oxli/*.cpp ./setup.py clean --all || true rm -f coverage-debug rm -Rf .coverage From b0bda477c85d25b912f2ef471fea1f590120c151 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 27 Nov 2016 22:41:00 -0800 Subject: [PATCH 050/185] Add save methods to Component and StreamingPartitioner --- khmer/_oxli/_oxli.pxd | 4 ++ khmer/_oxli/partitioning.pxd | 8 ++++ khmer/_oxli/partitioning.pyx | 78 ++++++++++++++++++++++++++----- lib/partitioning.cc | 6 +++ lib/partitioning.hh | 5 ++ tests/test_cython_partitioning.py | 27 +++++++++++ 6 files changed, 116 insertions(+), 12 deletions(-) diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index 6202a2a12e..17c88e2bb6 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -127,6 +127,9 @@ cdef extern from "traversal.hh": cdef extern from "partitioning.hh" namespace "khmer": cdef cppclass CpComponent "khmer::Component": + CpComponent() + CpComponent(uint64_t) + const uint64_t component_id set[HashIntoType] tags @@ -154,6 +157,7 @@ cdef extern from "partitioning.hh" namespace "khmer": void consume_pair(string&, string&) except +MemoryError uint64_t consume_fasta(string&) except +MemoryError + void add_component(ComponentPtr comp) void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) void find_connected_tags(queue[CpKmer]&, set[HashIntoType]&, diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index 14e40b7a9a..ae212b689a 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -1,6 +1,7 @@ from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr from libcpp.vector cimport vector from libc.stdint cimport uint32_t, uint8_t, uint64_t +from libc.stdio cimport FILE from _oxli cimport ComponentPtr, ComponentPtrSet, CpGuardedKmerCompMap from _oxli cimport CpHashtable, CpStreamingPartitioner, BoundedCounterType @@ -9,6 +10,8 @@ from _oxli cimport CpHashtable, CpStreamingPartitioner, BoundedCounterType cdef class Component: cdef ComponentPtr _this + cdef void save(self, FILE * fp) + @staticmethod cdef Component wrap(ComponentPtr ptr) @@ -18,10 +21,15 @@ cdef class Component: @staticmethod cdef float _mean_tag_count(ComponentPtr comp, CpHashtable * graph) + @staticmethod + cdef ComponentPtr load(uint64_t component_id, list tags) + cdef class StreamingPartitioner: cdef unique_ptr[CpStreamingPartitioner] _this cdef weak_ptr[ComponentPtrSet] _components cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map cdef CpHashtable * _graph_ptr + cdef object graph cdef readonly uint64_t n_consumed + cdef readonly dict component_dict diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 5796165031..bff9cbec2e 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -1,3 +1,4 @@ +# cython: c_string_type=unicode, c_string_encoding=utf8 import cython from cython.operator cimport dereference as deref, preincrement as inc @@ -7,7 +8,7 @@ from libcpp.vector cimport vector from libcpp.map cimport map from libcpp.set cimport set from libcpp.queue cimport queue -from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr +from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr, make_shared from libcpp.utility cimport pair from libc.stdint cimport uint32_t, uint8_t, uint64_t @@ -55,12 +56,6 @@ cdef class Component: else: raise NotImplementedError('Operator not available.') - @staticmethod - cdef Component wrap(ComponentPtr ptr): - cdef Component comp = Component() - comp._this = ptr - return comp - @staticmethod cdef vector[BoundedCounterType] _tag_counts(ComponentPtr comp, CpHashtable * graph): cdef uint64_t n_tags = deref(comp).get_n_tags() @@ -86,6 +81,34 @@ cdef class Component: acc += deref(graph).get_count(tag) return acc / n_tags + cdef void save(self, FILE* fp): + cdef HashIntoType tag + cdef int i + + fprintf(fp, "{\"component_id\": %llu, \"tags\": [", deref(self._this).component_id) + for i, tag in enumerate(deref(self._this).tags): + if i != 0: + fprintf(fp, ",") + fprintf(fp, "%llu", tag) + fprintf(fp, "]}") + + @staticmethod + cdef ComponentPtr load(uint64_t component_id, list tags): + cdef ComponentPtr comp + cdef HashIntoType tag + cdef int i, N = len(tags) + comp.reset(new CpComponent(component_id)) + for i in range(N): + tag = tags[i] + deref(comp).add_tag(tag) + return comp + + @staticmethod + cdef Component wrap(ComponentPtr ptr): + cdef Component comp = Component() + comp._this = ptr + return comp + cdef class StreamingPartitioner: @@ -93,6 +116,7 @@ cdef class StreamingPartitioner: if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): raise ValueError('Must take an object with Hashtable *') + self.graph = graph cdef CPyHashtable_Object* ptr = graph self._graph_ptr = deref(ptr).hashtable @@ -101,6 +125,7 @@ cdef class StreamingPartitioner: self._tag_component_map = deref(self._this).get_tag_component_map() self._components = deref(self._this).get_component_set() self.n_consumed = 0 + self.component_dict = {comp.component_id: comp for comp in self.components()} def consume(self, sequence): deref(self._this).consume(sequence.encode('utf-8')) @@ -143,18 +168,18 @@ cdef class StreamingPartitioner: def tag_components(self): cdef shared_ptr[CpGuardedKmerCompMap] locked cdef pair[HashIntoType,ComponentPtr] cpair - lockedptr = self._tag_component_map.lock() - if lockedptr: - for cpair in deref(lockedptr).data: + locked = self._tag_component_map.lock() + if locked: + for cpair in deref(locked).data: yield cpair.first, Component.wrap(cpair.second) else: - raise MemoryError("Can't locked underlying Component set") + raise MemoryError("Can't lock underlying Component set") def write_components(self, filename): cdef FILE* fp fp = fopen(filename.encode('utf-8'), 'wb') if fp == NULL: - raise IOError("Can't open file.") + raise IOError('Can\'t open file.') cdef ComponentPtr cmpptr cdef shared_ptr[ComponentPtrSet] lockedptr @@ -168,6 +193,35 @@ cdef class StreamingPartitioner: Component._mean_tag_count(cmpptr, self._graph_ptr)) fclose(fp) + def save(self, filename): + graph_filename = '{0}.graph'.format(filename) + comp_filename = '{0}.json'.format(filename) + bytes_graph_filename = graph_filename.encode('utf-8') + cdef char * c_graph_filename = bytes_graph_filename + self.graph.save(graph_filename) + + cdef FILE* fp = fopen(comp_filename.encode('utf-8'), 'w') + if fp == NULL: + raise IOError('Can\'t open file.') + + fprintf(fp, "{\"graph\": \"%s\",\n\"n_components\": %llu,\n", + c_graph_filename, deref(self._this).get_n_components()) + fprintf(fp, "\"n_tags\": %llu,\n", deref(self._this).get_n_tags()) + fprintf(fp, "\"components\": [\n") + + cdef Component comp + cdef int i + cdef shared_ptr[ComponentPtrSet] locked + locked = self._components.lock() + if locked: + for i, comp in enumerate(self.components()): + if i != 0: + fprintf(fp, ",\n") + comp.save(fp) + fprintf(fp, "\n]}") + fclose(fp) + + property n_components: def __get__(self): return deref(self._this).get_n_components() diff --git a/lib/partitioning.cc b/lib/partitioning.cc index e1610f9e58..59a98574dd 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -111,6 +111,12 @@ void StreamingPartitioner::consume_pair(const std::string& first, create_and_connect_components(tags); } +void StreamingPartitioner::add_component(ComponentPtr comp) +{ + components->insert(comp); + map_tags_to_component(comp->tags, comp); +} + void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, std::set& tags) diff --git a/lib/partitioning.hh b/lib/partitioning.hh index 5a37e084b5..9c7a7cc135 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -131,6 +131,10 @@ class Component { n_created++; } + explicit Component(uint64_t component_id): component_id(component_id) { + n_created++; + } + ~Component() { n_destroyed++; } @@ -205,6 +209,7 @@ class StreamingPartitioner { uint64_t consume_fasta(std::string const &filename); void map_tags_to_component(std::set& tags, ComponentPtr& comp); + void add_component(ComponentPtr comp); void find_connected_tags(KmerQueue& node_q, std::set& found_tags, std::set& seen, diff --git a/tests/test_cython_partitioning.py b/tests/test_cython_partitioning.py index ec8d639c3e..70a421d04d 100644 --- a/tests/test_cython_partitioning.py +++ b/tests/test_cython_partitioning.py @@ -278,6 +278,33 @@ def test_write_components(self, random_sequence, cov, n_components, tmpdir): for row in results: assert abs(float(row[2])-float(cov)) < 2 + @pytest.mark.parametrize("n_components", [1, 10, 50, 100]) + def test_save_partitioner(self, random_sequence, n_components, tmpdir): + import json + out_prefix = str(tmpdir.join('test_save')) + seqs = [] + for _ in range(n_components): + seqs.append(random_sequence(exclude=''.join(seqs))) + G = khmer.Countgraph(K, 1e6, 4) + sp = StreamingPartitioner(G) + for seq in seqs: + sp.consume(seq) + + sp.save(out_prefix) + + with open(out_prefix + '.json') as fp: + print(fp.read()) + fp.seek(0) + result = json.load(fp) + + assert 'graph' in result + assert result['graph'] == out_prefix + '.graph' + assert 'n_components' in result + assert result['n_components'] == n_components + result_comps = {d['component_id']: d for d in result['components']} + for comp in sp.components(): + assert comp.component_id in result_comps + class TestStreamingPartitionerPaired: From 9ec41bfbd6952871a9a8e15caf7ad5410f8fa35f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 27 Nov 2016 22:41:56 -0800 Subject: [PATCH 051/185] Add an option to require matching names for pairs or not --- khmer/_oxli/streaming.pxd | 1 + khmer/_oxli/streaming.pyx | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/khmer/_oxli/streaming.pxd b/khmer/_oxli/streaming.pxd index 762a7b6d76..c746727331 100644 --- a/khmer/_oxli/streaming.pxd +++ b/khmer/_oxli/streaming.pxd @@ -37,6 +37,7 @@ cdef class SplitPairedReader: cdef FastxParser left_parser cdef FastxParser right_parser cdef readonly int min_length + cdef readonly bool force_name_match cdef tuple _next(self) diff --git a/khmer/_oxli/streaming.pyx b/khmer/_oxli/streaming.pyx index 892d3e831d..5acfc7d0fe 100644 --- a/khmer/_oxli/streaming.pyx +++ b/khmer/_oxli/streaming.pyx @@ -174,11 +174,13 @@ cdef class SplitPairedReader: def __cinit__(self, FastxParser left_parser, FastxParser right_parser, - int min_length=-1): + int min_length=-1, + bool force_name_match=False): self.left_parser = left_parser self.right_parser = right_parser self.min_length = min_length + self.force_name_match = force_name_match def __iter__(self): cdef Sequence first, second @@ -218,11 +220,14 @@ cdef class SplitPairedReader: if left_has_next == False: return 0, None, None, None - if _check_is_pair(first, second): - return 2, first, second, None + if self.force_name_match: + if _check_is_pair(first, second): + return 2, first, second, None + else: + err = UnpairedReadsError('', first, second) + return -1, None, None, err else: - err = UnpairedReadsError(first, second) - return -1, None, None, err + return 2, first, second, None cdef class BrokenPairedReader: From 5a8a6ed8e2e8408d8cdb357e69a9cb70197dabb8 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 27 Nov 2016 22:42:29 -0800 Subject: [PATCH 052/185] Modify output names, update reporting interval, expose graph --- khmer/_oxli/app.pxd | 4 ++-- khmer/_oxli/app.pyx | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/khmer/_oxli/app.pxd b/khmer/_oxli/app.pxd index eb04a818db..b77365fceb 100644 --- a/khmer/_oxli/app.pxd +++ b/khmer/_oxli/app.pxd @@ -3,5 +3,5 @@ from partitioning cimport StreamingPartitioner cdef class PartitioningApp: cdef object args - cdef object graph - cdef StreamingPartitioner partitioner + cdef readonly object graph + cdef readonly StreamingPartitioner partitioner diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index 6b6010e72f..dc8330abe7 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -22,9 +22,9 @@ cdef class PartitioningApp: def __init__(self, args=sys.argv[1:]): self.args = self.parse_args(args) - self.args.write_stats = self.stats_interval > 0 + self.args.write_stats = self.args.stats_interval > 0 - self.graph = create_countgraph(self) + self.graph = create_countgraph(self.args) self.partitioner = StreamingPartitioner(self.graph) def parse_args(self, args): @@ -34,15 +34,15 @@ cdef class PartitioningApp: parser.add_argument('--pairing-mode', choices=['split', 'interleaved', 'single'], default='split') - parser.add_argument('-Z', dest='norm', default=10) - parser.add_argument('--stats-interval', default=0) + parser.add_argument('-Z', dest='norm', default=10, type=int) + parser.add_argument('--stats-interval', default=0, type=int) return parser.parse_args(args) def write_components(self, folder, n, sample): sample = os.path.basename(sample) filename = os.path.join(folder, - '{0}-{1}.stats.csv'.format(sample, n)) + '{0}.{1}.stats.csv'.format(n, sample)) print('# {0}: {1} tags, {2} components.'.format(n, self.partitioner.n_tags, self.partitioner.n_components)) print(' writing results to file -> {0}'.format(filename)) @@ -50,7 +50,7 @@ cdef class PartitioningApp: def run(self): - if self.args.write_stats > 0: + if self.args.write_stats: try: os.mkdir(self.args.stats_dir) except OSError as e: @@ -63,6 +63,8 @@ cdef class PartitioningApp: raise ValueError('Must have even number of samples!') else: samples = self.args.samples + + last = 0 for group in samples: if self.args.pairing_mode == 'split': sample_name = '{0}.{1}'.format(group[0], group[1]) @@ -79,14 +81,14 @@ cdef class PartitioningApp: if n % 1000 == 0: print (n, '...', sep='') if self.args.write_stats and n > 0 and n % self.args.stats_interval == 0: - self.write_components(self.args.stats_dir, n, sample_name) + self.write_components(self.args.stats_dir, last+n, sample_name) if paired: self.partitioner.consume_pair(first.sequence, second.sequence) else: self.partitioner.consume(first.sequence) + last = n + self.write_components(self.args.stats_dir, last+n, sample_name) - self.write_components(self.args.stats_dir, n, sample_name) - - + return self.partitioner From 8139102c41381d746c8f22a6e3f97fb46c2f25a2 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 28 Nov 2016 01:15:41 -0800 Subject: [PATCH 053/185] Add a load method for StreamingPartitioner --- khmer/_oxli/partitioning.pyx | 26 ++++++++++++++++++++++++++ tests/test_cython_partitioning.py | 22 ++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index bff9cbec2e..54b4e338c6 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -17,9 +17,11 @@ from libc.limits cimport UINT_MAX from libc.stdint cimport uintptr_t from libc.stdio cimport FILE, fopen, fwrite, fclose, stdout, stderr, fprintf +import json from _oxli cimport * from .._khmer import Countgraph from .._khmer import Nodegraph +from khmer import load_countgraph, load_nodegraph cdef class Component: @@ -221,6 +223,30 @@ cdef class StreamingPartitioner: fprintf(fp, "\n]}") fclose(fp) + @staticmethod + def load(filename): + + with open(filename) as fp: + data = json.load(fp) + + cdef object graph + graph_filename = data['graph'] + try: + graph = load_countgraph(graph_filename) + print('Loading', graph_filename, 'as CountGraph') + except OSError as e: + # maybe it was a nodegraph instead + graph = load_nodegraph(graph_filename) + print('Loading', graph_filename, 'as NodeGraph') + + partitioner = StreamingPartitioner(graph) + cdef ComponentPtr comp_ptr + for comp_info in data['components']: + comp_ptr = Component.load(comp_info['component_id'], + comp_info['tags']) + deref(partitioner._this).add_component(comp_ptr) + return partitioner + property n_components: def __get__(self): diff --git a/tests/test_cython_partitioning.py b/tests/test_cython_partitioning.py index 70a421d04d..0bb60fffa0 100644 --- a/tests/test_cython_partitioning.py +++ b/tests/test_cython_partitioning.py @@ -305,6 +305,28 @@ def test_save_partitioner(self, random_sequence, n_components, tmpdir): for comp in sp.components(): assert comp.component_id in result_comps + @pytest.mark.parametrize("n_components", [1, 10, 50, 100]) + def test_load_partitioner(self, random_sequence, n_components, tmpdir): + import json + out_prefix = str(tmpdir.join('test_save')) + seqs = [] + for _ in range(n_components): + seqs.append(random_sequence(exclude=''.join(seqs))) + G = khmer.Countgraph(K, 1e6, 4) + sp = StreamingPartitioner(G) + for seq in seqs: + sp.consume(seq) + + sp.save(out_prefix) + + sp2 = StreamingPartitioner.load(out_prefix + '.json') + assert sp.n_components == sp2.n_components + for (c1, c2) in zip(sp.components(), sp2.components()): + assert c1 == c2 + assert len(c1) == len(c2) + for t1, t2 in zip(c1, c2): + assert t1 == t2 + class TestStreamingPartitionerPaired: From e41bd4d359bbf3dde26d0869d29bbb68b78ce8a6 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 28 Nov 2016 01:27:33 -0800 Subject: [PATCH 054/185] Convert component_dict to property, update StreamingPartitioner properties from deprecated syntax --- khmer/_oxli/partitioning.pxd | 1 - khmer/_oxli/partitioning.pyx | 16 +++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index ae212b689a..95769d7141 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -32,4 +32,3 @@ cdef class StreamingPartitioner: cdef CpHashtable * _graph_ptr cdef object graph cdef readonly uint64_t n_consumed - cdef readonly dict component_dict diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 54b4e338c6..9e3236a566 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -127,7 +127,6 @@ cdef class StreamingPartitioner: self._tag_component_map = deref(self._this).get_tag_component_map() self._components = deref(self._this).get_component_set() self.n_consumed = 0 - self.component_dict = {comp.component_id: comp for comp in self.components()} def consume(self, sequence): deref(self._this).consume(sequence.encode('utf-8')) @@ -247,11 +246,14 @@ cdef class StreamingPartitioner: deref(partitioner._this).add_component(comp_ptr) return partitioner + @property + def component_dict(self): + return {comp.component_id: comp for comp in self.components()} - property n_components: - def __get__(self): - return deref(self._this).get_n_components() + @property + def n_components(self): + return deref(self._this).get_n_components() - property n_tags: - def __get__(self): - return deref(self._this).get_n_tags() + @property + def n_tags(self): + return deref(self._this).get_n_tags() From 7492b665db7b90211d75ad43d080977b616a690b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 28 Nov 2016 03:55:56 -0800 Subject: [PATCH 055/185] Add option to save partitioner to partitiong-streaming.py' --- khmer/_oxli/app.pyx | 3 +++ scripts/partition-streaming.py | 0 2 files changed, 3 insertions(+) mode change 100644 => 100755 scripts/partition-streaming.py diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index dc8330abe7..be1224fd25 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -31,6 +31,7 @@ cdef class PartitioningApp: parser = build_counting_args() parser.add_argument('--stats-dir', default='component-stats') parser.add_argument('samples', nargs='+') + parser.add_argument('--save', default=None) parser.add_argument('--pairing-mode', choices=['split', 'interleaved', 'single'], default='split') @@ -91,4 +92,6 @@ cdef class PartitioningApp: last = n self.write_components(self.args.stats_dir, last+n, sample_name) + if self.args.save is not None: + self.partitioner.save(self.args.save) return self.partitioner diff --git a/scripts/partition-streaming.py b/scripts/partition-streaming.py old mode 100644 new mode 100755 From 0a0ed67d1e0eb36b4d959a954c1901dd9554799b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 2 Dec 2016 16:41:04 -0800 Subject: [PATCH 056/185] Add a CompactingAssembler for building compact de bruijn graphs --- lib/assembler.cc | 32 ++++++++++++++++++++++++++++++++ lib/assembler.hh | 12 ++++++++++++ lib/traversal.cc | 33 +++++++++++++++++++++++++++++++++ lib/traversal.hh | 27 ++++++++++++++++++++++++++- 4 files changed, 103 insertions(+), 1 deletion(-) diff --git a/lib/assembler.cc b/lib/assembler.cc index 1bfc2262a4..2cbe4b40fc 100644 --- a/lib/assembler.cc +++ b/lib/assembler.cc @@ -163,6 +163,38 @@ const return contig; } +/******************************** + * Compacting Assembler + ********************************/ + + +std::string CompactingAssembler::assemble_right(const Kmer seed_kmer, + const Hashtable * stop_bf) +const +{ + std::list node_filters; + if (stop_bf) { + node_filters.push_back(get_stop_bf_filter(stop_bf)); + } + + CompactingAT cursor(graph, seed_kmer, node_filters); + return _assemble_directed(cursor); +} + + +std::string CompactingAssembler::assemble_left(const Kmer seed_kmer, + const Hashtable * stop_bf) +const +{ + std::list node_filters; + if (stop_bf) { + node_filters.push_back(get_stop_bf_filter(stop_bf)); + } + + CompactingAT cursor(graph, seed_kmer, node_filters); + return _assemble_directed(cursor); +} + /******************************** * Labeled Assembly diff --git a/lib/assembler.hh b/lib/assembler.hh index f5fc3a4bbe..09ff3212e1 100644 --- a/lib/assembler.hh +++ b/lib/assembler.hh @@ -53,6 +53,7 @@ namespace khmer class Hashtable; class LabelHash; + /** * \class LinearAssembler * @@ -106,6 +107,17 @@ std::string LinearAssembler::_assemble_directed(AssemblerTraverser &cursor) const; +class CompactingAssembler: public LinearAssembler +{ + + std::string assemble_right(const Kmer seed_kmer, + const Hashtable * stop_bf = 0) const; + + std::string assemble_left(const Kmer seed_kmer, + const Hashtable * stop_bf = 0) const; +}; + + /** * \class SimpleLabeledAssembler * diff --git a/lib/traversal.cc b/lib/traversal.cc index dc2de55131..5d72b382c2 100644 --- a/lib/traversal.cc +++ b/lib/traversal.cc @@ -340,6 +340,37 @@ char NonLoopingAT::next_symbol() return AssemblerTraverser::next_symbol(); } +/****************************************** + * CompactingAT + ******************************************/ + +template +CompactingAT::CompactingAT(const Hashtable * ht, + Kmer start_kmer, + KmerFilterList filters) : + AssemblerTraverser(ht, start_kmer, filters), traverser(ht) +{ +} + +template<> +char CompactingAT::next_symbol() +{ + if (traverser.degree_left(this->cursor) > 1) { + return '\0'; + } + return AssemblerTraverser::next_symbol(); +} + + +template<> +char CompactingAT::next_symbol() +{ + if (traverser.degree_right(this->cursor) > 1) { + return '\0'; + } + return AssemblerTraverser::next_symbol(); +} + template class NodeGatherer; template class NodeGatherer; template class NodeCursor; @@ -348,6 +379,8 @@ template class AssemblerTraverser; template class AssemblerTraverser; template class NonLoopingAT; template class NonLoopingAT; +template class CompactingAT; +template class CompactingAT; } // namespace khmer diff --git a/lib/traversal.hh b/lib/traversal.hh index 6eab0f3e52..5c6cddc177 100644 --- a/lib/traversal.hh +++ b/lib/traversal.hh @@ -87,6 +87,11 @@ public: explicit NodeGatherer(const Hashtable * ht, KmerFilter filter); + WordLength ksize() + { + return graph->ksize(); + } + /** * @brief Push a new filter on to the filter stack. */ @@ -297,8 +302,28 @@ public: Kmer start_kmer, KmerFilterList filters, SeenSet * visited); + + virtual char next_symbol(); +}; + + +template +class CompactingAT: public AssemblerTraverser +{ +protected: + + Traverser traverser; + +public: + + explicit CompactingAT(const Hashtable * ht, + Kmer start_kmer, + KmerFilterList filters); + virtual char next_symbol(); + }; -} +} //namespace khmer + #endif From f2d2c30394cfcc4ae8992c04b9530d06d9f2ad00 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 2 Dec 2016 16:41:25 -0800 Subject: [PATCH 057/185] Add pxd files to MANIFEST to distribute cython bindings --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 7162f8de8e..37773bb3e5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include versioneer.py MANIFEST.in CITATION CONTRIBUTING.md Doxyfile.in include LICENSE TODO .ycm_extra_conf.py recursive-include lib *.hh *.cc [Mm]akefile* get_version.py recursive-include khmer *.hh *.cc *.cpp +recursive-include khmer/_oxli *.pxd recursive-include third-party *.cc *.1 *.xsl README* sample* words* *.sh *.c recursive-include third-party manual* [Mm]akefile* *.pl *.dsp CHANGES *.txt *.h recursive-include third-party ChangeLog FAQ INDEX configure *.xsl From 825e23e3955570b75f3de24a0e7d7092040b9539 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 2 Dec 2016 16:42:39 -0800 Subject: [PATCH 058/185] Always merge into largest component --- lib/partitioning.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 59a98574dd..3ac2618ef5 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -238,8 +238,14 @@ void StreamingPartitioner::create_and_connect_components(std::set components->insert(new_comp); map_tags_to_component(tags, new_comp); } else { - // get the first component + // Choose the largest component as the root + // We want to minimize tag copying ComponentPtr root_comp = *(found_comps.begin()); + for (auto other : found_comps) { + if (other->get_n_tags() > root_comp->get_n_tags()) { + root_comp = other; + } + } #if(DEBUG_SP) std::cout << "Merge into: " << *root_comp << std::endl; #endif From f250009b5df546f688b4802f27598695bb947906 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 2 Dec 2016 16:43:14 -0800 Subject: [PATCH 059/185] Add locking to Partitioner --- lib/partitioning.cc | 6 +++++- lib/partitioning.hh | 43 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 3ac2618ef5..8db298c9af 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -29,7 +29,7 @@ inline std::ostream& operator<< (std::ostream& stream, Component& comp) { StreamingPartitioner::StreamingPartitioner(Hashtable * graph) : - graph(graph), _tag_density(DEFAULT_TAG_DENSITY) + graph(graph), _tag_density(DEFAULT_TAG_DENSITY), components_lock(0) { //if (auto graphptr = graph.lock()) { @@ -98,7 +98,9 @@ void StreamingPartitioner::consume(const std::string& seq) { std::set tags; consume_and_connect_tags(seq, tags); + acquire_components(); create_and_connect_components(tags); + release_components(); } @@ -108,7 +110,9 @@ void StreamingPartitioner::consume_pair(const std::string& first, std::set tags; consume_and_connect_tags(first, tags); consume_and_connect_tags(second, tags); + acquire_components(); create_and_connect_components(tags); + release_components(); } void StreamingPartitioner::add_component(ComponentPtr comp) diff --git a/lib/partitioning.hh b/lib/partitioning.hh index 9c7a7cc135..3c6339a28b 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -59,6 +59,10 @@ namespace khmer template class GuardedKmerMap { + private: + + uint32_t lock; + public: // Filter should be owned exclusively by GuardedKmerMap @@ -67,7 +71,7 @@ class GuardedKmerMap { explicit GuardedKmerMap(WordLength ksize, unsigned short n_tables, - uint64_t max_table_size) + uint64_t max_table_size): lock(0) { std::vector table_sizes = get_n_primes_near_x(n_tables, max_table_size); filter = std::unique_ptr(new Hashbits(ksize, table_sizes)); @@ -84,11 +88,31 @@ class GuardedKmerMap { return NULL; } + T get_threadsafe(HashIntoType kmer) const { + if (filter->get_count(kmer)) { + acquire_lock(); + auto search = data.find(kmer); + if (search != data.end()) { + release_lock(); + return search->second; + } + release_lock(); + } + + return NULL; + } + void set(HashIntoType kmer, T item) { filter->count(kmer); data[kmer] = item; } + void set_threadsafe(HashIntoType kmer, T item) { + acquire_lock(); + set(kmer, item); + release_lock(); + } + bool contains(HashIntoType kmer) const { return get(kmer) != NULL; } @@ -97,6 +121,14 @@ class GuardedKmerMap { return data.size(); } + inline void acquire_lock() { + while(!__sync_bool_compare_and_swap( &lock, 0, 1)); + } + + inline void release_lock() { + __sync_bool_compare_and_swap( &lock, 1, 0); + } + }; @@ -195,6 +227,7 @@ class StreamingPartitioner { // We should exclusively own tag_component_map. std::shared_ptr tag_component_map; std::shared_ptr components; + uint32_t components_lock; public: @@ -238,6 +271,14 @@ class StreamingPartitioner { std::weak_ptr get_tag_component_map() const { return std::weak_ptr(tag_component_map); } + + inline void acquire_components() { + while(!__sync_bool_compare_and_swap( &components_lock, 0, 1)); + } + + inline void release_components() { + __sync_bool_compare_and_swap( &components_lock, 1, 0); + } }; From d7257d191d296029858a227cee880f07b7fb8d59 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 2 Dec 2016 16:44:45 -0800 Subject: [PATCH 060/185] cdef some more variable types and fix stats writing at end of run --- khmer/_oxli/app.pyx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index be1224fd25..8395cd3d27 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -8,6 +8,8 @@ from .._khmer import Countgraph from .._khmer import Nodegraph from khmer.khmer_args import build_counting_args, create_countgraph +from libcpp cimport bool + from partitioning cimport StreamingPartitioner, Component from partitioning import StreamingPartitioner, Component @@ -64,7 +66,10 @@ cdef class PartitioningApp: raise ValueError('Must have even number of samples!') else: samples = self.args.samples - + + cdef int n + cdef bool paired + cdef Sequence first, second last = 0 for group in samples: if self.args.pairing_mode == 'split': @@ -90,8 +95,10 @@ cdef class PartitioningApp: else: self.partitioner.consume(first.sequence) last = n - self.write_components(self.args.stats_dir, last+n, sample_name) + if self.args.write_stats: + self.write_components(self.args.stats_dir, last+n, sample_name) if self.args.save is not None: self.partitioner.save(self.args.save) + return self.partitioner From 62707894ca9e31a0a09b0acb7010f9f9d11de100 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 2 Dec 2016 16:47:03 -0800 Subject: [PATCH 061/185] Expose LinearAssembler with cython bindings, flesh out Traverser --- khmer/_oxli/_oxli.pxd | 26 ++++++++++- khmer/_oxli/_oxli.pyx | 15 +++++++ khmer/_oxli/assembly.pxd | 20 +++++++++ khmer/_oxli/assembly.pyx | 61 +++++++++++++++++++++++++ khmer/_oxli/traversal.pyx | 93 +++++++++++++++++++++++++++++++++------ 5 files changed, 199 insertions(+), 16 deletions(-) create mode 100644 khmer/_oxli/_oxli.pyx create mode 100644 khmer/_oxli/assembly.pxd create mode 100644 khmer/_oxli/assembly.pyx diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index 17c88e2bb6..c24e7f93a9 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -45,6 +45,7 @@ cdef extern from "kmer_hash.hh" namespace "khmer": HashIntoType _hash(const string, const WordLength) string _revhash(HashIntoType, WordLength) + string _revcomp(const string&) ######################################################################## @@ -93,6 +94,8 @@ cdef extern from "hashtable.hh" namespace "khmer": const BoundedCounterType get_count(HashIntoType) const const WordLength ksize() const +cdef CpHashtable * get_hashtable_ptr(object graph) + ######################################################################## # @@ -116,6 +119,25 @@ cdef extern from "traversal.hh": uint32_t degree_left(const CpKmer&) const uint32_t degree_right(const CpKmer&) const +######################################################################## +# +# Assembler: wrapper for assembler.hh. +# +######################################################################## + + +cdef extern from "assembler.hh": + cdef cppclass CpLinearAssembler "khmer::LinearAssembler": + CpLinearAssembler(CpHashtable *) + + string assemble(const CpKmer, const CpHashtable *) const + string assemble_left(const CpKmer, const CpHashtable *) const + string assemble_right(const CpKmer, const CpHashtable *) const + + string assemble(const CpKmer) const + string assemble_left(const CpKmer) const + string assemble_right(const CpKmer) const + ######################################################################## # @@ -153,8 +175,8 @@ cdef extern from "partitioning.hh" namespace "khmer": cdef cppclass CpStreamingPartitioner "khmer::StreamingPartitioner": CpStreamingPartitioner(CpHashtable * ) except +MemoryError - void consume(string&) except +MemoryError - void consume_pair(string&, string&) except +MemoryError + void consume(string&) nogil except +MemoryError + void consume_pair(string&, string&) nogil except +MemoryError uint64_t consume_fasta(string&) except +MemoryError void add_component(ComponentPtr comp) diff --git a/khmer/_oxli/_oxli.pyx b/khmer/_oxli/_oxli.pyx new file mode 100644 index 0000000000..9cc9ee43ce --- /dev/null +++ b/khmer/_oxli/_oxli.pyx @@ -0,0 +1,15 @@ +# cython: c_string_type=unicode, c_string_encoding=utf8 +import cython +from cython.operator cimport dereference as deref + +from .._khmer import Countgraph +from .._khmer import Nodegraph + + +cdef CpHashtable * get_hashtable_ptr(object graph): + if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): + return NULL + + cdef CPyHashtable_Object* ptr = graph + return deref(ptr).hashtable + diff --git a/khmer/_oxli/assembly.pxd b/khmer/_oxli/assembly.pxd new file mode 100644 index 0000000000..2a3db02d4b --- /dev/null +++ b/khmer/_oxli/assembly.pxd @@ -0,0 +1,20 @@ +from libcpp.memory cimport unique_ptr +from libcpp.string import string + +from _oxli cimport CpLinearAssembler, CpHashtable, CpKmer +from hashing cimport Kmer + +cdef class LinearAssembler: + cdef unique_ptr[CpLinearAssembler] _this + + cdef public object graph + cdef CpHashtable * _graph_ptr + + cdef public object stop_filter + cdef CpHashtable * _stop_filter_ptr + + cdef str _assemble(self, Kmer start) + cdef str _assemble_left(self, Kmer start) + cdef str _assemble_right(self, Kmer start) + + diff --git a/khmer/_oxli/assembly.pyx b/khmer/_oxli/assembly.pyx new file mode 100644 index 0000000000..4d0da27ee5 --- /dev/null +++ b/khmer/_oxli/assembly.pyx @@ -0,0 +1,61 @@ +# cython: c_string_type=unicode, c_string_encoding=utf8 +import cython +from cython.operator cimport dereference as deref + +from _oxli import * +from _oxli cimport CpHashtable, CpLinearAssembler, CpKmer, get_hashtable_ptr + +cdef class LinearAssembler: + + def __cinit__(self, graph, stop_filter=None): + self._graph_ptr = get_hashtable_ptr(graph) + self.graph = graph + self.set_stop_filter(stop_filter=stop_filter) + + self._this.reset(new CpLinearAssembler(self._graph_ptr)) + + def set_stop_filter(self, stop_filter=None): + self.stop_filter = stop_filter + if stop_filter is not None: + self._stop_filter_ptr = get_hashtable_ptr(stop_filter) + else: + self._stop_filter_ptr = NULL + + + cdef str _assemble(self, Kmer kmer): + if self.stop_filter is None: + return deref(self._this).assemble(deref(kmer._this)) + else: + return deref(self._this).assemble(deref(kmer._this), self._stop_filter_ptr) + + def assemble(self, seed): + if isinstance(seed, Kmer): + return self._assemble(seed) + else: + return self._assemble(Kmer(seed)) + + + cdef str _assemble_left(self, Kmer kmer): + if self.stop_filter is None: + return deref(self._this).assemble_left(deref(kmer._this)) + else: + return deref(self._this).assemble_left(deref(kmer._this), self._stop_filter_ptr) + + def assemble_left(self, seed): + if isinstance(seed, Kmer): + return self._assemble_left(seed) + else: + return self._assemble_left(Kmer(seed)) + + + cdef str _assemble_right(self, Kmer kmer): + if self.stop_filter is None: + return deref(self._this).assemble_right(deref(kmer._this)) + else: + return deref(self._this).assemble_right(deref(kmer._this), self._stop_filter_ptr) + + def assemble_right(self, seed): + if isinstance(seed, Kmer): + return self._assemble_right(seed) + else: + return self._assemble_right(Kmer(seed)) diff --git a/khmer/_oxli/traversal.pyx b/khmer/_oxli/traversal.pyx index 9dca040012..ef82205074 100644 --- a/khmer/_oxli/traversal.pyx +++ b/khmer/_oxli/traversal.pyx @@ -5,8 +5,8 @@ from .._khmer import Countgraph from .._khmer import Nodegraph from _oxli cimport * -cimport hashing -import hashing +from hashing cimport Kmer +from hashing import Kmer cdef class Traverser: @@ -18,22 +18,87 @@ cdef class Traverser: self._graph_ptr = deref(ptr).hashtable self._this.reset(new CpTraverser(self._graph_ptr)) - ''' - def left_neighbors(self, Kmer kmer): + + def neighbors(self, node): + cdef Kmer kmer + if not isinstance(node, Kmer): + kmer = Kmer(node) + else: + kmer = node + cdef KmerQueue kmer_q - cdef Kmer neighbor - deref(self._this).traverse_left(kmer, kmer_q) - for neighbor in kmer_q: - yield neighbor - ''' + cdef CpKmer neighbor + cdef Kmer pyneighbor + deref(self._this).traverse(deref(kmer._this), kmer_q) + while(kmer_q.empty() == 0): + neighbor = kmer_q.front() + pyneighbor = Kmer.wrap(new CpKmer(neighbor), deref(self._graph_ptr).ksize()) + if pyneighbor.is_forward != kmer.is_forward: + pyneighbor.reverse_complement() + yield pyneighbor + kmer_q.pop() + + def right_neighbors(self, node): + cdef Kmer kmer + if not isinstance(node, Kmer): + kmer = Kmer(node) + else: + kmer = node - def right_neighbors(self, hashing.Kmer kmer): cdef KmerQueue kmer_q cdef CpKmer neighbor - cdef CpKmer* ptr + cdef Kmer pyneighbor deref(self._this).traverse_right(deref(kmer._this), kmer_q) while(kmer_q.empty() == 0): - neighbor = kmer_q.back() - ptr = new CpKmer(neighbor) - yield hashing.Kmer.wrap(ptr, deref(self._graph_ptr).ksize()) + neighbor = kmer_q.front() + pyneighbor = Kmer.wrap(new CpKmer(neighbor), deref(self._graph_ptr).ksize()) + if pyneighbor.is_forward != kmer.is_forward: + pyneighbor.reverse_complement() + yield pyneighbor kmer_q.pop() + + def left_neighbors(self, node): + cdef Kmer kmer + if not isinstance(node, Kmer): + kmer = Kmer(node) + else: + kmer = node + + cdef KmerQueue kmer_q + cdef CpKmer neighbor + cdef Kmer pyneighbor + deref(self._this).traverse_left(deref(kmer._this), kmer_q) + while(kmer_q.empty() == 0): + neighbor = kmer_q.front() + pyneighbor = Kmer.wrap(new CpKmer(neighbor), deref(self._graph_ptr).ksize()) + if pyneighbor.is_forward != kmer.is_forward: + pyneighbor.reverse_complement() + yield pyneighbor + kmer_q.pop() + + def degree(self, node): + cdef Kmer kmer + if not isinstance(node, Kmer): + kmer = Kmer(node) + else: + kmer = node + + return deref(self._this).degree(deref(kmer._this)) + + def left_degree(self, node): + cdef Kmer kmer + if not isinstance(node, Kmer): + kmer = Kmer(node) + else: + kmer = node + + return deref(self._this).degree_left(deref(kmer._this)) + + def right_degree(self, node): + cdef Kmer kmer + if not isinstance(node, Kmer): + kmer = Kmer(node) + else: + kmer = node + + return deref(self._this).degree_right(deref(kmer._this)) From c91165eec239ffc770ce40c37af3288e835f2a6a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 2 Dec 2016 16:47:49 -0800 Subject: [PATCH 062/185] Fix load issue with relative paths, fix enumeration of tags --- khmer/_oxli/partitioning.pxd | 2 +- khmer/_oxli/partitioning.pyx | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index 95769d7141..2397a524d7 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -30,5 +30,5 @@ cdef class StreamingPartitioner: cdef weak_ptr[ComponentPtrSet] _components cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map cdef CpHashtable * _graph_ptr - cdef object graph + cdef readonly object graph cdef readonly uint64_t n_consumed diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 9e3236a566..a6d6aa0f4f 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -18,6 +18,8 @@ from libc.stdint cimport uintptr_t from libc.stdio cimport FILE, fopen, fwrite, fclose, stdout, stderr, fprintf import json +import os + from _oxli cimport * from .._khmer import Countgraph from .._khmer import Nodegraph @@ -65,7 +67,7 @@ cdef class Component: counts = vector[BoundedCounterType](n_tags) cdef int idx cdef uint64_t tag - for idx, tag in deref(comp).tags: + for idx, tag in enumerate(deref(comp).tags): counts[idx] = deref(graph).get_count(tag) return counts @@ -227,9 +229,10 @@ cdef class StreamingPartitioner: with open(filename) as fp: data = json.load(fp) + directory = os.path.dirname(filename) cdef object graph - graph_filename = data['graph'] + graph_filename = os.path.join(directory, data['graph']) try: graph = load_countgraph(graph_filename) print('Loading', graph_filename, 'as CountGraph') From 9cb2fcaeb045fd9e690815a67852a8a6297e5c3b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 2 Dec 2016 16:52:14 -0800 Subject: [PATCH 063/185] Add reverse complementation options to Kmer wrapper --- khmer/_oxli/hashing.pxd | 5 ----- khmer/_oxli/hashing.pyx | 20 ++++++++++++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/khmer/_oxli/hashing.pxd b/khmer/_oxli/hashing.pxd index c7ebe7d695..67543ad832 100644 --- a/khmer/_oxli/hashing.pxd +++ b/khmer/_oxli/hashing.pxd @@ -7,8 +7,3 @@ cdef class Kmer: @staticmethod cdef Kmer wrap(CpKmer * cpkmer, WordLength K) - - @staticmethod - cdef Kmer create(HashIntoType tag, WordLength K) - - diff --git a/khmer/_oxli/hashing.pyx b/khmer/_oxli/hashing.pyx index 0a48ea64d3..cf66fdf25c 100644 --- a/khmer/_oxli/hashing.pyx +++ b/khmer/_oxli/hashing.pyx @@ -1,9 +1,11 @@ +# cython: c_string_type=unicode, c_string_encoding=utf8 + from libcpp.string cimport string from libcpp.memory cimport make_shared from libc.stdint cimport uint64_t from cython.operator cimport dereference as deref -from _oxli cimport _revhash +from _oxli cimport _revhash, HashIntoType, _revcomp cdef class Kmer: @@ -35,16 +37,26 @@ cdef class Kmer: def kmer_u(self): return deref(self._this).kmer_u + def reverse_complement(self): + cdef HashIntoType tmp = deref(self._this).kmer_f + deref(self._this).kmer_f = deref(self._this).kmer_r + deref(self._this).kmer_r = tmp + self.kmer = _revcomp(self.kmer.encode('utf-8')) + + @property + def is_forward(self): + return deref(self._this).is_forward() + @staticmethod cdef Kmer wrap(CpKmer * cpkmer, WordLength K): cdef Kmer kmer = Kmer() kmer._this.reset(cpkmer) - kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') + kmer.kmer = _revhash(kmer.kmer_u, K) return kmer @staticmethod - cdef Kmer create(HashIntoType tag, WordLength K): + def create(HashIntoType tag, WordLength K): cdef Kmer kmer = Kmer() deref(kmer._this).set_from_unique_hash(tag, K) - kmer.kmer = _revhash(kmer.kmer_u, K).decode('utf-8') + kmer.kmer = _revhash(kmer.kmer_u, K) return kmer From b17c4e9417f8028fabc18d6cbc10d4f4c650f119 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 4 Dec 2016 02:04:20 -0800 Subject: [PATCH 064/185] Add initial tests for cython wrapped assemblers --- tests/test_cython_assembly.py | 96 +++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/test_cython_assembly.py diff --git a/tests/test_cython_assembly.py b/tests/test_cython_assembly.py new file mode 100644 index 0000000000..1bb4bb060a --- /dev/null +++ b/tests/test_cython_assembly.py @@ -0,0 +1,96 @@ +# -*- coding: UTF-8 -*- +# +# This file is part of khmer, https://github.com/dib-lab/khmer/, and is +# Copyright (C) 2010-2015, Michigan State University. +# Copyright (C) 2015-2016, The Regents of the University of California. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of the Michigan State University nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Contact: khmer-project@idyll.org +# pylint: disable=missing-docstring,protected-access,no-member,invalid-name + +from __future__ import print_function +from __future__ import absolute_import + +import itertools +import random + +import khmer +from khmer.khmer_args import estimate_optimal_with_K_and_f as optimal_fp +from khmer import ReadParser +from khmer import reverse_complement as revcomp +from . import khmer_tst_utils as utils +from khmer._oxli.assembly import LinearAssembler, CompactingAssembler + +import pytest +import screed + +from .graph_features import * +from .graph_features import K + +def teardown(): + utils.cleanup() + + +class TestNonBranching: + + @pytest.mark.parametrize("assembler", [LinearAssembler, CompactingAssembler]) + def test_all_start_positions(self, linear_structure, assembler): + # assemble entire contig, starting from wherever + graph, contig = linear_structure + asm = assembler(graph) + + for start in range(0, len(contig), 150): + path = asm.assemble(contig[start:start + K]) + assert utils._equals_rc(path, contig), start + + @pytest.mark.parametrize("assembler", [LinearAssembler, CompactingAssembler]) + def test_all_left_to_beginning(self, linear_structure, assembler): + # assemble directed left + graph, contig = linear_structure + asm = assembler(graph) + + for start in range(0, len(contig), 150): + path = asm.assemble_left(contig[start:start + K]) + print(path, ', ', contig[:start]) + assert utils._equals_rc(path, contig[:start+K]), start + + @pytest.mark.parametrize("assembler", [LinearAssembler, CompactingAssembler]) + def test_all_right_to_end(self, linear_structure, assembler): + # assemble directed right + graph, contig = linear_structure + asm = assembler(graph) + + for start in range(0, len(contig), 150): + path = asm.assemble_right(contig[start:start + K]) + print(path, ', ', contig[:start]) + assert utils._equals_rc(path, contig[start:]), start + + From e65cd1c919a8912463c7d680b559a3712d07c6e9 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 13 Dec 2016 17:00:07 -0800 Subject: [PATCH 065/185] add compacting assembler and tests --- khmer/_oxli/_oxli.pxd | 5 ++++- khmer/_oxli/assembly.pxd | 6 +++++- khmer/_oxli/assembly.pyx | 20 +++++++++++++++++--- lib/assembler.cc | 12 ++++++------ lib/assembler.hh | 10 +++++++--- tests/test_cython_assembly.py | 28 ++++++++++++++++++++++++---- 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index c24e7f93a9..21732b8602 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -126,7 +126,7 @@ cdef extern from "traversal.hh": ######################################################################## -cdef extern from "assembler.hh": +cdef extern from "assembler.hh" namespace "khmer": cdef cppclass CpLinearAssembler "khmer::LinearAssembler": CpLinearAssembler(CpHashtable *) @@ -138,6 +138,9 @@ cdef extern from "assembler.hh": string assemble_left(const CpKmer) const string assemble_right(const CpKmer) const + cdef cppclass CpCompactingAssembler(CpLinearAssembler): + CpCompactingAssembler(CpHashtable *) + ######################################################################## # diff --git a/khmer/_oxli/assembly.pxd b/khmer/_oxli/assembly.pxd index 2a3db02d4b..51f2cd4184 100644 --- a/khmer/_oxli/assembly.pxd +++ b/khmer/_oxli/assembly.pxd @@ -1,7 +1,7 @@ from libcpp.memory cimport unique_ptr from libcpp.string import string -from _oxli cimport CpLinearAssembler, CpHashtable, CpKmer +from _oxli cimport CpLinearAssembler, CpCompactingAssembler, CpHashtable, CpKmer from hashing cimport Kmer cdef class LinearAssembler: @@ -18,3 +18,7 @@ cdef class LinearAssembler: cdef str _assemble_right(self, Kmer start) +cdef class CompactingAssembler(LinearAssembler): + pass + #cdef unique_ptr[CpCompactingAssembler] _this + diff --git a/khmer/_oxli/assembly.pyx b/khmer/_oxli/assembly.pyx index 4d0da27ee5..69b67eb0d7 100644 --- a/khmer/_oxli/assembly.pyx +++ b/khmer/_oxli/assembly.pyx @@ -3,7 +3,7 @@ import cython from cython.operator cimport dereference as deref from _oxli import * -from _oxli cimport CpHashtable, CpLinearAssembler, CpKmer, get_hashtable_ptr +from _oxli cimport CpHashtable, CpLinearAssembler, CpCompactingAssembler, CpKmer, get_hashtable_ptr cdef class LinearAssembler: @@ -11,8 +11,9 @@ cdef class LinearAssembler: self._graph_ptr = get_hashtable_ptr(graph) self.graph = graph self.set_stop_filter(stop_filter=stop_filter) - - self._this.reset(new CpLinearAssembler(self._graph_ptr)) + + if type(self) is LinearAssembler: + self._this.reset(new CpLinearAssembler(self._graph_ptr)) def set_stop_filter(self, stop_filter=None): self.stop_filter = stop_filter @@ -59,3 +60,16 @@ cdef class LinearAssembler: return self._assemble_right(seed) else: return self._assemble_right(Kmer(seed)) + +cdef class CompactingAssembler(LinearAssembler): + + def __cinit__(self, graph, stop_filter=None): + self._graph_ptr = get_hashtable_ptr(graph) + self.graph = graph + self.set_stop_filter(stop_filter=stop_filter) + + cdef CpCompactingAssembler* ptr = new CpCompactingAssembler(self._graph_ptr) + if type(self) is CompactingAssembler: + self._this.reset( ptr) + + diff --git a/lib/assembler.cc b/lib/assembler.cc index 2cbe4b40fc..4d117317c8 100644 --- a/lib/assembler.cc +++ b/lib/assembler.cc @@ -107,8 +107,8 @@ const } template <> -std::string LinearAssembler::_assemble_directed(AssemblerTraverser& - cursor) +std::string LinearAssembler:: +_assemble_directed(AssemblerTraverser& cursor) const { std::string contig = cursor.cursor.get_string_rep(_ksize); @@ -138,8 +138,8 @@ const } template<> -std::string LinearAssembler::_assemble_directed -(AssemblerTraverser& cursor) +std::string LinearAssembler:: +_assemble_directed(AssemblerTraverser& cursor) const { std::string contig = cursor.cursor.get_string_rep(_ksize); @@ -178,7 +178,7 @@ const } CompactingAT cursor(graph, seed_kmer, node_filters); - return _assemble_directed(cursor); + return LinearAssembler::_assemble_directed(cursor); } @@ -192,7 +192,7 @@ const } CompactingAT cursor(graph, seed_kmer, node_filters); - return _assemble_directed(cursor); + return LinearAssembler::_assemble_directed(cursor); } diff --git a/lib/assembler.hh b/lib/assembler.hh index 09ff3212e1..fddb7712ca 100644 --- a/lib/assembler.hh +++ b/lib/assembler.hh @@ -83,13 +83,13 @@ public: explicit LinearAssembler(const Hashtable * ht); - std::string assemble(const Kmer seed_kmer, + virtual std::string assemble(const Kmer seed_kmer, const Hashtable * stop_bf = 0) const; - std::string assemble_right(const Kmer seed_kmer, + virtual std::string assemble_right(const Kmer seed_kmer, const Hashtable * stop_bf = 0) const; - std::string assemble_left(const Kmer seed_kmer, + virtual std::string assemble_left(const Kmer seed_kmer, const Hashtable * stop_bf = 0) const; template @@ -109,6 +109,9 @@ std::string LinearAssembler::_assemble_directed(AssemblerTraverser class CompactingAssembler: public LinearAssembler { +public: + + explicit CompactingAssembler(const Hashtable * ht) : LinearAssembler(ht) {} std::string assemble_right(const Kmer seed_kmer, const Hashtable * stop_bf = 0) const; @@ -116,6 +119,7 @@ class CompactingAssembler: public LinearAssembler std::string assemble_left(const Kmer seed_kmer, const Hashtable * stop_bf = 0) const; }; +typedef CompactingAssembler CpCompactingAssembler; /** diff --git a/tests/test_cython_assembly.py b/tests/test_cython_assembly.py index 1bb4bb060a..451367c120 100644 --- a/tests/test_cython_assembly.py +++ b/tests/test_cython_assembly.py @@ -58,10 +58,9 @@ def teardown(): utils.cleanup() - +@pytest.mark.parametrize("assembler", [LinearAssembler, CompactingAssembler]) class TestNonBranching: - @pytest.mark.parametrize("assembler", [LinearAssembler, CompactingAssembler]) def test_all_start_positions(self, linear_structure, assembler): # assemble entire contig, starting from wherever graph, contig = linear_structure @@ -71,7 +70,6 @@ def test_all_start_positions(self, linear_structure, assembler): path = asm.assemble(contig[start:start + K]) assert utils._equals_rc(path, contig), start - @pytest.mark.parametrize("assembler", [LinearAssembler, CompactingAssembler]) def test_all_left_to_beginning(self, linear_structure, assembler): # assemble directed left graph, contig = linear_structure @@ -82,7 +80,6 @@ def test_all_left_to_beginning(self, linear_structure, assembler): print(path, ', ', contig[:start]) assert utils._equals_rc(path, contig[:start+K]), start - @pytest.mark.parametrize("assembler", [LinearAssembler, CompactingAssembler]) def test_all_right_to_end(self, linear_structure, assembler): # assemble directed right graph, contig = linear_structure @@ -94,3 +91,26 @@ def test_all_right_to_end(self, linear_structure, assembler): assert utils._equals_rc(path, contig[start:]), start +class TestCompactingAssembler: + + def test_beginning_to_branch_right(self, right_tip_structure): + # assemble from beginning of contig, up until branch point + graph, contig, L, HDN, R, tip = right_tip_structure + asm = CompactingAssembler(graph) + path = asm.assemble(contig[0:K]) + + assert len(path) == HDN.pos + K + assert utils._equals_rc(path, contig[:len(path)]) + + def test_end_to_branch_right(self, right_tip_structure): + # in the LinearAsembler, this would continue all the way + # to the beginning. The CompactingAssembler does an extra + # check of the node degree in the reverse direction. + graph, contig, L, HDN, R, tip = right_tip_structure + asm = CompactingAssembler(graph) + path = asm.assemble(contig[-K:]) + + assert len(path) == len(contig) - HDN.pos + assert utils._equals_rc(path, contig[HDN.pos:]) + + From be2f16688978503ba13d91bc81ff0b0b2fc49a0a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 9 Jan 2017 17:12:41 -0800 Subject: [PATCH 066/185] add tag-density, track new kmers --- khmer/_oxli/_oxli.pxd | 6 +++-- khmer/_oxli/app.pyx | 24 ++++++++++------- khmer/_oxli/partitioning.pxd | 1 + khmer/_oxli/partitioning.pyx | 14 ++++++---- lib/partitioning.cc | 51 +++++++++++++++++++++++------------- lib/partitioning.hh | 14 +++++++--- 6 files changed, 72 insertions(+), 38 deletions(-) diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index 21732b8602..5c94be27d0 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -177,9 +177,10 @@ cdef extern from "partitioning.hh" namespace "khmer": cdef cppclass CpStreamingPartitioner "khmer::StreamingPartitioner": CpStreamingPartitioner(CpHashtable * ) except +MemoryError + CpStreamingPartitioner(CpHashtable *, uint32_t) except +MemoryError - void consume(string&) nogil except +MemoryError - void consume_pair(string&, string&) nogil except +MemoryError + uint64_t consume(string&) nogil except +MemoryError + uint64_t consume_pair(string&, string&) nogil except +MemoryError uint64_t consume_fasta(string&) except +MemoryError void add_component(ComponentPtr comp) @@ -189,6 +190,7 @@ cdef extern from "partitioning.hh" namespace "khmer": set[HashIntoType]&) except +MemoryError uint64_t get_n_components() const uint64_t get_n_tags() const + uint64_t get_n_consumed() const ComponentPtr get_tag_component(string&) const ComponentPtr get_nearest_component(string&) const diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index 8395cd3d27..0c0ff9a983 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -27,7 +27,7 @@ cdef class PartitioningApp: self.args.write_stats = self.args.stats_interval > 0 self.graph = create_countgraph(self.args) - self.partitioner = StreamingPartitioner(self.graph) + self.partitioner = StreamingPartitioner(self.graph, tag_density=self.args.tag_density) def parse_args(self, args): parser = build_counting_args() @@ -39,10 +39,11 @@ cdef class PartitioningApp: default='split') parser.add_argument('-Z', dest='norm', default=10, type=int) parser.add_argument('--stats-interval', default=0, type=int) + parser.add_argument('--tag-density', default=None, type=int) return parser.parse_args(args) - def write_components(self, folder, n, sample): + def write_components(self, folder, n, sample, new_kmers): sample = os.path.basename(sample) filename = os.path.join(folder, '{0}.{1}.stats.csv'.format(n, sample)) @@ -50,6 +51,9 @@ cdef class PartitioningApp: self.partitioner.n_components)) print(' writing results to file -> {0}'.format(filename)) self.partitioner.write_components(filename) + with open(os.path.join(folder, 'global-stats.csv'), 'a') as fp: + fp.write('{0}, {1}, {2}, {3}\n'.format(n, self.partitioner.n_components, + self.partitioner.n_tags, new_kmers)) def run(self): @@ -70,6 +74,7 @@ cdef class PartitioningApp: cdef int n cdef bool paired cdef Sequence first, second + cdef int new_kmers = 0 last = 0 for group in samples: if self.args.pairing_mode == 'split': @@ -84,19 +89,20 @@ cdef class PartitioningApp: reader = BrokenPairedReader(FastxParser(group), min_length=self.args.ksize) for n, paired, first, second in reader: - if n % 1000 == 0: - print (n, '...', sep='') + if n % 10000 == 0: + print (n, self.partitioner.n_components, self.partitioner.n_tags) if self.args.write_stats and n > 0 and n % self.args.stats_interval == 0: - self.write_components(self.args.stats_dir, last+n, sample_name) - + self.write_components(self.args.stats_dir, last+n, sample_name, new_kmers) + new_kmers = 0 if paired: - self.partitioner.consume_pair(first.sequence, + new_kmers += self.partitioner.consume_pair(first.sequence, second.sequence) else: - self.partitioner.consume(first.sequence) + new_kmers += self.partitioner.consume(first.sequence) last = n if self.args.write_stats: - self.write_components(self.args.stats_dir, last+n, sample_name) + self.write_components(self.args.stats_dir, last, sample_name, new_kmers) + new_kmers = 0 if self.args.save is not None: self.partitioner.save(self.args.save) diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index 2397a524d7..5cb5ce02ae 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -31,4 +31,5 @@ cdef class StreamingPartitioner: cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map cdef CpHashtable * _graph_ptr cdef readonly object graph + cdef readonly object tag_density cdef readonly uint64_t n_consumed diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index a6d6aa0f4f..b0ab4015d4 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -116,7 +116,7 @@ cdef class Component: cdef class StreamingPartitioner: - def __cinit__(self, graph): + def __cinit__(self, graph, tag_density=None): if not (isinstance(graph, Countgraph) or isinstance(graph, Nodegraph)): raise ValueError('Must take an object with Hashtable *') @@ -124,20 +124,24 @@ cdef class StreamingPartitioner: cdef CPyHashtable_Object* ptr = graph self._graph_ptr = deref(ptr).hashtable - self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) + if tag_density is None: + self._this.reset(new CpStreamingPartitioner(self._graph_ptr)) + else: + self._this.reset(new CpStreamingPartitioner(self._graph_ptr, tag_density)) self._tag_component_map = deref(self._this).get_tag_component_map() self._components = deref(self._this).get_component_set() self.n_consumed = 0 def consume(self, sequence): - deref(self._this).consume(sequence.encode('utf-8')) self.n_consumed += 1 + return deref(self._this).consume(sequence.encode('utf-8')) def consume_pair(self, first, second): - deref(self._this).consume_pair(first.encode('utf-8'), - second.encode('utf-8')) self.n_consumed += 2 + return deref(self._this).consume_pair(first.encode('utf-8'), + second.encode('utf-8')) + def consume_fasta(self, filename): return deref(self._this).consume_fasta(filename.encode('utf-8')) diff --git a/lib/partitioning.cc b/lib/partitioning.cc index 8db298c9af..f91c4a47fd 100644 --- a/lib/partitioning.cc +++ b/lib/partitioning.cc @@ -28,8 +28,8 @@ inline std::ostream& operator<< (std::ostream& stream, Component& comp) { } -StreamingPartitioner::StreamingPartitioner(Hashtable * graph) : - graph(graph), _tag_density(DEFAULT_TAG_DENSITY), components_lock(0) +StreamingPartitioner::StreamingPartitioner(Hashtable * graph, uint32_t tag_density) : + graph(graph), _tag_density(tag_density), components_lock(0), n_consumed(0) { //if (auto graphptr = graph.lock()) { @@ -94,25 +94,27 @@ uint64_t StreamingPartitioner::consume_fasta(const std::string& filename) } -void StreamingPartitioner::consume(const std::string& seq) +uint64_t StreamingPartitioner::consume(const std::string& seq) { std::set tags; - consume_and_connect_tags(seq, tags); - acquire_components(); + uint64_t n_new = consume_and_connect_tags(seq, tags); + //acquire_components(); create_and_connect_components(tags); - release_components(); + //release_components(); + return n_new; } -void StreamingPartitioner::consume_pair(const std::string& first, +uint64_t StreamingPartitioner::consume_pair(const std::string& first, const std::string& second) { std::set tags; - consume_and_connect_tags(first, tags); - consume_and_connect_tags(second, tags); - acquire_components(); + uint64_t n_new = consume_and_connect_tags(first, tags); + n_new += consume_and_connect_tags(second, tags); + //acquire_components(); create_and_connect_components(tags); - release_components(); + //release_components(); + return n_new; } void StreamingPartitioner::add_component(ComponentPtr comp) @@ -122,7 +124,7 @@ void StreamingPartitioner::add_component(ComponentPtr comp) } -void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, +uint64_t StreamingPartitioner::consume_and_connect_tags(const std::string& seq, std::set& tags) { /* For the following comments, let G be the set of k-mers @@ -134,12 +136,15 @@ void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, #if(SP_DEBUG) std::cout << "Consume sequence." << std::endl; #endif + uint64_t n_new = 0; + ++n_consumed; + if(graph != NULL) { KmerIterator kmers(seq.c_str(), graph->ksize()); unsigned int since = _tag_density / 2 + 1; std::set seen; - std::set intersection; + KmerSet intersection; KmerQueue search_from; bool in_known_territory = false; @@ -158,13 +163,18 @@ void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, // If we had found a tag in the U&G component we just // left, add the component to the seen set. seen.insert(intersection.begin(), intersection.end()); - } + } /*else { + for (auto km : intersection) { + search_from.push(km); + } + }*/ intersection.clear(); search_from.push(kmer); in_known_territory = false; found_tag_in_territory = false; ++since; + ++n_new; } else { // Keep track of connected components in U&G: when we exit // this component, if there is a tag, we will want to add its nodes @@ -188,11 +198,14 @@ void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, } while (!kmers.done()); // always tag the last k-mer - tags.insert(kmer); + if (since >= _tag_density / 2) { + tags.insert(kmer); + } + search_from.push(kmer); - // now go back and make sure to tag the first k-mer + // now go back and make sure to search from the first k-mer kmer = kmers.first(); - tags.insert(kmer); + search_from.push(kmer); #if(DEBUG_SP) std::cout << "Done iterating k-mers" << std::endl; @@ -203,6 +216,8 @@ void StreamingPartitioner::consume_and_connect_tags(const std::string& seq, } else { throw khmer_ptr_exception("Hashtable has been deleted."); } + + return n_new; } @@ -330,7 +345,7 @@ void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, std::queue breadth_q(std::deque(node_q.size(), 0)); unsigned int cur_breadth = 0; - const unsigned int max_breadth = (2 * _tag_density) + 1; + const unsigned int max_breadth = _tag_density + 1; unsigned int total = 0; unsigned int nfound = 0; diff --git a/lib/partitioning.hh b/lib/partitioning.hh index 3c6339a28b..c2fde9517e 100644 --- a/lib/partitioning.hh +++ b/lib/partitioning.hh @@ -228,15 +228,17 @@ class StreamingPartitioner { std::shared_ptr tag_component_map; std::shared_ptr components; uint32_t components_lock; + uint64_t n_consumed; public: - explicit StreamingPartitioner(Hashtable * graph); + explicit StreamingPartitioner(Hashtable * graph, + uint32_t tag_density=DEFAULT_TAG_DENSITY); - void consume(const std::string& seq); - void consume_pair(const std::string& first, + uint64_t consume(const std::string& seq); + uint64_t consume_pair(const std::string& first, const std::string& second); - void consume_and_connect_tags(const std::string& seq, + uint64_t consume_and_connect_tags(const std::string& seq, std::set& tags); void create_and_connect_components(std::set& tags); @@ -256,6 +258,10 @@ class StreamingPartitioner { return tag_component_map->size(); } + uint64_t get_n_consumed() const { + return n_consumed; + } + void merge_components(ComponentPtr& root, ComponentPtrSet& comps); ComponentPtr get_tag_component(HashIntoType tag) const; From 5db005fd355bee4c143b62f8f980d2b9d41ab0e8 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 17 Jan 2017 10:57:50 -0800 Subject: [PATCH 067/185] Modify partitioning test --- tests/test_cython_partitioning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cython_partitioning.py b/tests/test_cython_partitioning.py index 0bb60fffa0..5434b29d7f 100644 --- a/tests/test_cython_partitioning.py +++ b/tests/test_cython_partitioning.py @@ -251,7 +251,7 @@ def test_streaming_multicomponents(self, random_sequence, n_components): comps = list(sp.components()) comp = comps[0] assert len(comps) == n_components - assert sp.n_components == (comp._n_created - comp._n_destroyed) + #assert sp.n_components == (comp._n_created - comp._n_destroyed) assert sp.n_consumed == len(seq_reads) @pytest.mark.parametrize("n_components", list(range(1,101, 20))) From faa10c8644d58023aaa2f31f0a39cb7e6b889302 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 7 Feb 2017 15:54:41 -0800 Subject: [PATCH 068/185] Use c++ prime function --- khmer/__init__.py | 43 +------------------------------------------ khmer/_oxli/utils.pxd | 8 ++++++++ khmer/_oxli/utils.pyx | 11 +++++++++++ lib/hashtable.hh | 5 ++++- 4 files changed, 24 insertions(+), 43 deletions(-) create mode 100644 khmer/_oxli/utils.pxd create mode 100644 khmer/_oxli/utils.pyx diff --git a/khmer/__init__.py b/khmer/__init__.py index 8fa301dbf6..e52d733b7b 100644 --- a/khmer/__init__.py +++ b/khmer/__init__.py @@ -73,6 +73,7 @@ # scripts/{abundance-dist-single,load-into-counting}.py from khmer import _oxli +from khmer._oxli.utils import get_n_primes_near_x import sys @@ -224,48 +225,6 @@ def calc_expected_collisions(graph, force=False, max_false_pos=.2): return fp_all -def is_prime(number): - """Check if a number is prime.""" - if number < 2: - return False - if number == 2: - return True - if number % 2 == 0: - return False - for _ in range(3, int(number ** 0.5) + 1, 2): - if number % _ == 0: - return False - return True - - -def get_n_primes_near_x(number, target): - """Backward-find primes smaller than target. - - Step backwards until a number of primes (other than 2) have been - found that are smaller than the target and return them. - - Keyword arguments: - number -- the number of primes to find - target -- the number to step backwards from - """ - if target == 1 and number == 1: - return [1] - - primes = [] - i = target - 1 - if i % 2 == 0: - i -= 1 - while len(primes) != number and i > 0: - if is_prime(i): - primes.append(int(i)) - i -= 2 - - if len(primes) != number: - raise RuntimeError("unable to find %d prime numbers < %d" % (number, - target)) - - return primes - # Expose the cpython objects with __new__ implementations. # These constructors add the functionality provided by the existing diff --git a/khmer/_oxli/utils.pxd b/khmer/_oxli/utils.pxd new file mode 100644 index 0000000000..3ff7d0a07e --- /dev/null +++ b/khmer/_oxli/utils.pxd @@ -0,0 +1,8 @@ +from libcpp.vector cimport vector +from libc.stdint cimport uint32_t, uint64_t +from libcpp cimport bool + + +cdef extern from "hashtable.hh" namespace "khmer": + cdef bool _is_prime "khmer::is_prime" (uint64_t n) + cdef vector[uint64_t] _get_n_primes_near_x "khmer::get_n_primes_near_x" (uint32_t, uint64_t) diff --git a/khmer/_oxli/utils.pyx b/khmer/_oxli/utils.pyx new file mode 100644 index 0000000000..f8c6c2a7a7 --- /dev/null +++ b/khmer/_oxli/utils.pyx @@ -0,0 +1,11 @@ + +def is_prime(n): + return _is_prime(n) + + +def get_n_primes_near_x(n_primes, x): + primes = _get_n_primes_near_x(n_primes, x) + if len(primes) != n_primes: + msg = "unable to find {0} prime numbers < {1}".format(n_primes, x) + raise RuntimeError(msg) + return primes diff --git a/lib/hashtable.hh b/lib/hashtable.hh index 8b215020d2..4490aaf622 100644 --- a/lib/hashtable.hh +++ b/lib/hashtable.hh @@ -83,7 +83,7 @@ inline bool is_prime(uint64_t n) if (n % 2 == 0) { return false; } - for (unsigned long long i=3; i < sqrt(n); i += 2) { + for (unsigned long long i=3; i < sqrt(n) + 1; i += 2) { if (n % i == 0) { return false; } @@ -108,6 +108,9 @@ inline std::vector get_n_primes_near_x(uint32_t n, uint64_t x) if (is_prime(i)) { primes.push_back(i); } + if (i == 1) { + break; + } i -= 2; } From 29febdaf10e5e78d187dd34a4a687c4e0ff2a80b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 8 Feb 2017 23:26:27 -0800 Subject: [PATCH 069/185] Add add method to cython binding --- khmer/_oxli/_oxli.pxd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/khmer/_oxli/_oxli.pxd b/khmer/_oxli/_oxli.pxd index eb5152eac5..575db81012 100644 --- a/khmer/_oxli/_oxli.pxd +++ b/khmer/_oxli/_oxli.pxd @@ -128,6 +128,8 @@ cdef extern from "hashtable.hh" namespace "khmer": HashIntoType hash_dna_top_strand(const char *) const HashIntoType hash_dna_bottom_strand(const char *) const string unhash_dna(HashIntoType) const + bool add(const char *) + bool add(HashIntoType) void count(const char *) void count(HashIntoType) const BoundedCounterType get_count(const char *) const From ed518f5db261b0233f789b5c548199b9b8e18b13 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 22 Feb 2017 18:43:02 -0800 Subject: [PATCH 070/185] Add citations to partitioning script --- khmer/_oxli/app.pyx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index ef79fa37e6..d959945d0e 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -7,6 +7,8 @@ import sys from .._khmer import Countgraph from .._khmer import Nodegraph from khmer.khmer_args import build_counting_args, create_countgraph +from khmer.khmer_logger import (configure_logging, log_info, log_error, + log_warn) from libcpp cimport bool @@ -30,7 +32,8 @@ cdef class PartitioningApp: self.partitioner = StreamingPartitioner(self.graph, tag_density=self.args.tag_density) def parse_args(self, args): - parser = build_counting_args() + parser = build_counting_args(descr='Partition a sample', + citations=['counting', 'SeqAn']) parser.add_argument('--stats-dir', default='component-stats') parser.add_argument('samples', nargs='+') parser.add_argument('--save', default=None) From 4038b9bff0ea30e0087f6ea41a59b1ad3f8115a1 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 28 Feb 2017 18:36:14 -0800 Subject: [PATCH 071/185] Add partitioning to lib Makefile --- src/oxli/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/oxli/Makefile b/src/oxli/Makefile index 681d5be9d6..dea6209c58 100644 --- a/src/oxli/Makefile +++ b/src/oxli/Makefile @@ -235,7 +235,8 @@ LIBOXLI_OBJS= \ assembler.o \ alphabets.o \ murmur3.o \ - storage.o + storage.o \ + partitioning.o PRECOMILE_OBJS ?= PRECLEAN_TARGS ?= @@ -267,7 +268,8 @@ HEADERS= \ kmer_filters.hh \ assembler.hh \ alphabets.hh \ - storage.hh + storage.hh \ + partitioning.hh OXLI_HEADERS = $(addprefix ../../include/oxli/,$(HEADERS)) # START OF RULES # From c98a3312043d75260049215639e561e66a9a2ec1 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 1 Mar 2017 17:46:18 -0800 Subject: [PATCH 072/185] Change output format --- khmer/_oxli/app.pyx | 73 +++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index d959945d0e..9fd9573e76 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -1,6 +1,10 @@ +# -*- coding: UTF-8 -*- +# cython: c_string_type=unicode, c_string_encoding=utf8 + from __future__ import print_function import argparse import itertools +import json import os import sys @@ -17,6 +21,7 @@ from partitioning import StreamingPartitioner, Component from parsing cimport BrokenPairedReader, SplitPairedReader, FastxParser, Sequence from parsing import BrokenPairedReader, SplitPairedReader, FastxParser, Sequence +from utils cimport _bstring def grouper(n, iterable): iterable = iter(iterable) @@ -26,7 +31,7 @@ cdef class PartitioningApp: def __init__(self, args=sys.argv[1:]): self.args = self.parse_args(args) - self.args.write_stats = self.args.stats_interval > 0 + self.args.write_results = self.args.output_interval > 0 self.graph = create_countgraph(self.args) self.partitioner = StreamingPartitioner(self.graph, tag_density=self.args.tag_density) @@ -34,37 +39,53 @@ cdef class PartitioningApp: def parse_args(self, args): parser = build_counting_args(descr='Partition a sample', citations=['counting', 'SeqAn']) - parser.add_argument('--stats-dir', default='component-stats') + parser.add_argument('--output-dir', default='partitioned') parser.add_argument('samples', nargs='+') - parser.add_argument('--save', default=None) + parser.add_argument('--save', action='store_true', default=False) parser.add_argument('--pairing-mode', choices=['split', 'interleaved', 'single'], default='split') parser.add_argument('-Z', dest='norm', default=10, type=int) - parser.add_argument('--stats-interval', default=0, type=int) + parser.add_argument('--output-interval', default=0, type=int) parser.add_argument('--tag-density', default=None, type=int) return parser.parse_args(args) - def write_components(self, folder, n, sample, new_kmers): - sample = os.path.basename(sample) - filename = os.path.join(folder, - '{0}.{1}.stats.csv'.format(n, sample)) + def write_results(self, folder, n, new_kmers): + filename = os.path.join(folder, '{0}.csv'.format(n)) print('# {0}: {1} tags, {2} components.'.format(n, self.partitioner.n_tags, self.partitioner.n_components)) print(' writing results to file -> {0}'.format(filename)) self.partitioner.write_components(filename) - with open(os.path.join(folder, 'global-stats.csv'), 'a') as fp: + with open(os.path.join(folder, 'global.csv'), 'a') as fp: fp.write('{0}, {1}, {2}, {3}\n'.format(n, self.partitioner.n_components, self.partitioner.n_tags, new_kmers)) + def prep_results_dir(self): + try: + os.mkdir(self.args.output_dir) + except OSError as e: + pass + + if self.args.save: + self.args.save = os.path.join(self.args.output_dir, 'partitioner') + + def write_meta(self, n_sequences, n_kmers): + meta = {'samples': self.args.samples, + 'pairing': self.args.pairing_mode, + 'K': self.args.ksize, + 'tag-density': self.args.tag_density, + 'n_sequences': n_sequences, + 'n_unique_kmers': n_kmers} + if self.args.save: + meta['partitioner'] = self.args.save + + with open(os.path.join(self.args.output_dir, 'meta'), 'w') as fp: + json.dump(meta, fp, indent=4) + def run(self): - if self.args.write_stats: - try: - os.mkdir(self.args.stats_dir) - except OSError as e: - pass + self.prep_results_dir() if self.args.pairing_mode == 'split': samples = list(grouper(2, self.args.samples)) @@ -75,9 +96,12 @@ cdef class PartitioningApp: samples = self.args.samples cdef int n + cdef int n_sequences = 0 cdef bool paired cdef Sequence first, second cdef int new_kmers = 0 + cdef int n_kmers = 0 + cdef int print_interval = self.args.output_interval if self.args.write_results else 10000 last = 0 for group in samples: if self.args.pairing_mode == 'split': @@ -92,22 +116,27 @@ cdef class PartitioningApp: reader = BrokenPairedReader(FastxParser(group), min_length=self.args.ksize) for n, paired, first, second in reader: - if n % 10000 == 0: + if n % print_interval == 0: print (n, self.partitioner.n_components, self.partitioner.n_tags) - if self.args.write_stats and n > 0 and n % self.args.stats_interval == 0: - self.write_components(self.args.stats_dir, last+n, sample_name, new_kmers) - new_kmers = 0 + if self.args.write_results and n > 0 and n % self.args.output_interval == 0: + self.write_results(self.args.output_dir, last+n, new_kmers) + total_kmers += new_kmers + n_kmers = 0 if paired: new_kmers += self.partitioner.consume_pair(first.sequence, - second.sequence) + second.sequence) else: new_kmers += self.partitioner.consume(first.sequence) last = n - if self.args.write_stats: - self.write_components(self.args.stats_dir, last, sample_name, new_kmers) + n_sequences += last + if self.args.write_results: + self.write_components(self.args.output_dir, last, new_kmers) + n_kmers += new_kmers new_kmers = 0 - if self.args.save is not None: + if self.args.save: self.partitioner.save(self.args.save) + self.write_meta(n_sequences, n_kmers) + return self.partitioner From 98fe61aa1fe59906843396952c593a0014efbfb6 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 1 Mar 2017 21:01:11 -0500 Subject: [PATCH 073/185] Fix wrong var name --- khmer/_oxli/app.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index 9fd9573e76..e802ba0cb7 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -120,8 +120,8 @@ cdef class PartitioningApp: print (n, self.partitioner.n_components, self.partitioner.n_tags) if self.args.write_results and n > 0 and n % self.args.output_interval == 0: self.write_results(self.args.output_dir, last+n, new_kmers) - total_kmers += new_kmers - n_kmers = 0 + n_kmers += new_kmers + new_kmers = 0 if paired: new_kmers += self.partitioner.consume_pair(first.sequence, second.sequence) From 665c4a1734591c584f682067052b533954616f88 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 8 Mar 2017 15:34:13 -0800 Subject: [PATCH 074/185] Add a tag density getter --- include/oxli/partitioning.hh | 4 ++++ khmer/_oxli/partitioning.pxd | 1 - khmer/_oxli/partitioning.pyx | 4 ++++ khmer/_oxli/wrapper.pxd | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index 5705aa9cec..312c4add1e 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -261,6 +261,10 @@ class StreamingPartitioner { return n_consumed; } + uint32_t get_tag_density() const { + return _tag_density; + } + void merge_components(ComponentPtr& root, ComponentPtrSet& comps); ComponentPtr get_tag_component(HashIntoType tag) const; diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index 770d422620..40ebebb731 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -31,5 +31,4 @@ cdef class StreamingPartitioner: cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map cdef CpHashgraph * _graph_ptr cdef readonly object graph - cdef readonly object tag_density cdef readonly uint64_t n_consumed diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 0c0b6adcd2..80d2f94670 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -261,3 +261,7 @@ cdef class StreamingPartitioner: @property def n_tags(self): return deref(self._this).get_n_tags() + + @property + def tag_density(self): + return deref(self._this).get_tag_density() diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index 60bead2ff7..966cff7e70 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -464,6 +464,7 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": uint64_t get_n_components() const uint64_t get_n_tags() const uint64_t get_n_consumed() const + uint32_t get_tag_density() const ComponentPtr get_tag_component(string&) const ComponentPtr get_nearest_component(string&) const From 9c694ad3b32af14fc40f5fb74a657711813b3d81 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 8 Mar 2017 15:34:36 -0800 Subject: [PATCH 075/185] Finish update of output format --- khmer/_oxli/app.pyx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index 9fd9573e76..9ba3fc0886 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -70,13 +70,13 @@ cdef class PartitioningApp: if self.args.save: self.args.save = os.path.join(self.args.output_dir, 'partitioner') - def write_meta(self, n_sequences, n_kmers): + def write_meta(self, n_sequences, total_kmers): meta = {'samples': self.args.samples, 'pairing': self.args.pairing_mode, 'K': self.args.ksize, - 'tag-density': self.args.tag_density, + 'tag-density': self.partitioner.tag_density, 'n_sequences': n_sequences, - 'n_unique_kmers': n_kmers} + 'n_unique_kmers': total_kmers} if self.args.save: meta['partitioner'] = self.args.save @@ -100,7 +100,7 @@ cdef class PartitioningApp: cdef bool paired cdef Sequence first, second cdef int new_kmers = 0 - cdef int n_kmers = 0 + cdef int total_kmers = 0 cdef int print_interval = self.args.output_interval if self.args.write_results else 10000 last = 0 for group in samples: @@ -121,7 +121,7 @@ cdef class PartitioningApp: if self.args.write_results and n > 0 and n % self.args.output_interval == 0: self.write_results(self.args.output_dir, last+n, new_kmers) total_kmers += new_kmers - n_kmers = 0 + new_kmers = 0 if paired: new_kmers += self.partitioner.consume_pair(first.sequence, second.sequence) @@ -130,13 +130,13 @@ cdef class PartitioningApp: last = n n_sequences += last if self.args.write_results: - self.write_components(self.args.output_dir, last, new_kmers) - n_kmers += new_kmers + self.write_results(self.args.output_dir, last, new_kmers) + total_kmers += new_kmers new_kmers = 0 if self.args.save: self.partitioner.save(self.args.save) - self.write_meta(n_sequences, n_kmers) + self.write_meta(n_sequences, total_kmers) return self.partitioner From f971cc6e3e7ff85dce03363c40228a10a76e7edc Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 8 Mar 2017 16:16:03 -0800 Subject: [PATCH 076/185] Move sample-reads-randompy to cythonized reader --- khmer/utils.py | 9 +++++++-- scripts/sample-reads-randomly.py | 31 +++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/khmer/utils.py b/khmer/utils.py index affab9b95f..32d85a28a9 100644 --- a/khmer/utils.py +++ b/khmer/utils.py @@ -36,6 +36,7 @@ from __future__ import print_function, unicode_literals from khmer._oxli.parsing import (check_is_left, check_is_right, check_is_pair, UnpairedReadsError, _split_left_right) +import itertools def print_error(msg): @@ -119,7 +120,7 @@ def broken_paired_reader(screed_iter, min_length=None, def write_record(record, fileobj): """Write sequence record to 'fileobj' in FASTA/FASTQ format.""" - if hasattr(record, 'quality'): + if hasattr(record, 'quality') and record.quality is not None: recstr = '@{name}\n{sequence}\n+\n{quality}\n'.format( name=record.name, sequence=record.sequence, @@ -140,7 +141,7 @@ def write_record_pair(read1, read2, fileobj): _rec_pair = '@%s\n%s\n+\n%s\n' * 2 _rec_pair_no_qual = '>%s\n%s\n' * 2 - if hasattr(read1, 'quality'): + if hasattr(read1, 'quality') and read1.quality is not None: assert hasattr(read2, 'quality') recstr = _rec_pair % (read1.name, read1.sequence, read1.quality, read2.name, read2.sequence, read2.quality) @@ -187,6 +188,10 @@ def num_reads(self): def total_length(self): return sum([len(r.sequence) for r in self.reads]) +def grouper(n, iterable): + iterable = iter(iterable) + return iter(lambda: list(itertools.islice(iterable, n)), []) + # vim: set filetype=python tabstop=4 softtabstop=4 shiftwidth=4 expandtab: # vim: set textwidth=79: diff --git a/scripts/sample-reads-randomly.py b/scripts/sample-reads-randomly.py index 5d1fa6ed1d..77513ba8dd 100755 --- a/scripts/sample-reads-randomly.py +++ b/scripts/sample-reads-randomly.py @@ -58,7 +58,8 @@ from khmer.kfile import (check_input_files, add_output_compression_type, get_file_writer) from khmer.khmer_args import sanitize_help, KhmerArgumentParser -from khmer.utils import write_record, broken_paired_reader +from khmer.utils import write_record, grouper +from khmer._oxli.parsing import FastxParser, BrokenPairedReader, SplitPairedReader DEFAULT_NUM_READS = int(1e5) DEFAULT_MAX_READS = int(1e8) @@ -96,6 +97,9 @@ def get_parser(): help='Provide a random seed for the generator') parser.add_argument('--force_single', default=False, action='store_true', help='Ignore read pair information if present') + parser.add_argument('--pairing-mode', + choices=['split', 'interleaved', 'single'], + default='interleaved') parser.add_argument('-o', '--output', dest='output_file', type=argparse.FileType('wb'), metavar="filename", default=None) @@ -163,16 +167,31 @@ def main(): % output_filename, file=sys.stderr) print('', file=sys.stderr) + if args.pairing_mode == 'split': + samples = list(grouper(2, args.filenames)) + for pair in samples: + if len(pair) != 2: + raise ValueError('Must have even number of samples!') + else: + samples = args.filenames + reads = [] for _ in range(num_samples): reads.append([]) # read through all the sequences and load/resample the reservoir - for filename in args.filenames: - print('opening', filename, 'for reading', file=sys.stderr) - - for count, (_, _, rcrd1, rcrd2) in enumerate(broken_paired_reader( - ReadParser(filename), force_single=args.force_single)): + for group in samples: + + if args.pairing_mode == 'split': + print('opening', group[0], 'and', group[1], 'for reading', file=sys.stderr) + reader = SplitPairedReader(FastxParser(group[0]), + FastxParser(group[1])) + else: + print('opening', group, 'for reading', file=sys.stderr) + reader = BrokenPairedReader(FastxParser(group), + force_single=args.force_single) + + for count, (_, _, rcrd1, rcrd2) in enumerate(reader): if count % 10000 == 0: print('...', count, 'reads scanned', file=sys.stderr) if count >= args.max_reads: From 47cbf6291f784562ccf69910a9009b33ea66b242 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 3 Apr 2017 21:51:21 -0700 Subject: [PATCH 077/185] Fix Count and Nodetable namespaces in wrapper --- khmer/_oxli/wrapper.pxd | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index 966cff7e70..b866fcd1ed 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -48,7 +48,7 @@ cdef extern from "oxli/kmer_hash.hh" namespace "oxli": bool is_forward() const void set_from_unique_hash(HashIntoType, WordLength) - cdef cppclass CpKmerFactory "oxli:KmerFactory": + cdef cppclass CpKmerFactory "oxli::KmerFactory": KmerFactory(WordLength) CpKmer build_kmer(HashIntoType) const @@ -56,7 +56,7 @@ cdef extern from "oxli/kmer_hash.hh" namespace "oxli": CpKmer build_kmer(string &) const CpKmer build_kmer(const char *) const - cdef cppclass CpKmerIterator "oxli:KmerIterator" (CpKmerFactory): + cdef cppclass CpKmerIterator "oxli::KmerIterator" (CpKmerFactory): CpKmerIterator(const char *, unsigned char) CpKmer first(HashIntoType &, HashIntoType &) CpKmer next(HashIntoType &, HashIntoType &) @@ -76,6 +76,7 @@ cdef extern from "oxli/kmer_hash.hh" namespace "oxli": HashIntoType _hash_forward(const char *, WordLength) string _revhash(HashIntoType, WordLength) string _revcomp(const string&) + HashIntoType _hash_murmur(const string&, const WordLength) HashIntoType _hash_murmur(const string&, HashIntoType&, HashIntoType&) HashIntoType _hash_murmur_forward(const string&) @@ -229,12 +230,10 @@ cdef extern from "oxli/hashtable.hh" namespace "oxli": vector[uint32_t] find_spectral_error_positions(string, BoundedCounterType) -cdef extern from "khmer/_cpy_counttable.hh" namespace "khmer": - cdef cppclass CpCounttable "khmer::Counttable" (CpHashtable): + cdef cppclass CpCounttable "oxli::Counttable" (CpHashtable): CpCounttable(WordLength, vector[uint64_t]) -cdef extern from "khmer/_cpy_nodetable.hh" namespace "khmer": - cdef cppclass CpNodetable "khmer::Nodetable" (CpHashtable): + cdef cppclass CpNodetable "oxli::Nodetable" (CpHashtable): CpNodetable(WordLength, vector[uint64_t]) From eb0cbe436dfe432cc1dcbd26f329c86aee77cc58 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 3 Apr 2017 21:51:50 -0700 Subject: [PATCH 078/185] Add repr to Kmer --- khmer/_oxli/hashing.pyx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/khmer/_oxli/hashing.pyx b/khmer/_oxli/hashing.pyx index cdc1b3dbc3..6bbfcdee6e 100644 --- a/khmer/_oxli/hashing.pyx +++ b/khmer/_oxli/hashing.pyx @@ -26,6 +26,9 @@ cdef class Kmer: def __hash__(self): return self.kmer_u + def __repr__(self): + return self.kmer + @property def kmer_f(self): return deref(self._this).kmer_f From 5a322b4aeca2e7ecc7f270bcc97eb26414734da2 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 3 Apr 2017 21:52:07 -0700 Subject: [PATCH 079/185] Add kmer generator to Sequence --- khmer/_oxli/parsing.pyx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/khmer/_oxli/parsing.pyx b/khmer/_oxli/parsing.pyx index 217f0f64d0..63e7e37e0f 100644 --- a/khmer/_oxli/parsing.pyx +++ b/khmer/_oxli/parsing.pyx @@ -75,6 +75,12 @@ cdef class Sequence: else: raise NotImplementedError('Operator not available') + def kmers(self, int K): + cdef int i = 0 + cdef unicode sequence = self.sequence + for i in range(0, len(self)-K+1): + yield sequence[i:i+K] + @property def name(self): cdef unicode name = self._obj.name From e2ab29720418c5346ad8010360c50149b801cd26 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 10 Apr 2017 01:23:17 -0700 Subject: [PATCH 080/185] Make LinearAssembler functions virtual --- include/oxli/assembler.hh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/include/oxli/assembler.hh b/include/oxli/assembler.hh index b2242974e3..6f635c03a0 100644 --- a/include/oxli/assembler.hh +++ b/include/oxli/assembler.hh @@ -82,13 +82,13 @@ public: explicit LinearAssembler(const Hashgraph * ht); - std::string assemble(const Kmer seed_kmer, + virtual std::string assemble(const Kmer seed_kmer, const Hashgraph * stop_bf = 0) const; - std::string assemble_right(const Kmer seed_kmer, + virtual std::string assemble_right(const Kmer seed_kmer, const Hashgraph * stop_bf = 0) const; - std::string assemble_left(const Kmer seed_kmer, + virtual std::string assemble_left(const Kmer seed_kmer, const Hashgraph * stop_bf = 0) const; template @@ -112,11 +112,13 @@ public: explicit CompactingAssembler(const Hashgraph* ht) : LinearAssembler(ht) {} - std::string assemble_right(const Kmer seed_kmer, - const Hashgraph * stop_bf = 0) const; + virtual std::string assemble_right(const Kmer seed_kmer, + const Hashgraph * stop_bf = 0, + SeenSet * visited = 0) const; - std::string assemble_left(const Kmer seed_kmer, - const Hashgraph * stop_bf = 0) const; + virtual std::string assemble_left(const Kmer seed_kmer, + const Hashgraph * stop_bf = 0, + SeenSet * visited = 0) const; }; typedef CompactingAssembler CpCompactingAssembler; From 0f5bbdee8249a279585978482366eec08420a2fb Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 10 Apr 2017 01:23:50 -0700 Subject: [PATCH 081/185] Remove smart pointer usage for Assembler wrappers --- khmer/_oxli/assembly.pxd | 3 ++- khmer/_oxli/assembly.pyx | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/khmer/_oxli/assembly.pxd b/khmer/_oxli/assembly.pxd index a48b57c440..ba466f5ccd 100644 --- a/khmer/_oxli/assembly.pxd +++ b/khmer/_oxli/assembly.pxd @@ -6,7 +6,8 @@ from hashing cimport Kmer cdef class LinearAssembler: - cdef unique_ptr[CpLinearAssembler] _this + #cdef unique_ptr[CpLinearAssembler] _this + cdef CpLinearAssembler * _this cdef public object graph cdef CpHashgraph * _graph_ptr diff --git a/khmer/_oxli/assembly.pyx b/khmer/_oxli/assembly.pyx index f69132fb08..df22d7a966 100644 --- a/khmer/_oxli/assembly.pyx +++ b/khmer/_oxli/assembly.pyx @@ -16,7 +16,8 @@ cdef class LinearAssembler: self.set_stop_filter(stop_filter=stop_filter) if type(self) is LinearAssembler: - self._this.reset(new CpLinearAssembler(self._graph_ptr)) + self._this = new CpLinearAssembler(self._graph_ptr) + #self._this.reset(new CpLinearAssembler(self._graph_ptr)) def set_stop_filter(self, stop_filter=None): self.stop_filter = stop_filter @@ -75,6 +76,7 @@ cdef class CompactingAssembler(LinearAssembler): cdef CpCompactingAssembler* ptr = new CpCompactingAssembler(self._graph_ptr) if type(self) is CompactingAssembler: - self._this.reset( ptr) + self._this = ptr + #self._this.reset( ptr) From 222deb48ba6550fa55418dc0094710d2ec9de6d6 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 24 Apr 2017 14:00:09 -0700 Subject: [PATCH 082/185] Remove NonLoopingAT and merge its functionality into AssemblerTraverser --- include/oxli/assembler.hh | 13 +++---- include/oxli/kmer_filters.hh | 2 +- include/oxli/traversal.hh | 43 +++++++++------------- src/oxli/assembler.cc | 69 ++++++++++++++++++++++++++---------- src/oxli/kmer_filters.cc | 2 +- src/oxli/traversal.cc | 54 +++++++++++++++++----------- 6 files changed, 111 insertions(+), 72 deletions(-) diff --git a/include/oxli/assembler.hh b/include/oxli/assembler.hh index 6f635c03a0..882fdaf3ee 100644 --- a/include/oxli/assembler.hh +++ b/include/oxli/assembler.hh @@ -112,13 +112,14 @@ public: explicit CompactingAssembler(const Hashgraph* ht) : LinearAssembler(ht) {} + virtual std::string assemble(const Kmer seed_kmer, + const Hashgraph * stop_bf) const; + virtual std::string assemble_right(const Kmer seed_kmer, - const Hashgraph * stop_bf = 0, - SeenSet * visited = 0) const; + const Hashgraph * stop_bf = 0) const; virtual std::string assemble_left(const Kmer seed_kmer, - const Hashgraph * stop_bf = 0, - SeenSet * visited = 0) const; + const Hashgraph * stop_bf = 0) const; }; typedef CompactingAssembler CpCompactingAssembler; @@ -158,7 +159,7 @@ public: const Hashgraph * stop_bf=0) const; template - void _assemble_directed(NonLoopingAT& start_cursor, + void _assemble_directed(AssemblerTraverser& start_cursor, StringVector& paths) const; }; @@ -187,7 +188,7 @@ public: BoundedCounterType get_junction_count(Kmer kmer_a, Kmer kmer_b) const; template - void _assemble_directed(NonLoopingAT& start_cursor, + void _assemble_directed(AssemblerTraverser& start_cursor, StringVector& paths) const; }; diff --git a/include/oxli/kmer_filters.hh b/include/oxli/kmer_filters.hh index 0121474e60..35113248bc 100644 --- a/include/oxli/kmer_filters.hh +++ b/include/oxli/kmer_filters.hh @@ -61,7 +61,7 @@ KmerFilter get_simple_label_intersect_filter(const LabelSet& src_labels, KmerFilter get_stop_bf_filter(const Hashtable * stop_bf); -KmerFilter get_visited_filter(const SeenSet * visited); +KmerFilter get_visited_filter(std::shared_ptr visited); KmerFilter get_junction_count_filter(const Kmer& src_node, Countgraph * junctions, diff --git a/include/oxli/traversal.hh b/include/oxli/traversal.hh index f060140b1b..b8075988da 100644 --- a/include/oxli/traversal.hh +++ b/include/oxli/traversal.hh @@ -246,10 +246,22 @@ public: template class AssemblerTraverser: public NodeCursor { +protected: + std::shared_ptr visited; public: using NodeCursor::NodeCursor; + explicit AssemblerTraverser(const Hashgraph * ht, + Kmer start_kmer, + KmerFilterList filters); + + explicit AssemblerTraverser(const Hashgraph * ht, + Kmer start_kmer, + KmerFilterList filters, + std::shared_ptr visited); + AssemblerTraverser(const AssemblerTraverser& other); + /** * @brief Get the next symbol. * @@ -278,32 +290,6 @@ public: }; -/** - * @brief An AssemblerTraverser which does not traverse to Kmers it has already encountered. - * - * Simply adds a new filter to check if the Kmer has been seen, and adds the Kmer to the set - * of seen Kmers after calling ::next_symbol. - * - * @tparam direction The direction to assemble. - */ -template -class NonLoopingAT: public AssemblerTraverser -{ -protected: - - SeenSet * visited; - -public: - - explicit NonLoopingAT(const Hashgraph * ht, - Kmer start_kmer, - KmerFilterList filters, - SeenSet * visited); - - virtual char next_symbol(); -}; - - template class CompactingAT: public AssemblerTraverser { @@ -317,6 +303,11 @@ public: Kmer start_kmer, KmerFilterList filters); + explicit CompactingAT(const Hashgraph * ht, + Kmer start_kmer, + KmerFilterList filters, + std::shared_ptr visited); + virtual char next_symbol(); }; diff --git a/src/oxli/assembler.cc b/src/oxli/assembler.cc index 7ea2235a3b..a61a453e82 100644 --- a/src/oxli/assembler.cc +++ b/src/oxli/assembler.cc @@ -66,8 +66,18 @@ const // something happen. It's not going to happen! return ""; } - std::string right_contig = assemble_right(seed_kmer, stop_bf); - std::string left_contig = assemble_left(seed_kmer, stop_bf); + + std::list node_filters; + if (stop_bf) { + node_filters.push_back(get_stop_bf_filter(stop_bf)); + } + + std::shared_ptr visited = std::make_shared(); + AssemblerTraverser rcursor(graph, seed_kmer, node_filters, visited); + AssemblerTraverser lcursor(graph, seed_kmer, node_filters, visited); + + std::string right_contig = _assemble_directed(rcursor); + std::string left_contig = _assemble_directed(lcursor); #if DEBUG_ASSEMBLY std::cout << "Left: " << left_contig << std::endl; @@ -167,6 +177,29 @@ const * Compacting Assembler ********************************/ +std::string CompactingAssembler::assemble(const Kmer seed_kmer, + const Hashgraph * stop_bf) +const +{ + if (graph->get_count(seed_kmer) == 0) { + return ""; + } + + std::list node_filters; + if (stop_bf) { + node_filters.push_back(get_stop_bf_filter(stop_bf)); + } + + std::shared_ptr visited = std::make_shared(); + CompactingAT rcursor(graph, seed_kmer, node_filters, visited); + CompactingAT lcursor(graph, seed_kmer, node_filters, visited); + + std::string right_contig = _assemble_directed(rcursor); + std::string left_contig = _assemble_directed(lcursor); + + right_contig = right_contig.substr(_ksize); + return left_contig + right_contig; +} std::string CompactingAssembler::assemble_right(const Kmer seed_kmer, const Hashgraph * stop_bf) @@ -227,20 +260,20 @@ const node_filters.push_back(get_stop_bf_filter(stop_bf)); } - SeenSet visited; + std::shared_ptr visited = std::make_shared(); #if DEBUG_ASSEMBLY std::cout << "Assemble Labeled RIGHT: " << seed_kmer.repr(_ksize) << std::endl; #endif StringVector right_paths; - NonLoopingAT rcursor(graph, seed_kmer, node_filters, &visited); + AssemblerTraverser rcursor(graph, seed_kmer, node_filters, visited); _assemble_directed(rcursor, right_paths); #if DEBUG_ASSEMBLY std::cout << "Assemble Labeled LEFT: " << seed_kmer.repr(_ksize) << std::endl; #endif StringVector left_paths; - NonLoopingAT lcursor(graph, seed_kmer, node_filters, &visited); + AssemblerTraverser lcursor(graph, seed_kmer, node_filters, visited); _assemble_directed(lcursor, left_paths); StringVector paths; @@ -253,12 +286,12 @@ const } } - visited.clear(); + visited->clear(); return paths; } template -void SimpleLabeledAssembler::_assemble_directed(NonLoopingAT& +void SimpleLabeledAssembler::_assemble_directed(AssemblerTraverser& start_cursor, StringVector& paths) const @@ -276,7 +309,7 @@ const std::cout << "Cursor: " << start_cursor.cursor.repr(_ksize) << std::endl; #endif StringVector segments; - std::vector< NonLoopingAT > cursors; + std::vector< AssemblerTraverser > cursors; segments.push_back(root_contig); cursors.push_back(start_cursor); @@ -284,7 +317,7 @@ const while(segments.size() != 0) { std::string segment = segments.back(); - NonLoopingAT cursor = cursors.back(); + AssemblerTraverser cursor = cursors.back(); #if DEBUG_ASSEMBLY std::cout << "Pop: " << segments.size() << " segments on stack." << std::endl; std::cout << "Segment: " << segment << std::endl; @@ -329,7 +362,7 @@ const // found some neighbors; extend them while(!branch_starts.empty()) { // spin off a cursor for the new branch - NonLoopingAT branch_cursor(cursor); + AssemblerTraverser branch_cursor(cursor); branch_cursor.cursor = branch_starts.front(); branch_starts.pop(); @@ -444,21 +477,21 @@ const node_filters.push_back(get_stop_bf_filter(stop_bf)); } - SeenSet visited; + std::shared_ptr visited = std::make_shared(); #if DEBUG_ASSEMBLY std::cout << "Assemble Junctions RIGHT: " << seed_kmer.repr( _ksize) << std::endl; #endif StringVector right_paths; - NonLoopingAT rcursor(graph, seed_kmer, node_filters, &visited); + AssemblerTraverser rcursor(graph, seed_kmer, node_filters, visited); _assemble_directed(rcursor, right_paths); #if DEBUG_ASSEMBLY std::cout << "Assemble Junctions LEFT: " << seed_kmer.repr(_ksize) << std::endl; #endif StringVector left_paths; - NonLoopingAT lcursor(graph, seed_kmer, node_filters, &visited); + AssemblerTraverser lcursor(graph, seed_kmer, node_filters, visited); _assemble_directed(lcursor, left_paths); StringVector paths; @@ -471,12 +504,12 @@ const } } - visited.clear(); + visited->clear(); return paths; } template -void JunctionCountAssembler::_assemble_directed(NonLoopingAT& +void JunctionCountAssembler::_assemble_directed(AssemblerTraverser& start_cursor, StringVector& paths) const @@ -494,7 +527,7 @@ const std::cout << "Cursor: " << start_cursor.cursor.repr(_ksize) << std::endl; #endif StringVector segments; - std::vector< NonLoopingAT > cursors; + std::vector< AssemblerTraverser > cursors; segments.push_back(root_contig); cursors.push_back(start_cursor); @@ -502,7 +535,7 @@ const while(segments.size() != 0) { std::string segment = segments.back(); - NonLoopingAT cursor = cursors.back(); + AssemblerTraverser cursor = cursors.back(); #if DEBUG_ASSEMBLY std::cout << "Pop: " << segments.size() << " segments on stack." << std::endl; std::cout << "Segment: " << segment << std::endl; @@ -531,7 +564,7 @@ const // found some neighbors; extend them while(!branch_starts.empty()) { // spin off a cursor for the new branch - NonLoopingAT branch_cursor(cursor); + AssemblerTraverser branch_cursor(cursor); branch_cursor.cursor = branch_starts.front(); branch_starts.pop(); diff --git a/src/oxli/kmer_filters.cc b/src/oxli/kmer_filters.cc index fda4322d69..987ec327f2 100644 --- a/src/oxli/kmer_filters.cc +++ b/src/oxli/kmer_filters.cc @@ -144,7 +144,7 @@ KmerFilter get_stop_bf_filter(const Hashtable * stop_bf) } -KmerFilter get_visited_filter(const SeenSet * visited) +KmerFilter get_visited_filter(std::shared_ptr visited) { #if DEBUG_FILTERS std::cout << "Create new visited filter with " << visited << diff --git a/src/oxli/traversal.cc b/src/oxli/traversal.cc index 1bd9251af6..629e5ab2e4 100644 --- a/src/oxli/traversal.cc +++ b/src/oxli/traversal.cc @@ -273,6 +273,34 @@ unsigned int Traverser::degree_right(const Kmer& node) const * AssemblerTraverser ******************************************/ +template +AssemblerTraverser::AssemblerTraverser(const Hashgraph * ht, + Kmer start_kmer, + KmerFilterList filters) : + NodeCursor(ht, start_kmer, filters) +{ + visited = std::make_shared(); + AssemblerTraverser::push_filter(get_visited_filter(visited)); +} + +template +AssemblerTraverser::AssemblerTraverser(const Hashgraph * ht, + Kmer start_kmer, + KmerFilterList filters, + std::shared_ptr visited) : + NodeCursor(ht, start_kmer, filters), visited(visited) +{ + AssemblerTraverser::push_filter(get_visited_filter(visited)); +} + +template +AssemblerTraverser::AssemblerTraverser(const AssemblerTraverser& other) : + AssemblerTraverser(other.graph, other.cursor, other.filters, other.visited) +{ + +} + + template <> std::string AssemblerTraverser::join_contigs(std::string& contig_a, std::string& contig_b, WordLength offset) @@ -297,6 +325,7 @@ char AssemblerTraverser::next_symbol() Kmer neighbor; Kmer cursor_next; + visited->insert(this->cursor); for (auto base : alphabets::DNA_SIMPLE) { // Get the putative neighbor for this base at the cursor position neighbor = NodeCursor::get_neighbor(this->cursor, base); @@ -323,33 +352,20 @@ char AssemblerTraverser::next_symbol() } } + /****************************************** - * NonLoopingAT + * CompactingAT ******************************************/ template -NonLoopingAT::NonLoopingAT(const Hashgraph * ht, +CompactingAT::CompactingAT(const Hashgraph * ht, Kmer start_kmer, KmerFilterList filters, - SeenSet * visited) : - AssemblerTraverser(ht, start_kmer, filters), visited(visited) -{ - AssemblerTraverser::push_filter(get_visited_filter(visited)); -} - -template -char NonLoopingAT::next_symbol() + std::shared_ptr visited) : + AssemblerTraverser(ht, start_kmer, filters, visited), traverser(ht) { -#if DEBUG_TRAVERSAL - std::cout << "Insert cursor to visited filter" << std::endl; -#endif - visited->insert(this->cursor); - return AssemblerTraverser::next_symbol(); } -/****************************************** - * CompactingAT - ******************************************/ template CompactingAT::CompactingAT(const Hashgraph * ht, @@ -384,8 +400,6 @@ template class NodeCursor; template class NodeCursor; template class AssemblerTraverser; template class AssemblerTraverser; -template class NonLoopingAT; -template class NonLoopingAT; template class CompactingAT; template class CompactingAT; From 92404504b840458837419421320f84d2edd1f204 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 24 Apr 2017 14:12:45 -0700 Subject: [PATCH 083/185] Split find connected tags call from consume_and_connect_tags, rename latter to seed_sequence --- include/oxli/partitioning.hh | 6 ++++-- khmer/_oxli/wrapper.pxd | 3 +++ src/oxli/partitioning.cc | 33 ++++++++++++++++++++------------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index 312c4add1e..1419134590 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -237,8 +237,10 @@ class StreamingPartitioner { uint64_t consume(const std::string& seq); uint64_t consume_pair(const std::string& first, const std::string& second); - uint64_t consume_and_connect_tags(const std::string& seq, - std::set& tags); + uint64_t seed_sequence(const std::string& seq, + std::set& tags, + KmerQueue& seeds, + std::set& seen); void create_and_connect_components(std::set& tags); uint64_t consume_fasta(std::string const &filename); diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index b866fcd1ed..cb3de1c2ed 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -455,6 +455,9 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": uint64_t consume_pair(string&, string&) nogil except +MemoryError uint64_t consume_fasta(string&) except +MemoryError + uint64_t seed_sequence(string&, set[HashIntoType]&, KmerQueue&, + set[HashIntoType]&) except +MemoryError + void create_and_connect_components(set[HashIntoType]&) void add_component(ComponentPtr comp) void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) void find_connected_tags(queue[CpKmer]&, diff --git a/src/oxli/partitioning.cc b/src/oxli/partitioning.cc index aaf2879974..bd4d44cd01 100644 --- a/src/oxli/partitioning.cc +++ b/src/oxli/partitioning.cc @@ -96,7 +96,11 @@ uint64_t StreamingPartitioner::consume_fasta(const std::string& filename) uint64_t StreamingPartitioner::consume(const std::string& seq) { std::set tags; - uint64_t n_new = consume_and_connect_tags(seq, tags); + KmerQueue seeds; + std::set seen; + + uint64_t n_new = seed_sequence(seq, tags, seeds, seen); + find_connected_tags(seeds, tags, seen, false); //acquire_components(); create_and_connect_components(tags); //release_components(); @@ -108,8 +112,12 @@ uint64_t StreamingPartitioner::consume_pair(const std::string& first, const std::string& second) { std::set tags; - uint64_t n_new = consume_and_connect_tags(first, tags); - n_new += consume_and_connect_tags(second, tags); + KmerQueue seeds; + std::set seen; + + uint64_t n_new = seed_sequence(first, tags, seeds, seen); + n_new += seed_sequence(second, tags, seeds, seen); + find_connected_tags(seeds, tags, seen, false); //acquire_components(); create_and_connect_components(tags); //release_components(); @@ -123,8 +131,11 @@ void StreamingPartitioner::add_component(ComponentPtr comp) } -uint64_t StreamingPartitioner::consume_and_connect_tags(const std::string& seq, - std::set& tags) + +uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, + std::set& tags, + KmerQueue& seeds, + std::set& seen) { /* For the following comments, let G be the set of k-mers * known in the graph before inserting the k-mers R from @@ -142,9 +153,7 @@ uint64_t StreamingPartitioner::consume_and_connect_tags(const std::string& seq, KmerIterator kmers(seq.c_str(), graph->ksize()); unsigned int since = _tag_density / 2 + 1; - std::set seen; KmerSet intersection; - KmerQueue search_from; bool in_known_territory = false; bool found_tag_in_territory = false; @@ -164,12 +173,12 @@ uint64_t StreamingPartitioner::consume_and_connect_tags(const std::string& seq, seen.insert(intersection.begin(), intersection.end()); } /*else { for (auto km : intersection) { - search_from.push(km); + seeds.push(km); } }*/ intersection.clear(); - search_from.push(kmer); + seeds.push(kmer); in_known_territory = false; found_tag_in_territory = false; ++since; @@ -200,18 +209,16 @@ uint64_t StreamingPartitioner::consume_and_connect_tags(const std::string& seq, if (since >= _tag_density / 2) { tags.insert(kmer); } - search_from.push(kmer); + seeds.push(kmer); // now go back and make sure to search from the first k-mer kmer = kmers.first(); - search_from.push(kmer); + seeds.push(kmer); #if(DEBUG_SP) std::cout << "Done iterating k-mers" << std::endl; std::cout << tags.size() << " tags in sequence" << std::endl; #endif - // Now search for tagged nodes connected to U. - find_connected_tags(search_from, tags, seen, false); } else { throw oxli_ptr_exception("Hashgraph has been deleted."); } From d8cb0cd2b98ff6fc81a9fc286df28155747c7dd4 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 26 Apr 2017 15:32:11 -0700 Subject: [PATCH 084/185] add args and kwargs for StreamingPartitioner subclassing --- khmer/_oxli/partitioning.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 80d2f94670..284de83632 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -116,7 +116,7 @@ cdef class Component: cdef class StreamingPartitioner: - def __cinit__(self, graph, tag_density=None): + def __cinit__(self, graph, tag_density=None, *args, **kwargs): self._graph_ptr = get_hashgraph_ptr(graph) if self._graph_ptr == NULL: raise ValueError('Must take an object with Hashgraph *') @@ -265,3 +265,4 @@ cdef class StreamingPartitioner: @property def tag_density(self): return deref(self._this).get_tag_density() + From cfea1bf9ffdd00446e97b0e701944b4dad951cd2 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 29 Apr 2017 22:12:52 -0700 Subject: [PATCH 085/185] Add slicing to Sequence --- khmer/_oxli/parsing.pyx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/khmer/_oxli/parsing.pyx b/khmer/_oxli/parsing.pyx index 63e7e37e0f..6f04e451f6 100644 --- a/khmer/_oxli/parsing.pyx +++ b/khmer/_oxli/parsing.pyx @@ -81,6 +81,10 @@ cdef class Sequence: for i in range(0, len(self)-K+1): yield sequence[i:i+K] + def __getitem__(self, x): + # not ideal + return self.sequence[x] + @property def name(self): cdef unicode name = self._obj.name From 04b161481ac16435a42bde6f3df3b5f71b035ab0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 29 Apr 2017 22:14:31 -0700 Subject: [PATCH 086/185] Report number of partitions merged --- include/oxli/partitioning.hh | 16 ++++++++-------- khmer/_oxli/wrapper.pxd | 15 +++++++++++++-- src/oxli/partitioning.cc | 14 ++++++++++---- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index 1419134590..bd6f2b7b83 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -218,11 +218,6 @@ class StreamingPartitioner { private: uint32_t _tag_density; - - // We're not graph's owner, simply an observer. - // Unforunately our ownership policies elsewhere are a mess - Hashgraph * graph; - //std::weak_ptr graph; // We should exclusively own tag_component_map. std::shared_ptr tag_component_map; std::shared_ptr components; @@ -230,6 +225,10 @@ class StreamingPartitioner { uint64_t n_consumed; public: + // We're not graph's owner, simply an observer. + // Unforunately our ownership policies elsewhere are a mess + Hashgraph * graph; + //std::weak_ptr graph; explicit StreamingPartitioner(Hashgraph * graph, uint32_t tag_density=DEFAULT_TAG_DENSITY); @@ -241,7 +240,7 @@ class StreamingPartitioner { std::set& tags, KmerQueue& seeds, std::set& seen); - void create_and_connect_components(std::set& tags); + uint32_t create_and_connect_components(std::set& tags); uint64_t consume_fasta(std::string const &filename); void map_tags_to_component(std::set& tags, ComponentPtr& comp); @@ -250,6 +249,9 @@ class StreamingPartitioner { std::set& found_tags, std::set& seen, bool truncate=false) const; + uint32_t merge_components(ComponentPtr& root, ComponentPtrSet& comps); + + uint64_t get_n_components() const { return components->size(); @@ -267,8 +269,6 @@ class StreamingPartitioner { return _tag_density; } - void merge_components(ComponentPtr& root, ComponentPtrSet& comps); - ComponentPtr get_tag_component(HashIntoType tag) const; ComponentPtr get_tag_component(std::string& tag) const; diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index cb3de1c2ed..351708cb6f 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -450,26 +450,37 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": cdef cppclass CpStreamingPartitioner "oxli::StreamingPartitioner": CpStreamingPartitioner(CpHashgraph * ) except +MemoryError CpStreamingPartitioner(CpHashgraph *, uint32_t) except +MemoryError - + + CpHashgraph * graph uint64_t consume(string&) nogil except +MemoryError uint64_t consume_pair(string&, string&) nogil except +MemoryError uint64_t consume_fasta(string&) except +MemoryError uint64_t seed_sequence(string&, set[HashIntoType]&, KmerQueue&, set[HashIntoType]&) except +MemoryError - void create_and_connect_components(set[HashIntoType]&) + uint32_t create_and_connect_components(set[HashIntoType]&) void add_component(ComponentPtr comp) void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) + void find_connected_tags(queue[CpKmer]&, set[HashIntoType]&, set[HashIntoType]&) except +MemoryError + + void find_connected_tags(queue[CpKmer]&, + set[HashIntoType]&, + set[HashIntoType]&, + bool) except +MemoryError + uint64_t get_n_components() const uint64_t get_n_tags() const uint64_t get_n_consumed() const uint32_t get_tag_density() const ComponentPtr get_tag_component(string&) const + ComponentPtr get_tag_component(HashIntoType) const + ComponentPtr get_nearest_component(string&) const + ComponentPtr get_nearest_component(CpKmer) const weak_ptr[ComponentPtrSet] get_component_set() weak_ptr[CpGuardedKmerCompMap] get_tag_component_map() diff --git a/src/oxli/partitioning.cc b/src/oxli/partitioning.cc index bd4d44cd01..2696ae9445 100644 --- a/src/oxli/partitioning.cc +++ b/src/oxli/partitioning.cc @@ -93,6 +93,7 @@ uint64_t StreamingPartitioner::consume_fasta(const std::string& filename) } + uint64_t StreamingPartitioner::consume(const std::string& seq) { std::set tags; @@ -227,7 +228,7 @@ uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, } -void StreamingPartitioner::create_and_connect_components(std::set &tags) +uint32_t StreamingPartitioner::create_and_connect_components(std::set &tags) { // Now resolve components. First, get components from existing tags. @@ -254,7 +255,8 @@ void StreamingPartitioner::create_and_connect_components(std::set #if(DEBUG_SP) std::cout << found_comps.size() << " unique components." << std::endl; #endif - + + uint32_t n_merged = 1; if (found_comps.size() == 0) { ComponentPtr new_comp = std::make_shared(); #if(DEBUG_SP) @@ -278,15 +280,17 @@ void StreamingPartitioner::create_and_connect_components(std::set root_comp->add_tag(tags); map_tags_to_component(tags, root_comp); if (found_comps.size() > 1) { - merge_components(root_comp, found_comps); + n_merged = merge_components(root_comp, found_comps); } } + return n_merged; } -void StreamingPartitioner::merge_components(ComponentPtr& root, +uint32_t StreamingPartitioner::merge_components(ComponentPtr& root, ComponentPtrSet& comps) { + uint32_t n_merged = 1; for (auto other : comps) { if (*other == *root) { continue; @@ -294,10 +298,12 @@ void StreamingPartitioner::merge_components(ComponentPtr& root, root->add_tag(other->tags); // transfer the tags from the other comp map_tags_to_component(other->tags, root); // set the other's tags to point to root components->erase(other); // remove other component entirely + n_merged++; } comps.clear(); // should call destructor on all the merged comps, unless they have // and active Python wrapper; this leaves them as sole owners + return n_merged; } From eb47283c82270c2648764f4373cd7b068f59c896 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 29 Apr 2017 22:15:01 -0700 Subject: [PATCH 087/185] Use shared_ptr for StreamingPartiioner _this --- khmer/_oxli/partitioning.pxd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index 40ebebb731..483ecc8b81 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -26,9 +26,10 @@ cdef class Component: cdef class StreamingPartitioner: - cdef unique_ptr[CpStreamingPartitioner] _this + cdef shared_ptr[CpStreamingPartitioner] _this cdef weak_ptr[ComponentPtrSet] _components cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map cdef CpHashgraph * _graph_ptr cdef readonly object graph cdef readonly uint64_t n_consumed + From d87348a6df26b37109e0cd82825e13022a72e3d8 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 30 Apr 2017 19:50:28 -0700 Subject: [PATCH 088/185] Fix bad string conversion --- khmer/_oxli/partitioning.pyx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 284de83632..511dbbde20 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -21,6 +21,7 @@ import json import os from wrapper cimport * +from utils cimport _bstring from .._khmer import Countgraph from .._khmer import Nodegraph from khmer import load_countgraph, load_nodegraph @@ -145,7 +146,8 @@ cdef class StreamingPartitioner: def get_tag_component(self, kmer): cdef ComponentPtr compptr - compptr = deref(self._this).get_tag_component(kmer.encode('utf-8')) + cdef string kmer_s = _bstring(kmer) + compptr = deref(self._this).get_tag_component(kmer_s) if compptr == NULL: return None else: @@ -153,7 +155,8 @@ cdef class StreamingPartitioner: def get_nearest_component(self, kmer): cdef ComponentPtr compptr - compptr = deref(self._this).get_nearest_component(kmer.encode('utf-8')) + cdef string kmer_s = _bstring(kmer) + compptr = deref(self._this).get_nearest_component(kmer_s) if compptr == NULL: return None else: From 4533631a46ba1d579116f1b2776691f558c1d357 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 5 May 2017 23:00:12 -0400 Subject: [PATCH 089/185] Add informative messages to KmerIterator exceptions --- src/oxli/kmer_hash.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/oxli/kmer_hash.cc b/src/oxli/kmer_hash.cc index bc49841df1..9c2f21fb32 100644 --- a/src/oxli/kmer_hash.cc +++ b/src/oxli/kmer_hash.cc @@ -248,7 +248,7 @@ Kmer KmerIterator::first(HashIntoType& f, HashIntoType& r) Kmer KmerIterator::next(HashIntoType& f, HashIntoType& r) { if (done()) { - throw oxli_exception(); + throw oxli_exception("KmerIterator done."); } if (!initialized) { @@ -259,7 +259,7 @@ Kmer KmerIterator::next(HashIntoType& f, HashIntoType& r) unsigned char ch = _seq[index]; index++; if (!(index <= length)) { - throw oxli_exception(); + throw oxli_exception("KmerIterator index <= length"); } // left-shift the previous hash over From 365fb49412ae64dcbd821edd2e543e2eff2ae366 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 5 May 2017 23:01:56 -0400 Subject: [PATCH 090/185] Apply min_length filter to both pairs in SplitPairedReader --- khmer/_oxli/parsing.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/khmer/_oxli/parsing.pyx b/khmer/_oxli/parsing.pyx index 6f04e451f6..dd4358760f 100644 --- a/khmer/_oxli/parsing.pyx +++ b/khmer/_oxli/parsing.pyx @@ -266,7 +266,7 @@ cdef class SplitPairedReader: raise err if self.min_length > 0: - if len(first) >= self.min_length or \ + if len(first) >= self.min_length and \ len(second) >= self.min_length: yield read_num, True, first, second From 975591c5e185a27624714aeffd55dd5bf0e7982f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 6 May 2017 12:48:25 +0800 Subject: [PATCH 091/185] Move GuardedKmerMap into its own file and template in thread safety --- include/oxli/gmap.hh | 143 +++++++++++++++++++++++++++++++++++ include/oxli/partitioning.hh | 85 ++------------------- khmer/_oxli/wrapper.pxd | 4 +- setup.py | 2 +- src/oxli/Makefile | 7 +- src/oxli/partitioning.cc | 4 +- 6 files changed, 158 insertions(+), 87 deletions(-) create mode 100644 include/oxli/gmap.hh diff --git a/include/oxli/gmap.hh b/include/oxli/gmap.hh new file mode 100644 index 0000000000..1a5cff3ee4 --- /dev/null +++ b/include/oxli/gmap.hh @@ -0,0 +1,143 @@ +/* +This file is part of khmer, https://github.com/dib-lab/khmer/, and is +Copyright (C) 2015-2016, The Regents of the University of California. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the Michigan State University nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +LICENSE (END) + +Contact: khmer-project@idyll.org +*/ +#ifndef GMAP_HH +#define GMAP_HH + +#include +#include + +#include "oxli.hh" +#include "kmer_hash.hh" +#include "hashtable.hh" +#include "hashgraph.hh" + +namespace oxli { + + +template +class GuardedHashMap { + + public: + + // Filter should be owned exclusively by GuardedKmerMap + std::unique_ptr filter; + std::map data; + + explicit GuardedHashMap(WordLength ksize, + unsigned short n_tables, + uint64_t max_table_size) + { + std::vector table_sizes = get_n_primes_near_x(n_tables, max_table_size); + filter = std::unique_ptr(new Nodegraph(ksize, table_sizes)); + } + + T get(HashIntoType kmer) const + { + if (filter->get_count(kmer)) { + auto search = data.find(kmer); + if (search != data.end()) { + return search->second; + } + } + + return NULL; + } + + void set(HashIntoType kmer, T item) + { + filter->count(kmer); + data[kmer] = item; + } + + bool contains(HashIntoType kmer) const + { + return get(kmer) != NULL; + } + + uint64_t size() const + { + return data.size(); + } +}; + +template +class GuardedHashMap: public GuardedHashMap +{ + private: + + uint32_t lock; + + public: + + using GuardedHashMap::GuardedHashMap; + using GuardedHashMap::filter; + using GuardedHashMap::data; + + explicit GuardedHashMap(WordLength ksize, + unsigned short n_tables, + uint64_t max_table_size) : + GuardedHashMap(ksize, n_tables, max_table_size), + lock(0) + { + } + + T get(HashIntoType kmer) const + { + if (filter->get_count(kmer)) { + while(!__sync_bool_compare_and_swap( &lock, 0, 1)); + auto search = data.find(kmer); + if (search != data.end()) { + __sync_bool_compare_and_swap( &lock, 1, 0); + return search->second; + } + __sync_bool_compare_and_swap( &lock, 1, 0); + } + + return NULL; + } + + void set(HashIntoType kmer, T item) + { + while(!__sync_bool_compare_and_swap( &lock, 0, 1)); + set(kmer, item); + __sync_bool_compare_and_swap( &lock, 1, 0); + } +}; + +} + +#endif diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index bd6f2b7b83..5c51ead0d4 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -40,6 +40,7 @@ Contact: khmer-project@idyll.org #include #include +#include "gmap.hh" #include "oxli.hh" #include "kmer_hash.hh" #include "hashtable.hh" @@ -55,82 +56,6 @@ namespace oxli { -template -class GuardedKmerMap { - - private: - - uint32_t lock; - - public: - - // Filter should be owned exclusively by GuardedKmerMap - std::unique_ptr filter; - std::map data; - - explicit GuardedKmerMap(WordLength ksize, - unsigned short n_tables, - uint64_t max_table_size): lock(0) - { - std::vector table_sizes = get_n_primes_near_x(n_tables, max_table_size); - filter = std::unique_ptr(new Nodegraph(ksize, table_sizes)); - } - - T get(HashIntoType kmer) const { - if (filter->get_count(kmer)) { - auto search = data.find(kmer); - if (search != data.end()) { - return search->second; - } - } - - return NULL; - } - - T get_threadsafe(HashIntoType kmer) const { - if (filter->get_count(kmer)) { - acquire_lock(); - auto search = data.find(kmer); - if (search != data.end()) { - release_lock(); - return search->second; - } - release_lock(); - } - - return NULL; - } - - void set(HashIntoType kmer, T item) { - filter->count(kmer); - data[kmer] = item; - } - - void set_threadsafe(HashIntoType kmer, T item) { - acquire_lock(); - set(kmer, item); - release_lock(); - } - - bool contains(HashIntoType kmer) const { - return get(kmer) != NULL; - } - - uint64_t size() const { - return data.size(); - } - - inline void acquire_lock() { - while(!__sync_bool_compare_and_swap( &lock, 0, 1)); - } - - inline void release_lock() { - __sync_bool_compare_and_swap( &lock, 1, 0); - } - -}; - - class Component; typedef std::shared_ptr ComponentPtr; @@ -143,7 +68,7 @@ class ComponentPtrCompare { class ComponentPtrCompare; typedef std::set ComponentPtrSet; -typedef GuardedKmerMap GuardedKmerCompMap; +typedef GuardedHashMap GuardedHashCompMap; class Component { @@ -219,7 +144,7 @@ class StreamingPartitioner { uint32_t _tag_density; // We should exclusively own tag_component_map. - std::shared_ptr tag_component_map; + std::shared_ptr tag_component_map; std::shared_ptr components; uint32_t components_lock; uint64_t n_consumed; @@ -279,8 +204,8 @@ class StreamingPartitioner { return std::weak_ptr(components); } - std::weak_ptr get_tag_component_map() const { - return std::weak_ptr(tag_component_map); + std::weak_ptr get_tag_component_map() const { + return std::weak_ptr(tag_component_map); } inline void acquire_components() { diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index 351708cb6f..352c164424 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -440,7 +440,7 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": ctypedef shared_ptr[CpComponent] ComponentPtr ctypedef set[ComponentPtr] ComponentPtrSet - cdef cppclass CpGuardedKmerCompMap "oxli::GuardedKmerCompMap": + cdef cppclass CpGuardedHashCompMap "oxli::GuardedHashCompMap": map[HashIntoType, ComponentPtr] data ComponentPtr get(HashIntoType) @@ -483,6 +483,6 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": ComponentPtr get_nearest_component(CpKmer) const weak_ptr[ComponentPtrSet] get_component_set() - weak_ptr[CpGuardedKmerCompMap] get_tag_component_map() + weak_ptr[CpGuardedHashCompMap] get_tag_component_map() diff --git a/setup.py b/setup.py index ef0f8ed421..733aadfccf 100755 --- a/setup.py +++ b/setup.py @@ -155,7 +155,7 @@ def build_dir(): "oxli", "kmer_hash", "hashtable", "labelhash", "hashgraph", "hllcounter", "khmer_exception", "read_aligner", "subset", "read_parsers", "kmer_filters", "traversal", "assembler", "alphabets", "storage", - "partitioning"]) + "partitioning", "gmap"]) SOURCES = glob.glob(path_join("src", "khmer", "_cpy_*.cc")) SOURCES.extend(path_join("src", "oxli", bn + ".cc") for bn in [ diff --git a/src/oxli/Makefile b/src/oxli/Makefile index dea6209c58..391685e934 100644 --- a/src/oxli/Makefile +++ b/src/oxli/Makefile @@ -236,7 +236,8 @@ LIBOXLI_OBJS= \ alphabets.o \ murmur3.o \ storage.o \ - partitioning.o + partitioning.o \ + gmap.o PRECOMILE_OBJS ?= PRECLEAN_TARGS ?= @@ -269,7 +270,9 @@ HEADERS= \ assembler.hh \ alphabets.hh \ storage.hh \ - partitioning.hh + partitioning.hh \ + gmap.hh + OXLI_HEADERS = $(addprefix ../../include/oxli/,$(HEADERS)) # START OF RULES # diff --git a/src/oxli/partitioning.cc b/src/oxli/partitioning.cc index 2696ae9445..2432202984 100644 --- a/src/oxli/partitioning.cc +++ b/src/oxli/partitioning.cc @@ -42,8 +42,8 @@ StreamingPartitioner::StreamingPartitioner(Hashgraph * graph, uint32_t tag_densi // positive rate to be about the same as the graph, we should make its table // sizes proportional by the number of tags. Here, we use _tag_density-2 // because we always tag the first and last k-mers in a read. - tag_component_map = std::unique_ptr( - new GuardedKmerCompMap(graph->ksize(), + tag_component_map = std::unique_ptr( + new GuardedHashCompMap(graph->ksize(), graph->n_tables(), graph_max_table_size / (_tag_density-2))); components = std::make_shared(); From 9e1eb53c6b5fd6b3b3819fd8b9d066dc907b4108 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 6 May 2017 19:58:47 +0800 Subject: [PATCH 092/185] Remove unused merged method from Component --- include/oxli/partitioning.hh | 9 --------- khmer/_oxli/wrapper.pxd | 1 - 2 files changed, 10 deletions(-) diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index 5c51ead0d4..6f3f21432b 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -95,15 +95,6 @@ class Component { n_destroyed++; } - void merge(ComponentPtrSet other_comps) { - for (auto other : other_comps) { - if (*other == *this) { - continue; - } - this->add_tag(other->tags); - } - } - uint64_t get_n_created() const { return n_created; } diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index 352c164424..6fe27f5134 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -430,7 +430,6 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": const uint64_t component_id set[HashIntoType] tags - void merge(set[shared_ptr[CpComponent]]) void add_tag(HashIntoType) void add_tag(set[HashIntoType]) uint64_t get_n_tags() const From cf9b75f0ee99733361ce0700099f65bdcef04382 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 7 May 2017 16:14:15 +0800 Subject: [PATCH 093/185] Factor out tag to component mapping from Partitioner and remove most stdset usage, temp breaks loading --- include/oxli/partitioning.hh | 137 ++++++++++++------ khmer/_oxli/partitioning.pxd | 6 +- khmer/_oxli/partitioning.pyx | 21 ++- khmer/_oxli/wrapper.pxd | 46 ++++-- src/oxli/partitioning.cc | 272 +++++++++++++++++------------------ 5 files changed, 278 insertions(+), 204 deletions(-) diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index 6f3f21432b..3bcfe8abc6 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -68,6 +68,8 @@ class ComponentPtrCompare { class ComponentPtrCompare; typedef std::set ComponentPtrSet; +typedef std::vector ComponentPtrVector; +typedef std::vector TagVector; typedef GuardedHashMap GuardedHashCompMap; @@ -77,13 +79,14 @@ class Component { static uint64_t n_created; static uint64_t n_destroyed; + bool alive; public: const uint64_t component_id; - std::set tags; + TagVector tags; - explicit Component(): component_id(n_created) { + explicit Component(): component_id(n_created), alive(true) { n_created++; } @@ -95,6 +98,15 @@ class Component { n_destroyed++; } + void kill() { + tags.clear(); + alive = false; + } + + bool is_alive() const { + return alive; + } + uint64_t get_n_created() const { return n_created; } @@ -104,13 +116,13 @@ class Component { } void add_tag(HashIntoType tag) { - tags.insert(tag); + tags.push_back(tag); } - void add_tag(std::set& new_tags) { - for (auto tag: new_tags) { - add_tag(tag); - } + void add_tags(TagVector& new_tags) { + tags.insert(tags.end(), + new_tags.begin(), + new_tags.end()); } uint64_t get_n_tags() const { @@ -129,15 +141,71 @@ class Component { }; -class StreamingPartitioner { +class ComponentMap { private: - uint32_t _tag_density; // We should exclusively own tag_component_map. std::shared_ptr tag_component_map; - std::shared_ptr components; + std::shared_ptr components; uint32_t components_lock; + uint64_t component_counter; + uint64_t n_live_components; + + public: + + + explicit ComponentMap(WordLength ksize, + WordLength n_tables, + uint64_t max_table_size); + + void create_component(TagVector& tags); + uint32_t create_and_merge_components(TagVector& tags); + void map_tags_to_component(TagVector& tags, ComponentPtr& comp); + uint32_t merge_components(ComponentPtr& root, ComponentPtrSet& comps); + + bool contains(HashIntoType tag) + { + return tag_component_map->contains(tag); + } + + ComponentPtr get(HashIntoType tag) const { + return tag_component_map->get(tag); + } + + uint64_t get_n_components() const { + return n_live_components; + } + + uint64_t get_n_tags() const { + return tag_component_map->size(); + } + + std::weak_ptr get_components() const { + return std::weak_ptr(components); + } + + std::weak_ptr get_tag_component_map() const { + return std::weak_ptr(tag_component_map); + } + + inline void acquire_components() { + while(!__sync_bool_compare_and_swap( &components_lock, 0, 1)); + } + + inline void release_components() { + __sync_bool_compare_and_swap( &components_lock, 1, 0); + } +}; + + +class StreamingPartitioner { + + private: + + uint32_t _tag_density; + // We should exclusively own tag_component_map. + std::shared_ptr partitions; uint64_t n_consumed; public: @@ -152,59 +220,40 @@ class StreamingPartitioner { uint64_t consume(const std::string& seq); uint64_t consume_pair(const std::string& first, const std::string& second); + uint64_t seed_sequence(const std::string& seq, - std::set& tags, + TagVector& tags, KmerQueue& seeds, std::set& seen); - uint32_t create_and_connect_components(std::set& tags); uint64_t consume_fasta(std::string const &filename); - void map_tags_to_component(std::set& tags, ComponentPtr& comp); - void add_component(ComponentPtr comp); void find_connected_tags(KmerQueue& node_q, - std::set& found_tags, + TagVector& found_tags, std::set& seen, bool truncate=false) const; - uint32_t merge_components(ComponentPtr& root, ComponentPtrSet& comps); - - - - uint64_t get_n_components() const { - return components->size(); - } - - uint64_t get_n_tags() const { - return tag_component_map->size(); - } - uint64_t get_n_consumed() const { - return n_consumed; - } - - uint32_t get_tag_density() const { - return _tag_density; - } - - ComponentPtr get_tag_component(HashIntoType tag) const; - ComponentPtr get_tag_component(std::string& tag) const; - + ComponentPtr get_tag_component(std::string& kmer) const; ComponentPtr get_nearest_component(Kmer kmer) const; ComponentPtr get_nearest_component(std::string& kmer) const; - std::weak_ptr get_component_set() const { - return std::weak_ptr(components); + std::weak_ptr get_components() const { + return partitions->get_components(); } std::weak_ptr get_tag_component_map() const { - return std::weak_ptr(tag_component_map); + return partitions->get_tag_component_map(); } - inline void acquire_components() { - while(!__sync_bool_compare_and_swap( &components_lock, 0, 1)); + uint64_t get_n_components() const { + return partitions->get_n_components(); } - inline void release_components() { - __sync_bool_compare_and_swap( &components_lock, 1, 0); + uint64_t get_n_tags() const { + return partitions->get_n_tags(); + } + + uint32_t get_tag_density() const { + return _tag_density; } }; diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index 483ecc8b81..50d8ef1d0c 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -3,7 +3,7 @@ from libcpp.vector cimport vector from libc.stdint cimport uint32_t, uint8_t, uint64_t from libc.stdio cimport FILE -from wrapper cimport ComponentPtr, ComponentPtrSet, CpGuardedKmerCompMap +from wrapper cimport ComponentPtr, ComponentPtrVector, CpGuardedHashCompMap from wrapper cimport CpHashgraph, CpStreamingPartitioner, BoundedCounterType @@ -27,8 +27,8 @@ cdef class Component: cdef class StreamingPartitioner: cdef shared_ptr[CpStreamingPartitioner] _this - cdef weak_ptr[ComponentPtrSet] _components - cdef weak_ptr[CpGuardedKmerCompMap] _tag_component_map + cdef weak_ptr[ComponentPtrVector] _components + cdef weak_ptr[CpGuardedHashCompMap] _tag_component_map cdef CpHashgraph * _graph_ptr cdef readonly object graph cdef readonly uint64_t n_consumed diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 511dbbde20..df79c3c234 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -44,6 +44,12 @@ cdef class Component: def _n_destroyed(self): return deref(self._this).get_n_destroyed() + def __repr__(self): + status = 'ALIVE' if deref(self._this).is_alive() else 'DEAD' + return ''.format(self.component_id, + len(self), + status) + def __len__(self): return deref(self._this).get_n_tags() @@ -129,7 +135,7 @@ cdef class StreamingPartitioner: self._this.reset(new CpStreamingPartitioner(self._graph_ptr, tag_density)) self._tag_component_map = deref(self._this).get_tag_component_map() - self._components = deref(self._this).get_component_set() + self._components = deref(self._this).get_components() self.n_consumed = 0 def consume(self, sequence): @@ -163,17 +169,18 @@ cdef class StreamingPartitioner: return Component.wrap(compptr) def components(self): - cdef shared_ptr[ComponentPtrSet] locked + cdef shared_ptr[ComponentPtrVector] locked cdef ComponentPtr cmpptr lockedptr = self._components.lock() if lockedptr: for cmpptr in deref(lockedptr): - yield Component.wrap(cmpptr) + if cmpptr != NULL: + yield Component.wrap(cmpptr) else: raise MemoryError("Can't locked underlying Component set") def tag_components(self): - cdef shared_ptr[CpGuardedKmerCompMap] locked + cdef shared_ptr[CpGuardedHashCompMap] locked cdef pair[HashIntoType,ComponentPtr] cpair locked = self._tag_component_map.lock() if locked: @@ -189,7 +196,7 @@ cdef class StreamingPartitioner: raise IOError('Can\'t open file.') cdef ComponentPtr cmpptr - cdef shared_ptr[ComponentPtrSet] lockedptr + cdef shared_ptr[ComponentPtrVector] lockedptr lockedptr = self._components.lock() if lockedptr: @@ -218,7 +225,7 @@ cdef class StreamingPartitioner: cdef Component comp cdef int i - cdef shared_ptr[ComponentPtrSet] locked + cdef shared_ptr[ComponentPtrVector] locked locked = self._components.lock() if locked: for i, comp in enumerate(self.components()): @@ -227,7 +234,7 @@ cdef class StreamingPartitioner: comp.save(fp) fprintf(fp, "\n]}") fclose(fp) - + @staticmethod def load(filename): diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index 6fe27f5134..1e53501ce6 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -423,21 +423,28 @@ cdef extern from "oxli/hllcounter.hh" namespace "oxli": cdef extern from "oxli/partitioning.hh" namespace "oxli": + ctypedef vector[HashIntoType] TagVector + cdef cppclass CpComponent "oxli::Component": CpComponent() CpComponent(uint64_t) const uint64_t component_id - set[HashIntoType] tags + vector[HashIntoType] tags + + void kill() + bool is_alive() const void add_tag(HashIntoType) - void add_tag(set[HashIntoType]) + void add_tags(TagVector&) + uint64_t get_n_tags() const uint64_t get_n_created() const uint64_t get_n_destroyed() const ctypedef shared_ptr[CpComponent] ComponentPtr ctypedef set[ComponentPtr] ComponentPtrSet + ctypedef vector[ComponentPtr] ComponentPtrVector cdef cppclass CpGuardedHashCompMap "oxli::GuardedHashCompMap": map[HashIntoType, ComponentPtr] data @@ -446,6 +453,22 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": void set(HashIntoType, ComponentPtr) bool contains(HashIntoType) + cdef cppclass CpComponentMap "oxli::ComponentMap": + CpComponentMap(WordLength, WordLength, uint64_t) + + void create_component(TagVector&) + uint32_t create_and_merge_components(TagVector&) + void map_tags_to_component(TagVector&, ComponentPtr&) + uint32_t merge_components(ComponentPtr&, ComponentPtrSet&) + + bool contains(HashIntoType) + ComponentPtr get(HashIntoType) const + + uint64_t get_n_components() const + uint64_t get_n_tags() const + weak_ptr[ComponentPtrVector] get_components() + weak_ptr[CpGuardedHashCompMap] get_tag_component_map() + cdef cppclass CpStreamingPartitioner "oxli::StreamingPartitioner": CpStreamingPartitioner(CpHashgraph * ) except +MemoryError CpStreamingPartitioner(CpHashgraph *, uint32_t) except +MemoryError @@ -455,33 +478,28 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": uint64_t consume_pair(string&, string&) nogil except +MemoryError uint64_t consume_fasta(string&) except +MemoryError - uint64_t seed_sequence(string&, set[HashIntoType]&, KmerQueue&, + uint64_t seed_sequence(string&, TagVector&, KmerQueue&, set[HashIntoType]&) except +MemoryError - uint32_t create_and_connect_components(set[HashIntoType]&) - void add_component(ComponentPtr comp) - void map_tags_to_component(set[HashIntoType]&, ComponentPtr&) void find_connected_tags(queue[CpKmer]&, - set[HashIntoType]&, + TagVector&, set[HashIntoType]&) except +MemoryError void find_connected_tags(queue[CpKmer]&, - set[HashIntoType]&, + TagVector&, set[HashIntoType]&, bool) except +MemoryError - uint64_t get_n_components() const - uint64_t get_n_tags() const + uint64_t get_n_consumed() const uint32_t get_tag_density() const + uint64_t get_n_components() const + uint64_t get_n_tags() const ComponentPtr get_tag_component(string&) const - ComponentPtr get_tag_component(HashIntoType) const - ComponentPtr get_nearest_component(string&) const ComponentPtr get_nearest_component(CpKmer) const - weak_ptr[ComponentPtrSet] get_component_set() + weak_ptr[ComponentPtrVector] get_components() weak_ptr[CpGuardedHashCompMap] get_tag_component_map() - diff --git a/src/oxli/partitioning.cc b/src/oxli/partitioning.cc index 2432202984..b75cc0475e 100644 --- a/src/oxli/partitioning.cc +++ b/src/oxli/partitioning.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include "oxli/hashtable.hh" #include "oxli/hashgraph.hh" @@ -26,43 +27,136 @@ inline std::ostream& operator<< (std::ostream& stream, Component& comp) { return stream; } +//////////////////////////////////////////////////////////////////////////////// -StreamingPartitioner::StreamingPartitioner(Hashgraph * graph, uint32_t tag_density) : - graph(graph), _tag_density(tag_density), components_lock(0), n_consumed(0) + +ComponentMap::ComponentMap(WordLength ksize, + WordLength n_tables, + uint64_t max_table_size) : components_lock(0), + component_counter(0), + n_live_components(0) +{ + + tag_component_map = std::unique_ptr( + new GuardedHashCompMap(ksize, + n_tables, + max_table_size)); + components = std::make_shared(); +} + +void ComponentMap::map_tags_to_component(TagVector& tags, + ComponentPtr& comp) +{ + for (auto tag: tags) { + tag_component_map->set(tag, comp); + comp->add_tag(tag); + } +} + +void ComponentMap::create_component(TagVector& tags) +{ + ComponentPtr new_comp = std::make_shared(component_counter); + component_counter++; + n_live_components++; + components->push_back(new_comp); + map_tags_to_component(tags, new_comp); + + std::cout << "new component=" << *new_comp << std::endl; + std::cout << components->size() << " components in vector" << std::endl; +} + + +uint32_t ComponentMap::create_and_merge_components(TagVector& tags) +{ + + // Now resolve components. First, get components from existing tags. + ComponentPtrSet found_comps; + TagVector new_tags; + for (auto tag: tags) { + ComponentPtr comp; + if ((comp = tag_component_map->get(tag)) != NULL) { + found_comps.insert(comp); + } else { + new_tags.push_back(tag); + } + } + + uint32_t n_merged = 1; + if (found_comps.size() == 0) { + create_component(tags); + } else { + // Choose the largest component as the root + // We want to minimize tag copying + ComponentPtr root_comp = *(found_comps.begin()); + for (auto other : found_comps) { + if (other->get_n_tags() > root_comp->get_n_tags()) { + root_comp = other; + } + } + // map the new tags to this component + root_comp->add_tags(new_tags); + map_tags_to_component(new_tags, root_comp); + if (found_comps.size() > 1) { + n_merged = merge_components(root_comp, found_comps); + } + } + return n_merged; +} + + +uint32_t ComponentMap::merge_components(ComponentPtr& root, + ComponentPtrSet& comps) +{ + uint32_t n_merged = 1; + std::cout << "Merge with root=" << *root << std::endl; + for (auto other : comps) { + std::cout << "\tmerge in " << *other << std::endl; + if (*other == *root) { + continue; + } + root->add_tags(other->tags); // transfer the tags from the other comp + map_tags_to_component(other->tags, root); + (*components)[other->component_id]->kill(); + (*components)[other->component_id] = nullptr; + n_live_components--; + n_merged++; + + } + // and active Python wrapper; this leaves them as sole owners + return n_merged; +} + + +//////////////////////////////////////////////////////////////////////////////// + + +StreamingPartitioner::StreamingPartitioner(Hashgraph * graph, + uint32_t tag_density) : + graph(graph), + _tag_density(tag_density), + n_consumed(0) { //if (auto graphptr = graph.lock()) { if (graph != NULL) { - std::vector graph_table_sizes = graph->get_tablesizes(); - uint64_t graph_max_table_size = *std::max_element(graph_table_sizes.begin(), - graph_table_sizes.end()); - // We can guess that, given N k-mers in the graph, there will be // approximately N / _tag_density tags. If we want to the filter false // positive rate to be about the same as the graph, we should make its table // sizes proportional by the number of tags. Here, we use _tag_density-2 // because we always tag the first and last k-mers in a read. - tag_component_map = std::unique_ptr( - new GuardedHashCompMap(graph->ksize(), - graph->n_tables(), - graph_max_table_size / (_tag_density-2))); - components = std::make_shared(); + std::vector graph_table_sizes = graph->get_tablesizes(); + uint64_t graph_max_table_size = *std::max_element(graph_table_sizes.begin(), + graph_table_sizes.end()); + + partitions = std::make_shared(graph->ksize(), + graph->n_tables(), + graph_max_table_size / (_tag_density-2)); } else { throw oxli_ptr_exception("Hashgraph has been deleted."); } } -void StreamingPartitioner::map_tags_to_component(std::set& tags, - ComponentPtr& comp) -{ - for (auto tag: tags) { - tag_component_map->set(tag, comp); - comp->add_tag(tag); - } -} - - uint64_t StreamingPartitioner::consume_fasta(const std::string& filename) { ReadParserPtr parser = get_parser(filename); @@ -93,26 +187,25 @@ uint64_t StreamingPartitioner::consume_fasta(const std::string& filename) } - uint64_t StreamingPartitioner::consume(const std::string& seq) { - std::set tags; + TagVector tags; KmerQueue seeds; std::set seen; uint64_t n_new = seed_sequence(seq, tags, seeds, seen); find_connected_tags(seeds, tags, seen, false); //acquire_components(); - create_and_connect_components(tags); + partitions->create_and_merge_components(tags); //release_components(); return n_new; } uint64_t StreamingPartitioner::consume_pair(const std::string& first, - const std::string& second) + const std::string& second) { - std::set tags; + TagVector tags; KmerQueue seeds; std::set seen; @@ -120,23 +213,23 @@ uint64_t StreamingPartitioner::consume_pair(const std::string& first, n_new += seed_sequence(second, tags, seeds, seen); find_connected_tags(seeds, tags, seen, false); //acquire_components(); - create_and_connect_components(tags); + partitions->create_and_merge_components(tags); //release_components(); return n_new; } -void StreamingPartitioner::add_component(ComponentPtr comp) + +ComponentPtr StreamingPartitioner::get_tag_component(std::string& kmer) const { - components->insert(comp); - map_tags_to_component(comp->tags, comp); + HashIntoType h = graph->hash_dna(kmer.c_str()); + return partitions->get(h); } - uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, - std::set& tags, - KmerQueue& seeds, - std::set& seen) + TagVector& tags, + KmerQueue& seeds, + std::set& seen) { /* For the following comments, let G be the set of k-mers * known in the graph before inserting the k-mers R from @@ -190,10 +283,10 @@ uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, // to the seen set, as we do not need to traverse from them in the tag search. intersection.insert(kmer); in_known_territory = true; - kmer_tagged = tag_component_map->contains(kmer); + kmer_tagged = partitions->contains(kmer); if (kmer_tagged) { since = 1; - tags.insert(kmer); + tags.push_back(kmer); found_tag_in_territory = true; } else { ++since; @@ -201,14 +294,14 @@ uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, } if (since >= _tag_density) { - tags.insert(kmer); + tags.push_back(kmer); since = 1; } } while (!kmers.done()); // always tag the last k-mer if (since >= _tag_density / 2) { - tags.insert(kmer); + tags.push_back(kmer); } seeds.push(kmer); @@ -227,99 +320,6 @@ uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, return n_new; } - -uint32_t StreamingPartitioner::create_and_connect_components(std::set &tags) -{ - - // Now resolve components. First, get components from existing tags. - ComponentPtrSet found_comps; -#if(DEBUG_SP) - std::cout << "Get found comps: " << std::endl; -#endif - for (auto tag: tags) { -#if(DEBUG_SP) - std::cout << "Tag: " << tag; -#endif - ComponentPtr comp; - if ((comp = tag_component_map->get(tag)) != NULL) { -#if(DEBUG_SP) - std::cout << "->" << *comp; -#endif - found_comps.insert(comp); - } -#if(DEBUG_SP) - std::cout << std::endl; -#endif - } - -#if(DEBUG_SP) - std::cout << found_comps.size() << " unique components." << std::endl; -#endif - - uint32_t n_merged = 1; - if (found_comps.size() == 0) { - ComponentPtr new_comp = std::make_shared(); -#if(DEBUG_SP) - std::cout << "Build new comp: " << *new_comp << std::endl; -#endif - components->insert(new_comp); - map_tags_to_component(tags, new_comp); - } else { - // Choose the largest component as the root - // We want to minimize tag copying - ComponentPtr root_comp = *(found_comps.begin()); - for (auto other : found_comps) { - if (other->get_n_tags() > root_comp->get_n_tags()) { - root_comp = other; - } - } -#if(DEBUG_SP) - std::cout << "Merge into: " << *root_comp << std::endl; -#endif - // map the new tags to this component - root_comp->add_tag(tags); - map_tags_to_component(tags, root_comp); - if (found_comps.size() > 1) { - n_merged = merge_components(root_comp, found_comps); - } - } - return n_merged; -} - - -uint32_t StreamingPartitioner::merge_components(ComponentPtr& root, - ComponentPtrSet& comps) -{ - uint32_t n_merged = 1; - for (auto other : comps) { - if (*other == *root) { - continue; - } - root->add_tag(other->tags); // transfer the tags from the other comp - map_tags_to_component(other->tags, root); // set the other's tags to point to root - components->erase(other); // remove other component entirely - n_merged++; - - } - comps.clear(); // should call destructor on all the merged comps, unless they have - // and active Python wrapper; this leaves them as sole owners - return n_merged; -} - - -ComponentPtr StreamingPartitioner::get_tag_component(HashIntoType tag) const -{ - return tag_component_map->get(tag); -} - - -ComponentPtr StreamingPartitioner::get_tag_component(std::string& kmer) const -{ - HashIntoType h = graph->hash_dna(kmer.c_str()); - return get_tag_component(h); -} - - ComponentPtr StreamingPartitioner::get_nearest_component(std::string& kmer) const { Kmer hashed = graph->build_kmer(kmer); @@ -329,7 +329,7 @@ ComponentPtr StreamingPartitioner::get_nearest_component(std::string& kmer) cons ComponentPtr StreamingPartitioner::get_nearest_component(Kmer kmer) const { - std::set tags; + TagVector tags; std::set seen; KmerQueue node_q; node_q.push(kmer); @@ -337,7 +337,7 @@ ComponentPtr StreamingPartitioner::get_nearest_component(Kmer kmer) const find_connected_tags(node_q, tags, seen, true); if (tags.size() > 0) { HashIntoType tag = *(tags.begin()); - return tag_component_map->get(tag); + return partitions->get(tag); } else { return NULL; } @@ -345,7 +345,7 @@ ComponentPtr StreamingPartitioner::get_nearest_component(Kmer kmer) const void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, - std::set& found_tags, + TagVector& found_tags, std::set& seen, bool truncate) const { @@ -380,8 +380,8 @@ void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, total++; // Found a tag! - if (tag_component_map->contains(node)) { - found_tags.insert(node); + if (partitions->contains(node)) { + found_tags.push_back(node); if (truncate) { return; } From adfba25d062e606d879a56dc2fac8657a5ba5892 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sun, 7 May 2017 16:22:04 +0800 Subject: [PATCH 094/185] remove stale comment --- include/oxli/partitioning.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index 3bcfe8abc6..ac56388f9e 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -204,7 +204,6 @@ class StreamingPartitioner { private: uint32_t _tag_density; - // We should exclusively own tag_component_map. std::shared_ptr partitions; uint64_t n_consumed; From 64172655a9d3235bbf2df038bb7d71bbfbc16d22 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 8 May 2017 11:04:24 +0800 Subject: [PATCH 095/185] Have StreamingPartitioner derive ComponentnMap --- include/oxli/partitioning.hh | 30 +++++------------ khmer/_oxli/partitioning.pyx | 11 ++++--- khmer/_oxli/wrapper.pxd | 13 +++----- src/oxli/partitioning.cc | 53 +++++++++++++++---------------- tests/test_cython_partitioning.py | 8 ++--- 5 files changed, 48 insertions(+), 67 deletions(-) diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index ac56388f9e..4e99cdf22f 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -164,7 +164,7 @@ class ComponentMap { void map_tags_to_component(TagVector& tags, ComponentPtr& comp); uint32_t merge_components(ComponentPtr& root, ComponentPtrSet& comps); - bool contains(HashIntoType tag) + bool contains(HashIntoType tag) const { return tag_component_map->contains(tag); } @@ -199,13 +199,13 @@ class ComponentMap { }; -class StreamingPartitioner { +class StreamingPartitioner: public ComponentMap { private: uint32_t _tag_density; - std::shared_ptr partitions; uint64_t n_consumed; + uint64_t _cstr_get_max_table_size(Hashgraph * graph); public: // We're not graph's owner, simply an observer. @@ -219,37 +219,23 @@ class StreamingPartitioner { uint64_t consume(const std::string& seq); uint64_t consume_pair(const std::string& first, const std::string& second); + uint64_t consume_fasta(std::string const &filename); uint64_t seed_sequence(const std::string& seq, TagVector& tags, KmerQueue& seeds, std::set& seen); - uint64_t consume_fasta(std::string const &filename); void find_connected_tags(KmerQueue& node_q, TagVector& found_tags, std::set& seen, bool truncate=false) const; - ComponentPtr get_tag_component(std::string& kmer) const; - ComponentPtr get_nearest_component(Kmer kmer) const; - ComponentPtr get_nearest_component(std::string& kmer) const; + ComponentPtr get(std::string& kmer) const; + ComponentPtr get(HashIntoType h) const; + ComponentPtr find_nearest_component(Kmer kmer) const; + ComponentPtr find_nearest_component(std::string& kmer) const; - std::weak_ptr get_components() const { - return partitions->get_components(); - } - - std::weak_ptr get_tag_component_map() const { - return partitions->get_tag_component_map(); - } - - uint64_t get_n_components() const { - return partitions->get_n_components(); - } - - uint64_t get_n_tags() const { - return partitions->get_n_tags(); - } uint32_t get_tag_density() const { return _tag_density; diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index df79c3c234..5cd6d6bf49 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -150,19 +150,19 @@ cdef class StreamingPartitioner: def consume_fasta(self, filename): return deref(self._this).consume_fasta(filename.encode('utf-8')) - def get_tag_component(self, kmer): + def get(self, kmer): cdef ComponentPtr compptr cdef string kmer_s = _bstring(kmer) - compptr = deref(self._this).get_tag_component(kmer_s) + compptr = deref(self._this).get(kmer_s) if compptr == NULL: return None else: return Component.wrap(compptr) - def get_nearest_component(self, kmer): + def find_nearest_component(self, kmer): cdef ComponentPtr compptr cdef string kmer_s = _bstring(kmer) - compptr = deref(self._this).get_nearest_component(kmer_s) + compptr = deref(self._this).find_nearest_component(kmer_s) if compptr == NULL: return None else: @@ -234,7 +234,7 @@ cdef class StreamingPartitioner: comp.save(fp) fprintf(fp, "\n]}") fclose(fp) - + ''' @staticmethod def load(filename): @@ -259,6 +259,7 @@ cdef class StreamingPartitioner: comp_info['tags']) deref(partitioner._this).add_component(comp_ptr) return partitioner + ''' @property def component_dict(self): diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index 1e53501ce6..86efee4531 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -469,7 +469,7 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": weak_ptr[ComponentPtrVector] get_components() weak_ptr[CpGuardedHashCompMap] get_tag_component_map() - cdef cppclass CpStreamingPartitioner "oxli::StreamingPartitioner": + cdef cppclass CpStreamingPartitioner "oxli::StreamingPartitioner" (CpComponentMap): CpStreamingPartitioner(CpHashgraph * ) except +MemoryError CpStreamingPartitioner(CpHashgraph *, uint32_t) except +MemoryError @@ -490,16 +490,11 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": set[HashIntoType]&, bool) except +MemoryError + ComponentPtr find_nearest_component(string&) const + ComponentPtr find_nearest_component(CpKmer) const uint64_t get_n_consumed() const uint32_t get_tag_density() const - uint64_t get_n_components() const - uint64_t get_n_tags() const - - ComponentPtr get_tag_component(string&) const - ComponentPtr get_nearest_component(string&) const - ComponentPtr get_nearest_component(CpKmer) const - weak_ptr[ComponentPtrVector] get_components() - weak_ptr[CpGuardedHashCompMap] get_tag_component_map() + ComponentPtr get(string&) const diff --git a/src/oxli/partitioning.cc b/src/oxli/partitioning.cc index b75cc0475e..fd85469c4d 100644 --- a/src/oxli/partitioning.cc +++ b/src/oxli/partitioning.cc @@ -132,28 +132,21 @@ uint32_t ComponentMap::merge_components(ComponentPtr& root, StreamingPartitioner::StreamingPartitioner(Hashgraph * graph, uint32_t tag_density) : + ComponentMap::ComponentMap(graph->ksize(), + graph->n_tables(), + _cstr_get_max_table_size(graph)), graph(graph), _tag_density(tag_density), n_consumed(0) { +} - //if (auto graphptr = graph.lock()) { - if (graph != NULL) { - // We can guess that, given N k-mers in the graph, there will be - // approximately N / _tag_density tags. If we want to the filter false - // positive rate to be about the same as the graph, we should make its table - // sizes proportional by the number of tags. Here, we use _tag_density-2 - // because we always tag the first and last k-mers in a read. - std::vector graph_table_sizes = graph->get_tablesizes(); - uint64_t graph_max_table_size = *std::max_element(graph_table_sizes.begin(), - graph_table_sizes.end()); - - partitions = std::make_shared(graph->ksize(), - graph->n_tables(), - graph_max_table_size / (_tag_density-2)); - } else { - throw oxli_ptr_exception("Hashgraph has been deleted."); - } + +uint64_t StreamingPartitioner::_cstr_get_max_table_size(Hashgraph * graph) +{ + std::vector graph_table_sizes = graph->get_tablesizes(); + return *std::max_element(graph_table_sizes.begin(), + graph_table_sizes.end()); } @@ -196,7 +189,7 @@ uint64_t StreamingPartitioner::consume(const std::string& seq) uint64_t n_new = seed_sequence(seq, tags, seeds, seen); find_connected_tags(seeds, tags, seen, false); //acquire_components(); - partitions->create_and_merge_components(tags); + create_and_merge_components(tags); //release_components(); return n_new; } @@ -213,16 +206,22 @@ uint64_t StreamingPartitioner::consume_pair(const std::string& first, n_new += seed_sequence(second, tags, seeds, seen); find_connected_tags(seeds, tags, seen, false); //acquire_components(); - partitions->create_and_merge_components(tags); + create_and_merge_components(tags); //release_components(); return n_new; } -ComponentPtr StreamingPartitioner::get_tag_component(std::string& kmer) const +ComponentPtr StreamingPartitioner::get(std::string& kmer) const { HashIntoType h = graph->hash_dna(kmer.c_str()); - return partitions->get(h); + return ComponentMap::get(h); +} + + +ComponentPtr StreamingPartitioner::get(HashIntoType h) const +{ + return ComponentMap::get(h); } @@ -283,7 +282,7 @@ uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, // to the seen set, as we do not need to traverse from them in the tag search. intersection.insert(kmer); in_known_territory = true; - kmer_tagged = partitions->contains(kmer); + kmer_tagged = this->contains(kmer); if (kmer_tagged) { since = 1; tags.push_back(kmer); @@ -320,14 +319,14 @@ uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, return n_new; } -ComponentPtr StreamingPartitioner::get_nearest_component(std::string& kmer) const +ComponentPtr StreamingPartitioner::find_nearest_component(std::string& kmer) const { Kmer hashed = graph->build_kmer(kmer); - return get_nearest_component(hashed); + return find_nearest_component(hashed); } -ComponentPtr StreamingPartitioner::get_nearest_component(Kmer kmer) const +ComponentPtr StreamingPartitioner::find_nearest_component(Kmer kmer) const { TagVector tags; std::set seen; @@ -337,7 +336,7 @@ ComponentPtr StreamingPartitioner::get_nearest_component(Kmer kmer) const find_connected_tags(node_q, tags, seen, true); if (tags.size() > 0) { HashIntoType tag = *(tags.begin()); - return partitions->get(tag); + return this->get(tag); } else { return NULL; } @@ -380,7 +379,7 @@ void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, total++; // Found a tag! - if (partitions->contains(node)) { + if (this->contains(node)) { found_tags.push_back(node); if (truncate) { return; diff --git a/tests/test_cython_partitioning.py b/tests/test_cython_partitioning.py index 5434b29d7f..1f5292c711 100644 --- a/tests/test_cython_partitioning.py +++ b/tests/test_cython_partitioning.py @@ -90,7 +90,7 @@ def test_component_n_tags(self, random_sequence): sp.consume(seq) tags = [t for t,c in sp.tag_components()] - comp = sp.get_nearest_component(seq[:K]) + comp = sp.find_nearest_component(seq[:K]) assert len(tags) == len(comp) def test_tag_components_iter(self, random_sequence): @@ -114,7 +114,7 @@ def test_tag_components_iter(self, random_sequence): assert len(comps) == 2 assert len(tags) == sum([len(c) for c in comps]) - def test_get_nearest_component(self, random_sequence): + def test_find_nearest_component(self, random_sequence): seq1 = random_sequence() seq2 = random_sequence(exclude=seq1) @@ -124,8 +124,8 @@ def test_get_nearest_component(self, random_sequence): sp.consume(seq1) sp.consume(seq2) - c1 = sp.get_nearest_component(seq1[:K]) - c2 = sp.get_nearest_component(seq2[:K]) + c1 = sp.find_nearest_component(seq1[:K]) + c2 = sp.find_nearest_component(seq2[:K]) assert c1.component_id != c2.component_id for tag in c1: From 2d3759b9d09b57acb8d42df515706426a18cf3c1 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 9 May 2017 22:49:59 -0700 Subject: [PATCH 096/185] Add some coverage dist tracking --- include/oxli/hist.hh | 99 ++++++++++++++++++++++++++++++++++++ include/oxli/partitioning.hh | 18 +++++-- khmer/_oxli/wrapper.pxd | 2 + setup.py | 2 +- src/oxli/Makefile | 6 ++- 5 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 include/oxli/hist.hh diff --git a/include/oxli/hist.hh b/include/oxli/hist.hh new file mode 100644 index 0000000000..ab599060d6 --- /dev/null +++ b/include/oxli/hist.hh @@ -0,0 +1,99 @@ +/* +This file is part of khmer, https://github.com/dib-lab/khmer/, and is +Copyright (C) 2015-2016, The Regents of the University of California. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the Michigan State University nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +LICENSE (END) + +Contact: khmer-project@idyll.org +*/ +#ifndef HIST_HH +#define HIST_HH + +#include +#include +#include + +#include "oxli.hh" +#include "kmer_hash.hh" + +namespace oxli { + +inline size_t highest_bit(uint64_t num) +{ + if (!num) + return 0; + + int pos = 1; + + while (num >>= 1) { + pos += 1; + } + + return pos; +} + + +template +class Histogram { + + public: + + uint64_t bins[n_bins]; + + Histogram() { + clear(); + } + + void add(uint64_t val) { + size_t bin = highest_bit(val); + if (bin >= n_bins) { + bins[n_bins-1] += 1; + } else { + bins[bin] += 1; + } + } + + void clear() { + for (auto&& b : bins) { + b = 0; + } + } +}; + +template class Histogram<8>; +template class Histogram<16>; +template class Histogram<32>; +template class Histogram<64>; + + +} + +#endif diff --git a/include/oxli/partitioning.hh b/include/oxli/partitioning.hh index 4e99cdf22f..2f02026c87 100644 --- a/include/oxli/partitioning.hh +++ b/include/oxli/partitioning.hh @@ -41,6 +41,7 @@ Contact: khmer-project@idyll.org #include #include "gmap.hh" +#include "hist.hh" #include "oxli.hh" #include "kmer_hash.hh" #include "hashtable.hh" @@ -85,6 +86,7 @@ class Component { const uint64_t component_id; TagVector tags; + Histogram<16> coverage; explicit Component(): component_id(n_created), alive(true) { n_created++; @@ -98,6 +100,13 @@ class Component { n_destroyed++; } + void update_coverage(Hashgraph * graph) { + coverage.clear(); + for (auto tag: tags) { + coverage.add(graph->get_count(tag)); + } + } + void kill() { tags.clear(); alive = false; @@ -129,15 +138,18 @@ class Component { return tags.size(); } - friend bool operator==(const Component& lhs, const Component& rhs) { + friend bool operator==(const Component& lhs, + const Component& rhs) { return lhs.component_id == rhs.component_id; } - friend bool operator<(const Component& lhs, const Component& rhs) { + friend bool operator<(const Component& lhs, + const Component& rhs) { return lhs.component_id < rhs.component_id; } - friend std::ostream& operator<< (std::ostream& stream, const Component& comp); + friend std::ostream& operator<< (std::ostream& stream, + const Component& comp); }; diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index 86efee4531..776ba2e7bf 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -442,6 +442,8 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": uint64_t get_n_created() const uint64_t get_n_destroyed() const + void update_coverage(CpHashgraph *) + ctypedef shared_ptr[CpComponent] ComponentPtr ctypedef set[ComponentPtr] ComponentPtrSet ctypedef vector[ComponentPtr] ComponentPtrVector diff --git a/setup.py b/setup.py index 733aadfccf..3cd7b72c74 100755 --- a/setup.py +++ b/setup.py @@ -155,7 +155,7 @@ def build_dir(): "oxli", "kmer_hash", "hashtable", "labelhash", "hashgraph", "hllcounter", "khmer_exception", "read_aligner", "subset", "read_parsers", "kmer_filters", "traversal", "assembler", "alphabets", "storage", - "partitioning", "gmap"]) + "partitioning", "gmap", "hist"]) SOURCES = glob.glob(path_join("src", "khmer", "_cpy_*.cc")) SOURCES.extend(path_join("src", "oxli", bn + ".cc") for bn in [ diff --git a/src/oxli/Makefile b/src/oxli/Makefile index 391685e934..5c1ddf389a 100644 --- a/src/oxli/Makefile +++ b/src/oxli/Makefile @@ -237,7 +237,8 @@ LIBOXLI_OBJS= \ murmur3.o \ storage.o \ partitioning.o \ - gmap.o + gmap.o \ + hist.o PRECOMILE_OBJS ?= PRECLEAN_TARGS ?= @@ -271,7 +272,8 @@ HEADERS= \ alphabets.hh \ storage.hh \ partitioning.hh \ - gmap.hh + gmap.hh \ + hist.hh OXLI_HEADERS = $(addprefix ../../include/oxli/,$(HEADERS)) From 1727b9e611b0686952cd79c6945946c11522b743 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 9 May 2017 23:31:29 -0700 Subject: [PATCH 097/185] comment debug stuff --- src/oxli/partitioning.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/oxli/partitioning.cc b/src/oxli/partitioning.cc index fd85469c4d..bbfa08f39f 100644 --- a/src/oxli/partitioning.cc +++ b/src/oxli/partitioning.cc @@ -61,8 +61,8 @@ void ComponentMap::create_component(TagVector& tags) components->push_back(new_comp); map_tags_to_component(tags, new_comp); - std::cout << "new component=" << *new_comp << std::endl; - std::cout << components->size() << " components in vector" << std::endl; + //std::cout << "new component=" << *new_comp << std::endl; + //std::cout << components->size() << " components in vector" << std::endl; } @@ -108,9 +108,9 @@ uint32_t ComponentMap::merge_components(ComponentPtr& root, ComponentPtrSet& comps) { uint32_t n_merged = 1; - std::cout << "Merge with root=" << *root << std::endl; + //std::cout << "Merge with root=" << *root << std::endl; for (auto other : comps) { - std::cout << "\tmerge in " << *other << std::endl; + //std::cout << "\tmerge in " << *other << std::endl; if (*other == *root) { continue; } From 2de73ce420b0d5517c40bee21e19cb3bac17ea8b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 9 May 2017 23:33:39 -0700 Subject: [PATCH 098/185] colors --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3cd7b72c74..464dad64c3 100755 --- a/setup.py +++ b/setup.py @@ -168,7 +168,7 @@ def build_dir(): "MurmurHash3"]) # Don't forget to update lib/Makefile with these flags! -EXTRA_COMPILE_ARGS = ['-O3', '-std=c++11', '-pedantic'] +EXTRA_COMPILE_ARGS = ['-O3', '-std=c++11', '-pedantic', '-fdiagnostics-color'] EXTRA_LINK_ARGS = [] if sys.platform == 'darwin': From c11cc00e695584651a0976e3a7a0810afec85c95 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 18 May 2017 09:32:11 -0700 Subject: [PATCH 099/185] Add coverage tracking output --- include/oxli/hist.hh | 2 +- khmer/_oxli/app.pyx | 8 ++++++++ khmer/_oxli/partitioning.pyx | 25 +++++++++++++++++++++++++ khmer/_oxli/wrapper.pxd | 12 ++++++++++++ src/oxli/Makefile | 4 +--- 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/include/oxli/hist.hh b/include/oxli/hist.hh index ab599060d6..51942c142d 100644 --- a/include/oxli/hist.hh +++ b/include/oxli/hist.hh @@ -73,7 +73,7 @@ class Histogram { } void add(uint64_t val) { - size_t bin = highest_bit(val); + size_t bin = highest_bit(val) - 1; if (bin >= n_bins) { bins[n_bins-1] += 1; } else { diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index 9ba3fc0886..126f51a6c6 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -60,6 +60,8 @@ cdef class PartitioningApp: with open(os.path.join(folder, 'global.csv'), 'a') as fp: fp.write('{0}, {1}, {2}, {3}\n'.format(n, self.partitioner.n_components, self.partitioner.n_tags, new_kmers)) + cov_filename = os.path.join(folder, '{0}.coverage.csv'.format(n)) + self.partitioner.write_component_coverage(cov_filename) def prep_results_dir(self): try: @@ -140,3 +142,9 @@ cdef class PartitioningApp: self.write_meta(n_sequences, total_kmers) return self.partitioner + + +cdef class DynamicPartitioning(PartitioningApp): + + def run(self): + pass diff --git a/khmer/_oxli/partitioning.pyx b/khmer/_oxli/partitioning.pyx index 5cd6d6bf49..445c86dcae 100644 --- a/khmer/_oxli/partitioning.pyx +++ b/khmer/_oxli/partitioning.pyx @@ -201,12 +201,37 @@ cdef class StreamingPartitioner: if lockedptr: for cmpptr in deref(lockedptr): + if cmpptr == NULL: + continue fprintf(fp, "%llu,%llu,%f\n", deref(cmpptr).component_id, deref(cmpptr).get_n_tags(), Component._mean_tag_count(cmpptr, self._graph_ptr)) fclose(fp) + def write_component_coverage(self, filename): + cdef FILE* fp + fp = fopen(filename.encode('utf-8'), 'wb') + if fp == NULL: + raise IOError('Can\'t open file.') + + cdef ComponentPtr cmpptr + cdef shared_ptr[ComponentPtrVector] lockedptr + cdef size_t i + lockedptr = self._components.lock() + + if lockedptr: + for cmpptr in deref(lockedptr): + if cmpptr == NULL: + continue + deref(cmpptr).update_coverage(self._graph_ptr) + fprintf(fp, "%llu", + deref(cmpptr).component_id) + for i in range(16): + fprintf(fp, ",%llu", deref(cmpptr).coverage.bins[i]) + fprintf(fp, "\n") + fclose(fp) + def save(self, filename): graph_filename = '{0}.graph'.format(filename) comp_filename = '{0}.json'.format(filename) diff --git a/khmer/_oxli/wrapper.pxd b/khmer/_oxli/wrapper.pxd index 776ba2e7bf..e43fcc91d5 100644 --- a/khmer/_oxli/wrapper.pxd +++ b/khmer/_oxli/wrapper.pxd @@ -421,6 +421,17 @@ cdef extern from "oxli/hllcounter.hh" namespace "oxli": void set_erate(double) +cdef extern from "oxli/hist.hh" namespace "oxli": + + cdef cppclass CpHistogram "oxli::Histogram<16>": + uint64_t[16] bins + + CpHistogram() + + void add(uint64_t) + void clear() + + cdef extern from "oxli/partitioning.hh" namespace "oxli": ctypedef vector[HashIntoType] TagVector @@ -429,6 +440,7 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": CpComponent() CpComponent(uint64_t) + CpHistogram coverage const uint64_t component_id vector[HashIntoType] tags diff --git a/src/oxli/Makefile b/src/oxli/Makefile index 5c1ddf389a..fd95ec146f 100644 --- a/src/oxli/Makefile +++ b/src/oxli/Makefile @@ -236,9 +236,7 @@ LIBOXLI_OBJS= \ alphabets.o \ murmur3.o \ storage.o \ - partitioning.o \ - gmap.o \ - hist.o + partitioning.o PRECOMILE_OBJS ?= PRECLEAN_TARGS ?= From 1378244c396cae791947a4bbf5ec5ad908b50877 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 6 Sep 2017 15:36:40 -0700 Subject: [PATCH 100/185] Move hashing functions to Cython --- include/khmer/_cpy_khmer.hh | 12 --- khmer/__init__.py | 36 ++++---- khmer/_oxli/hashing.pxd | 21 ++++- khmer/_oxli/hashing.pyx | 48 +++++++++++ src/khmer/_cpy_khmer.cc | 168 +----------------------------------- 5 files changed, 88 insertions(+), 197 deletions(-) diff --git a/include/khmer/_cpy_khmer.hh b/include/khmer/_cpy_khmer.hh index a9c9e8b82c..874675ced3 100644 --- a/include/khmer/_cpy_khmer.hh +++ b/include/khmer/_cpy_khmer.hh @@ -77,18 +77,6 @@ Contact: khmer-project@idyll.org namespace khmer { -PyObject * forward_hash(PyObject * self, PyObject * args); - -PyObject * forward_hash_no_rc(PyObject * self, PyObject * args); - -PyObject * reverse_hash(PyObject * self, PyObject * args); - -PyObject * murmur3_forward_hash(PyObject * self, PyObject * args); - -PyObject * murmur3_forward_hash_no_rc(PyObject * self, PyObject * args); - -PyObject * reverse_complement(PyObject * self, PyObject * args); - PyObject * get_version_cpp( PyObject * self, PyObject * args ); extern PyMethodDef KhmerMethods[]; diff --git a/khmer/__init__.py b/khmer/__init__.py index 22d8470f20..76fc9d12b7 100644 --- a/khmer/__init__.py +++ b/khmer/__init__.py @@ -43,18 +43,6 @@ from khmer._khmer import Read -from khmer._khmer import forward_hash -# tests/test_{functions,countgraph,counting_single}.py - -from khmer._khmer import forward_hash_no_rc # tests/test_functions.py - -from khmer._khmer import reverse_hash # tests/test_functions.py -# tests/counting_single.py - -from khmer._khmer import hash_murmur3 # tests/test_functions.py -from khmer._khmer import hash_no_rc_murmur3 # tests/test_functions.py - -from khmer._khmer import reverse_complement from khmer._khmer import get_version_cpp as __version_cpp__ # tests/test_version.py @@ -65,17 +53,33 @@ from khmer._khmer import FILETYPES +from khmer._oxli.assembly import (LinearAssembler, SimpleLabeledAssembler, + JunctionCountAssembler) + from khmer._oxli.graphs import (Counttable, QFCounttable, Nodetable, SmallCounttable, Countgraph, SmallCountgraph, Nodegraph) + +from khmer._oxli.hashing import (forward_hash, forward_hash_no_rc, + reverse_hash, hash_murmur3, + hash_no_rc_murmur3, + reverse_complement) + +from khmer._oxli.hashset import HashSet + +from khmer._oxli.hllcounter import HLLCounter + from khmer._oxli.labeling import GraphLabels + from khmer._oxli.legacy_partitioning import SubsetPartition, PrePartitionInfo + from khmer._oxli.parsing import FastxParser + from khmer._oxli.readaligner import ReadAligner from khmer._oxli.utils import get_n_primes_near_x, is_prime -import sys +import sys from struct import pack, unpack from ._version import get_versions @@ -214,9 +218,3 @@ def calc_expected_collisions(graph, force=False, max_false_pos=.2): return fp_all - -from khmer._oxli.assembly import (LinearAssembler, SimpleLabeledAssembler, - JunctionCountAssembler) -from khmer._oxli.hashset import HashSet -from khmer._oxli.hllcounter import HLLCounter -from khmer._oxli.labeling import GraphLabels diff --git a/khmer/_oxli/hashing.pxd b/khmer/_oxli/hashing.pxd index e0bd6bcf16..6f90aa3b07 100644 --- a/khmer/_oxli/hashing.pxd +++ b/khmer/_oxli/hashing.pxd @@ -50,7 +50,8 @@ cdef extern from "oxli/kmer_hash.hh" namespace "oxli": HashIntoType _hash_murmur(const string&, const WordLength) HashIntoType _hash_murmur(const string&, HashIntoType&, HashIntoType&) - HashIntoType _hash_murmur_forward(const string&) + HashIntoType _hash_murmur_forward(const string&, + const WordLength) cdef extern from "oxli/oxli.hh" namespace "oxli": @@ -65,3 +66,21 @@ cdef class Kmer: @staticmethod cdef Kmer wrap(CpKmer * cpkmer, WordLength K) + + +cpdef HashIntoType forward_hash(str kmer, unsigned int K) + + +cpdef HashIntoType forward_hash_no_rc(str kmer, WordLength K) + + +cpdef str reverse_hash(object h, int K) + + +cpdef str reverse_complement(str sequence) + + +cpdef hash_murmur3(str s) + + +cpdef hash_no_rc_murmur3(str s) diff --git a/khmer/_oxli/hashing.pyx b/khmer/_oxli/hashing.pyx index 0035eca73c..cf947fb860 100644 --- a/khmer/_oxli/hashing.pyx +++ b/khmer/_oxli/hashing.pyx @@ -6,6 +6,8 @@ from libc.stdint cimport uint64_t from cython.operator cimport dereference as deref from khmer._oxli.oxli_types cimport * +from khmer._oxli.utils cimport _bstring, _ustring + cdef class Kmer: @@ -63,3 +65,49 @@ cdef class Kmer: deref(kmer._this).set_from_unique_hash(tag, K) kmer.kmer = _revhash(kmer.kmer_u, K) return kmer + + +cpdef HashIntoType forward_hash(str kmer, unsigned int K): + '''Run the 2-bit hash algorithm on the given K-mer.''' + + if K > 32: + raise ValueError("k-mer size must be <= 32") + if len(kmer) != K: + raise ValueError("k-mer length must equal K") + + return _hash(_bstring(kmer), K) + + +cpdef HashIntoType forward_hash_no_rc(str kmer, WordLength K): + '''Run the 2-bit hash function in only the given + sequence orientation.''' + + if K > 32: + raise ValueError("k-mer size must be <= 32") + if len(kmer) != K: + raise ValueError("k-mer length must equal K") + + return _hash_forward(_bstring(kmer), K) + + +cpdef str reverse_hash(object h, int K): + if K > 32: + raise ValueError("k-mer size must be <= 32") + + cdef HashIntoType _h = h + return _revhash(_h, K) + + +cpdef str reverse_complement(str sequence): + cdef string s = _revcomp(_bstring(sequence)) + return s + + +cpdef hash_murmur3(str s): + cdef HashIntoType h = _hash_murmur(_bstring(s), len(s)) + return h + + +cpdef hash_no_rc_murmur3(str s): + cdef HashIntoType h = _hash_murmur_forward(_bstring(s), len(s)) + return h diff --git a/src/khmer/_cpy_khmer.cc b/src/khmer/_cpy_khmer.cc index d1a70a0e21..2f19806851 100644 --- a/src/khmer/_cpy_khmer.cc +++ b/src/khmer/_cpy_khmer.cc @@ -59,136 +59,6 @@ extern "C" { } namespace khmer { - -PyObject * forward_hash(PyObject * self, PyObject * args) -{ - const char * kmer; - WordLength ksize; - - if (!PyArg_ParseTuple(args, "sb", &kmer, &ksize)) { - return NULL; - } - - if (ksize > KSIZE_MAX) { - PyErr_Format(PyExc_ValueError, "k-mer size must be <= %u", KSIZE_MAX); - return NULL; - } - - if (strlen(kmer) != ksize) { - PyErr_Format(PyExc_ValueError, "k-mer size different from ksize"); - return NULL; - } - - try { - PyObject * hash = nullptr; - const HashIntoType h(_hash(kmer, ksize)); - convert_HashIntoType_to_PyObject(h, &hash); - return hash; - } catch (oxli_exception &e) { - PyErr_SetString(PyExc_ValueError, e.what()); - return NULL; - } -} - -PyObject * forward_hash_no_rc(PyObject * self, PyObject * args) -{ - const char * kmer; - WordLength ksize; - - if (!PyArg_ParseTuple(args, "sb", &kmer, &ksize)) { - return NULL; - } - - if (ksize > KSIZE_MAX) { - PyErr_Format(PyExc_ValueError, "k-mer size must be <= %u", KSIZE_MAX); - return NULL; - } - - if (strlen(kmer) != ksize) { - PyErr_SetString(PyExc_ValueError, - "k-mer length must equal the k-size"); - return NULL; - } - - PyObject * hash = nullptr; - const HashIntoType h(_hash_forward(kmer, ksize)); - convert_HashIntoType_to_PyObject(h, &hash); - return hash; -} - -PyObject * reverse_hash(PyObject * self, PyObject * args) -{ - PyObject * val; - HashIntoType hash; - WordLength ksize; - - if (!PyArg_ParseTuple(args, "Ob", &val, &ksize)) { - return NULL; - } - - if (PyLong_Check(val) || PyInt_Check(val)) { - if (!convert_PyLong_to_HashIntoType(val, hash)) { - return NULL; - } - } else { - PyErr_SetString(PyExc_TypeError, - "Hash value must be an integer."); - return NULL; - } - - if (ksize > KSIZE_MAX) { - PyErr_Format(PyExc_ValueError, "k-mer size must be <= %u", KSIZE_MAX); - return NULL; - } - - return PyUnicode_FromString(_revhash(hash, ksize).c_str()); -} - -PyObject * murmur3_forward_hash(PyObject * self, PyObject * args) -{ - const char * kmer; - - if (!PyArg_ParseTuple(args, "s", &kmer)) { - return NULL; - } - - PyObject * hash = nullptr; - const HashIntoType h(_hash_murmur(kmer, strlen(kmer))); - convert_HashIntoType_to_PyObject(h, &hash); - return hash; -} - -PyObject * murmur3_forward_hash_no_rc(PyObject * self, PyObject * args) -{ - const char * kmer; - - if (!PyArg_ParseTuple(args, "s", &kmer)) { - return NULL; - } - - PyObject * hash = nullptr; - const HashIntoType h(_hash_murmur_forward(kmer, strlen(kmer))); - convert_HashIntoType_to_PyObject(h, &hash); - return hash; -} - -PyObject * reverse_complement(PyObject * self, PyObject * args) -{ - const char * sequence; - if (!PyArg_ParseTuple(args, "s", &sequence)) { - return NULL; - } - - std::string s(sequence); - try { - s = _revcomp(s); - } catch (oxli_exception &e) { - PyErr_SetString(PyExc_RuntimeError, e.what()); - return NULL; - } - return PyUnicode_FromString(s.c_str()); -} - // // technique for resolving literal below found here: // https://gcc.gnu.org/onlinedocs/gcc-4.9.1/cpp/Stringification.html @@ -205,47 +75,15 @@ get_version_cpp( PyObject * self, PyObject * args ) PyMethodDef KhmerMethods[] = { { - "forward_hash", forward_hash, - METH_VARARGS, "", - }, - { - "forward_hash_no_rc", forward_hash_no_rc, - METH_VARARGS, "", - }, - { - "reverse_hash", reverse_hash, - METH_VARARGS, "", - }, - { - "hash_murmur3", - murmur3_forward_hash, - METH_VARARGS, - "Calculate the hash value of a k-mer using MurmurHash3 " - "(with reverse complement)", - }, - { - "hash_no_rc_murmur3", - murmur3_forward_hash_no_rc, - METH_VARARGS, - "Calculate the hash value of a k-mer using MurmurHash3 " - "(no reverse complement)", - }, - { - "reverse_complement", - reverse_complement, - METH_VARARGS, - "Calculate the reverse-complement of the DNA sequence " - "with alphabet ACGT", - }, - { - "get_version_cpp", get_version_cpp, - METH_VARARGS, "return the VERSION c++ compiler option" + "get_version_cpp", get_version_cpp, METH_VARARGS, + "return the VERSION c++ compiler option" }, { NULL, NULL, 0, NULL } // sentinel }; } // namespace khmer + // // Module machinery. // From dd2e8e47b1aeb6872535d326a0c62fb20a56fb4d Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 6 Sep 2017 16:15:04 -0700 Subject: [PATCH 101/185] Convert get_version_cpp to Cython --- include/khmer/_cpy_khmer.hh | 2 -- include/oxli/oxli.hh | 2 ++ khmer/__init__.py | 3 +-- khmer/_oxli/utils.pxd | 6 ++++++ khmer/_oxli/utils.pyx | 3 ++- setup.py | 2 +- src/khmer/_cpy_khmer.cc | 12 ------------ src/oxli/oxli.cc | 13 +++++++++++++ 8 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 src/oxli/oxli.cc diff --git a/include/khmer/_cpy_khmer.hh b/include/khmer/_cpy_khmer.hh index 874675ced3..4ff7a2d2e9 100644 --- a/include/khmer/_cpy_khmer.hh +++ b/include/khmer/_cpy_khmer.hh @@ -77,8 +77,6 @@ Contact: khmer-project@idyll.org namespace khmer { -PyObject * get_version_cpp( PyObject * self, PyObject * args ); - extern PyMethodDef KhmerMethods[]; } diff --git a/include/oxli/oxli.hh b/include/oxli/oxli.hh index 1d3a074f9c..67bfd38eca 100644 --- a/include/oxli/oxli.hh +++ b/include/oxli/oxli.hh @@ -107,6 +107,8 @@ private:\ namespace oxli { +extern std::string get_version_cpp(); + // largest number we can count up to, exactly. (8 bytes) typedef unsigned long long int ExactCounterType; diff --git a/khmer/__init__.py b/khmer/__init__.py index 76fc9d12b7..1d5082cfb9 100644 --- a/khmer/__init__.py +++ b/khmer/__init__.py @@ -43,8 +43,6 @@ from khmer._khmer import Read - -from khmer._khmer import get_version_cpp as __version_cpp__ # tests/test_version.py from khmer._khmer import ReadParser # sandbox/to-casava-1.8-fastq.py @@ -78,6 +76,7 @@ from khmer._oxli.readaligner import ReadAligner from khmer._oxli.utils import get_n_primes_near_x, is_prime +from khmer._oxli.utils import get_version_cpp as __version_cpp__ import sys from struct import pack, unpack diff --git a/khmer/_oxli/utils.pxd b/khmer/_oxli/utils.pxd index ae487c38cd..8cc4781ca9 100644 --- a/khmer/_oxli/utils.pxd +++ b/khmer/_oxli/utils.pxd @@ -1,4 +1,5 @@ # -*- coding: UTF-8 -*- +from libcpp.string cimport string from libcpp.vector cimport vector from libc.stdint cimport uint32_t, uint64_t from libcpp cimport bool @@ -12,6 +13,9 @@ cdef extern from "oxli/hashtable.hh" namespace "oxli": cdef bool _is_prime "oxli::is_prime" (uint64_t n) cdef vector[uint64_t] _get_n_primes_near_x "oxli::get_n_primes_near_x" (uint32_t, uint64_t) +cdef extern from "oxli/oxli.hh" namespace "oxli": + cdef string _get_version_cpp "oxli::get_version_cpp" () + cdef bytes _bstring(s) cdef unicode _ustring(s) @@ -21,3 +25,5 @@ cpdef bool is_num(object n) cdef void _flatten_fill(double * fill_to, object fill_from) cdef void _fill(double * fill_to, object fill_from) + +cpdef str get_version_cpp() diff --git a/khmer/_oxli/utils.pyx b/khmer/_oxli/utils.pyx index c225a1a490..d90ed5cc20 100644 --- a/khmer/_oxli/utils.pyx +++ b/khmer/_oxli/utils.pyx @@ -60,4 +60,5 @@ cdef void _fill(double * fill_to, object fill_from): for idx, item in enumerate(fill_from): fill_to[idx] = item - +cpdef str get_version_cpp(): + return _get_version_cpp() diff --git a/setup.py b/setup.py index ca4ecbb181..b6984e9e4f 100755 --- a/setup.py +++ b/setup.py @@ -165,7 +165,7 @@ def build_dir(): ]] SOURCES.extend(path_join("src", "oxli", bn + ".cc") for bn in [ "read_parsers", "kmer_hash", "hashtable", "hashgraph", - "labelhash", "subset", "read_aligner", + "labelhash", "subset", "read_aligner", "oxli", "hllcounter", "traversal", "kmer_filters", "assembler", "alphabets", "storage"]) diff --git a/src/khmer/_cpy_khmer.cc b/src/khmer/_cpy_khmer.cc index 2f19806851..58896366da 100644 --- a/src/khmer/_cpy_khmer.cc +++ b/src/khmer/_cpy_khmer.cc @@ -64,20 +64,8 @@ namespace khmer { // https://gcc.gnu.org/onlinedocs/gcc-4.9.1/cpp/Stringification.html // -PyObject * -get_version_cpp( PyObject * self, PyObject * args ) -{ -#define xstr(s) str(s) -#define str(s) #s - std::string dVersion = xstr(VERSION); - return PyUnicode_FromString(dVersion.c_str()); -} PyMethodDef KhmerMethods[] = { - { - "get_version_cpp", get_version_cpp, METH_VARARGS, - "return the VERSION c++ compiler option" - }, { NULL, NULL, 0, NULL } // sentinel }; diff --git a/src/oxli/oxli.cc b/src/oxli/oxli.cc new file mode 100644 index 0000000000..6f643213e2 --- /dev/null +++ b/src/oxli/oxli.cc @@ -0,0 +1,13 @@ +#include + +namespace oxli { + +std::string get_version_cpp() +{ +#define _macro_xstr(s) _macro_str(s) +#define _macro_str(s) #s + std::string dVersion = _macro_xstr(VERSION); + return dVersion; +} + +} From 17c6bab4f6ce105ff42a8e3ea43c0ed82ae16032 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 6 Sep 2017 16:15:19 -0700 Subject: [PATCH 102/185] Allow hash functions to accept string derivitives --- khmer/_oxli/hashing.pxd | 10 +++++----- khmer/_oxli/hashing.pyx | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/khmer/_oxli/hashing.pxd b/khmer/_oxli/hashing.pxd index 6f90aa3b07..ae052e060d 100644 --- a/khmer/_oxli/hashing.pxd +++ b/khmer/_oxli/hashing.pxd @@ -68,19 +68,19 @@ cdef class Kmer: cdef Kmer wrap(CpKmer * cpkmer, WordLength K) -cpdef HashIntoType forward_hash(str kmer, unsigned int K) +cpdef HashIntoType forward_hash(object kmer, unsigned int K) -cpdef HashIntoType forward_hash_no_rc(str kmer, WordLength K) +cpdef HashIntoType forward_hash_no_rc(object kmer, WordLength K) cpdef str reverse_hash(object h, int K) -cpdef str reverse_complement(str sequence) +cpdef str reverse_complement(object sequence) -cpdef hash_murmur3(str s) +cpdef hash_murmur3(object s) -cpdef hash_no_rc_murmur3(str s) +cpdef hash_no_rc_murmur3(object s) diff --git a/khmer/_oxli/hashing.pyx b/khmer/_oxli/hashing.pyx index cf947fb860..265b1ef789 100644 --- a/khmer/_oxli/hashing.pyx +++ b/khmer/_oxli/hashing.pyx @@ -67,7 +67,7 @@ cdef class Kmer: return kmer -cpdef HashIntoType forward_hash(str kmer, unsigned int K): +cpdef HashIntoType forward_hash(object kmer, unsigned int K): '''Run the 2-bit hash algorithm on the given K-mer.''' if K > 32: @@ -78,7 +78,7 @@ cpdef HashIntoType forward_hash(str kmer, unsigned int K): return _hash(_bstring(kmer), K) -cpdef HashIntoType forward_hash_no_rc(str kmer, WordLength K): +cpdef HashIntoType forward_hash_no_rc(object kmer, WordLength K): '''Run the 2-bit hash function in only the given sequence orientation.''' @@ -98,16 +98,16 @@ cpdef str reverse_hash(object h, int K): return _revhash(_h, K) -cpdef str reverse_complement(str sequence): +cpdef str reverse_complement(object sequence): cdef string s = _revcomp(_bstring(sequence)) return s -cpdef hash_murmur3(str s): +cpdef hash_murmur3(object s): cdef HashIntoType h = _hash_murmur(_bstring(s), len(s)) return h -cpdef hash_no_rc_murmur3(str s): +cpdef hash_no_rc_murmur3(object s): cdef HashIntoType h = _hash_murmur_forward(_bstring(s), len(s)) return h From 533c57c01ea0a69fdb6fa02bdae9863cda6c92db Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 6 Sep 2017 16:30:02 -0700 Subject: [PATCH 103/185] cythonize FILETYPES dict --- khmer/__init__.py | 6 +----- khmer/_oxli/utils.pxd | 13 ++++++++++++- khmer/_oxli/utils.pyx | 12 ++++++++++++ src/khmer/_cpy_khmer.cc | 11 ----------- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/khmer/__init__.py b/khmer/__init__.py index 1d5082cfb9..7d4a500221 100644 --- a/khmer/__init__.py +++ b/khmer/__init__.py @@ -35,8 +35,6 @@ # pylint: disable=too-few-public-methods,no-init,missing-docstring """This is khmer; please see http://khmer.readthedocs.io/.""" - -from __future__ import print_function from collections import namedtuple from math import log import json @@ -49,8 +47,6 @@ # tests/test_read_parsers.py,scripts/{filter-abund-single,load-graph}.py # scripts/{abundance-dist-single,load-into-counting}.py -from khmer._khmer import FILETYPES - from khmer._oxli.assembly import (LinearAssembler, SimpleLabeledAssembler, JunctionCountAssembler) @@ -75,7 +71,7 @@ from khmer._oxli.readaligner import ReadAligner -from khmer._oxli.utils import get_n_primes_near_x, is_prime +from khmer._oxli.utils import get_n_primes_near_x, is_prime, FILETYPES from khmer._oxli.utils import get_version_cpp as __version_cpp__ import sys diff --git a/khmer/_oxli/utils.pxd b/khmer/_oxli/utils.pxd index 8cc4781ca9..63321e2e92 100644 --- a/khmer/_oxli/utils.pxd +++ b/khmer/_oxli/utils.pxd @@ -13,8 +13,19 @@ cdef extern from "oxli/hashtable.hh" namespace "oxli": cdef bool _is_prime "oxli::is_prime" (uint64_t n) cdef vector[uint64_t] _get_n_primes_near_x "oxli::get_n_primes_near_x" (uint32_t, uint64_t) -cdef extern from "oxli/oxli.hh" namespace "oxli": +cdef extern from "oxli/oxli.hh": cdef string _get_version_cpp "oxli::get_version_cpp" () + cdef const char * SAVED_SIGNATURE + cdef int SAVED_FORMAT_VERSION + cdef int SAVED_COUNTING_HT + cdef int SAVED_HASHBITS + cdef int SAVED_TAGS + cdef int SAVED_STOPTAGS + cdef int SAVED_SUBSET + cdef int SAVED_LABELSET + cdef int SAVED_SMALLCOUNT + cdef int SAVED_QFCOUNT + cdef bytes _bstring(s) diff --git a/khmer/_oxli/utils.pyx b/khmer/_oxli/utils.pyx index d90ed5cc20..664fc4327d 100644 --- a/khmer/_oxli/utils.pyx +++ b/khmer/_oxli/utils.pyx @@ -6,6 +6,18 @@ from cpython.version cimport PY_MAJOR_VERSION from cython import short, int, long +FILETYPES = \ +{ + "COUNTING_HT": SAVED_COUNTING_HT, + "HASHBITS": SAVED_HASHBITS, + "TAGS": SAVED_TAGS, + "STOPTAGS": SAVED_STOPTAGS, + "SUBSET": SAVED_SUBSET, + "LABELSET": SAVED_LABELSET, + "SMALLCOUNT": SAVED_SMALLCOUNT +} + + def is_prime(n): return _is_prime(n) diff --git a/src/khmer/_cpy_khmer.cc b/src/khmer/_cpy_khmer.cc index 58896366da..736e19e439 100644 --- a/src/khmer/_cpy_khmer.cc +++ b/src/khmer/_cpy_khmer.cc @@ -106,17 +106,6 @@ MOD_INIT(_khmer) return MOD_ERROR_VAL; } - PyObject * filetype_dict = Py_BuildValue("{s,i,s,i,s,i,s,i,s,i,s,i,s,i}", - "COUNTING_HT", SAVED_COUNTING_HT, - "HASHBITS", SAVED_HASHBITS, - "TAGS", SAVED_TAGS, - "STOPTAGS", SAVED_STOPTAGS, - "SUBSET", SAVED_SUBSET, - "LABELSET", SAVED_LABELSET, - "SMALLCOUNT", SAVED_SMALLCOUNT); - if (PyModule_AddObject( m, "FILETYPES", filetype_dict ) < 0) { - return MOD_ERROR_VAL; - } Py_INCREF(&khmer_Read_Type); if (PyModule_AddObject( m, "Read", From 80ba62c6e20ee618a9b8f66cb04c8f052da09f9f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 6 Sep 2017 17:03:16 -0700 Subject: [PATCH 104/185] move extraction functions to graph classes --- khmer/__init__.py | 101 +--------------------------------- khmer/_oxli/graphs.pyx | 116 ++++++++++++++++++++++++++++++++++++++- khmer/khmer_args.py | 5 +- tests/test_countgraph.py | 32 +++++++++++ tests/test_functions.py | 62 --------------------- tests/test_nodegraph.py | 34 +++++++++++- 6 files changed, 183 insertions(+), 167 deletions(-) diff --git a/khmer/__init__.py b/khmer/__init__.py index 7d4a500221..fe4f2b5db0 100644 --- a/khmer/__init__.py +++ b/khmer/__init__.py @@ -35,7 +35,7 @@ # pylint: disable=too-few-public-methods,no-init,missing-docstring """This is khmer; please see http://khmer.readthedocs.io/.""" -from collections import namedtuple + from math import log import json @@ -52,7 +52,7 @@ from khmer._oxli.graphs import (Counttable, QFCounttable, Nodetable, SmallCounttable, Countgraph, SmallCountgraph, - Nodegraph) + Nodegraph, _buckets_per_byte) from khmer._oxli.hashing import (forward_hash, forward_hash_no_rc, reverse_hash, hash_murmur3, @@ -75,108 +75,13 @@ from khmer._oxli.utils import get_version_cpp as __version_cpp__ import sys -from struct import pack, unpack + from ._version import get_versions __version__ = get_versions()['version'] del get_versions -_buckets_per_byte = { - # calculated by hand from settings in third-part/cqf/gqf.h - 'qfcounttable': 1 / 1.26, - 'countgraph': 1, - 'smallcountgraph': 2, - 'nodegraph': 8, -} - - -def extract_nodegraph_info(filename): - """Open the given nodegraph file and return a tuple of information. - - Returns: the k-mer size, the table size, the number of tables, the version - of the table format, and the type of table flag. - - Keyword argument: - filename -- the name of the nodegraph file to inspect - """ - ksize = None - n_tables = None - table_size = None - signature = None - version = None - ht_type = None - occupied = None - - uint_size = len(pack('I', 0)) - uchar_size = len(pack('B', 0)) - ulonglong_size = len(pack('Q', 0)) - - try: - with open(filename, 'rb') as nodegraph: - signature, = unpack('4s', nodegraph.read(4)) - version, = unpack('B', nodegraph.read(1)) - ht_type, = unpack('B', nodegraph.read(1)) - ksize, = unpack('I', nodegraph.read(uint_size)) - n_tables, = unpack('B', nodegraph.read(uchar_size)) - occupied, = unpack('Q', nodegraph.read(ulonglong_size)) - table_size, = unpack('Q', nodegraph.read(ulonglong_size)) - if signature != b"OXLI": - raise ValueError("Node graph '{}' is missing file type " - "signature".format(filename) + str(signature)) - except: - raise ValueError("Node graph '{}' is corrupt ".format(filename)) - - return ksize, round(table_size, -2), n_tables, version, ht_type, occupied - - -def extract_countgraph_info(filename): - """Open the given countgraph file and return a tuple of information. - - Return: the k-mer size, the table size, the number of tables, the bigcount - flag, the version of the table format, and the type of table flag. - - Keyword argument: - filename -- the name of the countgraph file to inspect - """ - CgInfo = namedtuple("CgInfo", ['ksize', 'n_tables', 'table_size', - 'use_bigcount', 'version', 'ht_type', - 'n_occupied']) - ksize = None - n_tables = None - table_size = None - signature = None - version = None - ht_type = None - use_bigcount = None - occupied = None - - uint_size = len(pack('I', 0)) - ulonglong_size = len(pack('Q', 0)) - - try: - with open(filename, 'rb') as countgraph: - signature, = unpack('4s', countgraph.read(4)) - version, = unpack('B', countgraph.read(1)) - ht_type, = unpack('B', countgraph.read(1)) - if ht_type != FILETYPES['SMALLCOUNT']: - use_bigcount, = unpack('B', countgraph.read(1)) - else: - use_bigcount = None - ksize, = unpack('I', countgraph.read(uint_size)) - n_tables, = unpack('B', countgraph.read(1)) - occupied, = unpack('Q', countgraph.read(ulonglong_size)) - table_size, = unpack('Q', countgraph.read(ulonglong_size)) - if signature != b'OXLI': - raise ValueError("Count graph file '{}' is missing file type " - "signature. ".format(filename) + str(signature)) - except: - raise ValueError("Count graph file '{}' is corrupt ".format(filename)) - - return CgInfo(ksize, n_tables, round(table_size, -2), use_bigcount, - version, ht_type, occupied) - - def calc_expected_collisions(graph, force=False, max_false_pos=.2): """Do a quick & dirty expected collision rate calculation on a graph. diff --git a/khmer/_oxli/graphs.pyx b/khmer/_oxli/graphs.pyx index 5a1f19143c..7eb084d132 100644 --- a/khmer/_oxli/graphs.pyx +++ b/khmer/_oxli/graphs.pyx @@ -1,4 +1,6 @@ from math import log +from struct import pack, unpack +from collections import namedtuple from cython.operator cimport dereference as deref from cpython.buffer cimport (PyBuffer_FillInfo, PyBUF_FULL_RO) @@ -11,7 +13,7 @@ from libcpp.set cimport set from libcpp.string cimport string from khmer._oxli.utils cimport _bstring, is_str, is_num -from khmer._oxli.utils import get_n_primes_near_x +from khmer._oxli.utils import get_n_primes_near_x, FILETYPES from khmer._oxli.parsing cimport (CpFastxReader, CPyReadParser_Object, get_parser, CpReadParser, FastxParserPtr) from khmer._oxli.hashset cimport HashSet @@ -25,6 +27,13 @@ from khmer._khmer import ReadParser CYTHON_TABLES = (Hashtable, Nodetable, Counttable, SmallCounttable, QFCounttable, Nodegraph, Countgraph, SmallCountgraph) +_buckets_per_byte = { + # calculated by hand from settings in third-part/cqf/gqf.h + 'qfcounttable': 1 / 1.26, + 'countgraph': 1, + 'smallcountgraph': 2, + 'nodegraph': 8, +} cdef class Hashtable: @@ -400,6 +409,53 @@ cdef class Counttable(Hashtable): self._ct_this = make_shared[CpCounttable](k, primes) self._ht_this = self._ct_this + @staticmethod + def extract_info(filename): + """Open the given countgraph file and return a tuple of information. + + Return: the k-mer size, the table size, the number of tables, the bigcount + flag, the version of the table format, and the type of table flag. + + Keyword argument: + filename -- the name of the countgraph file to inspect + """ + CgInfo = namedtuple("CgInfo", ['ksize', 'n_tables', 'table_size', + 'use_bigcount', 'version', 'ht_type', + 'n_occupied']) + ksize = None + n_tables = None + table_size = None + signature = None + version = None + ht_type = None + use_bigcount = None + occupied = None + + uint_size = len(pack('I', 0)) + ulonglong_size = len(pack('Q', 0)) + + try: + with open(filename, 'rb') as countgraph: + signature, = unpack('4s', countgraph.read(4)) + version, = unpack('B', countgraph.read(1)) + ht_type, = unpack('B', countgraph.read(1)) + if ht_type != FILETYPES['SMALLCOUNT']: + use_bigcount, = unpack('B', countgraph.read(1)) + else: + use_bigcount = None + ksize, = unpack('I', countgraph.read(uint_size)) + n_tables, = unpack('B', countgraph.read(1)) + occupied, = unpack('Q', countgraph.read(ulonglong_size)) + table_size, = unpack('Q', countgraph.read(ulonglong_size)) + if signature != b'OXLI': + raise ValueError("Count graph file '{}' is missing file type " + "signature. ".format(filename) + str(signature)) + except: + raise ValueError("Count graph file '{}' is corrupt ".format(filename)) + + return CgInfo(ksize, n_tables, round(table_size, -2), use_bigcount, + version, ht_type, occupied) + cdef class SmallCounttable(Hashtable): @@ -417,6 +473,10 @@ cdef class SmallCounttable(Hashtable): sizes[i] = (sizes[i] // 2) + 1 return self._get_raw_tables(table_ptrs, sizes) + @staticmethod + def extract_info(filename): + return Counttable.extract_info(filename) + cdef class Nodetable(Hashtable): @@ -427,6 +487,47 @@ cdef class Nodetable(Hashtable): self._nt_this = make_shared[CpNodetable](k, primes) self._ht_this = self._nt_this + @staticmethod + def extract_info(filename): + """Open the given nodegraph file and return a tuple of information. + + Returns: the k-mer size, the table size, the number of tables, the version + of the table format, and the type of table flag. + + Keyword argument: + filename -- the name of the nodegraph file to inspect + """ + ksize = None + n_tables = None + table_size = None + signature = None + version = None + ht_type = None + occupied = None + + uint_size = len(pack('I', 0)) + uchar_size = len(pack('B', 0)) + ulonglong_size = len(pack('Q', 0)) + + try: + with open(filename, 'rb') as nodegraph: + signature, = unpack('4s', nodegraph.read(4)) + version, = unpack('B', nodegraph.read(1)) + ht_type, = unpack('B', nodegraph.read(1)) + ksize, = unpack('I', nodegraph.read(uint_size)) + n_tables, = unpack('B', nodegraph.read(uchar_size)) + occupied, = unpack('Q', nodegraph.read(ulonglong_size)) + table_size, = unpack('Q', nodegraph.read(ulonglong_size)) + if signature != b"OXLI": + raise ValueError("Node graph '{}' is missing file type " + "signature".format(filename) + str(signature)) + except: + raise ValueError("Node graph '{}' is corrupt ".format(filename)) + + return ksize, round(table_size, -2), n_tables, version, ht_type, occupied + + + cdef class Hashgraph(Hashtable): @@ -830,6 +931,12 @@ cdef class Countgraph(Hashgraph): return subset + @staticmethod + def extract_info(filename): + return Counttable.extract_info(filename) + + + cdef class SmallCountgraph(Hashgraph): @@ -852,6 +959,9 @@ cdef class SmallCountgraph(Hashgraph): sizes[i] = sizes[i] // 2 + 1 return self._get_raw_tables(table_ptrs, sizes) + @staticmethod + def extract_info(filename): + return Counttable.extract_info(filename) cdef class Nodegraph(Hashgraph): @@ -870,3 +980,7 @@ cdef class Nodegraph(Hashgraph): def update(self, Nodegraph other): deref(self._ng_this).update_from(deref(other._ng_this)) + + @staticmethod + def extract_info(filename): + return Nodetable.extract_info(filename) diff --git a/khmer/khmer_args.py b/khmer/khmer_args.py index ea438ffc40..f47218c63c 100644 --- a/khmer/khmer_args.py +++ b/khmer/khmer_args.py @@ -51,8 +51,7 @@ import screed import khmer -from khmer import extract_countgraph_info -from khmer import __version__ +from khmer import __version__, Countgraph from .utils import print_error from .khmer_logger import log_info, log_warn, configure_logging @@ -262,7 +261,7 @@ def check_conflicting_args(args, hashtype): infoset = None if hashtype in ('countgraph', 'smallcountgraph'): - infoset = extract_countgraph_info(args.loadgraph) + infoset = Countgraph.extract_info(args.loadgraph) if infoset is not None: ksize = infoset.ksize max_tablesize = infoset.table_size diff --git a/tests/test_countgraph.py b/tests/test_countgraph.py index 2c8409ca5a..16e25f61e7 100644 --- a/tests/test_countgraph.py +++ b/tests/test_countgraph.py @@ -116,6 +116,38 @@ def test_revhash_1(): assert hi.reverse_hash(hashval) == kmer +def test_extract_countgraph_info_badfile(): + try: + Countgraph.extract_info( + utils.get_test_data('test-abund-read-2.fa')) + assert 0, 'this should fail' + except ValueError: + pass + + +def test_extract_countgraph_info(): + fn = utils.get_temp_filename('test_extract_counting.ct') + for size in [1e6, 2e6, 5e6, 1e7]: + ht = khmer.Countgraph(25, size, 4) + ht.save(fn) + + try: + info = Countgraph.extract_info(fn) + except ValueError as err: + assert 0, 'Should not throw a ValueErorr: ' + str(err) + ksize, n_tables, table_size, _, _, _, _ = info + print(ksize, table_size, n_tables) + + assert(ksize) == 25 + assert table_size == size + assert n_tables == 4 + + try: + os.remove(fn) + except OSError as err: + assert 0, '...failed to remove ' + fn + str(err) + + class Test_Countgraph(object): def setup(self): diff --git a/tests/test_functions.py b/tests/test_functions.py index a88fd52b78..65cf660645 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -190,68 +190,6 @@ def test_get_primes_fal(): assert "unable to find 5 prime numbers < 5" in str(err) -def test_extract_countgraph_info_badfile(): - try: - khmer.extract_countgraph_info( - utils.get_test_data('test-abund-read-2.fa')) - assert 0, 'this should fail' - except ValueError: - pass - - -def test_extract_countgraph_info(): - fn = utils.get_temp_filename('test_extract_counting.ct') - for size in [1e6, 2e6, 5e6, 1e7]: - ht = khmer.Countgraph(25, size, 4) - ht.save(fn) - - try: - info = khmer.extract_countgraph_info(fn) - except ValueError as err: - assert 0, 'Should not throw a ValueErorr: ' + str(err) - ksize, n_tables, table_size, _, _, _, _ = info - print(ksize, table_size, n_tables) - - assert(ksize) == 25 - assert table_size == size - assert n_tables == 4 - - try: - os.remove(fn) - except OSError as err: - assert 0, '...failed to remove ' + fn + str(err) - - -def test_extract_nodegraph_info_badfile(): - try: - khmer.extract_nodegraph_info( - utils.get_test_data('test-abund-read-2.fa')) - assert 0, 'this should fail' - except ValueError: - pass - - -def test_extract_nodegraph_info(): - fn = utils.get_temp_filename('test_extract_nodegraph.pt') - for size in [1e6, 2e6, 5e6, 1e7]: - ht = khmer.Nodegraph(25, size, 4) - ht.save(fn) - - info = khmer.extract_nodegraph_info(fn) - ksize, table_size, n_tables, _, _, _ = info - print(ksize, table_size, n_tables) - - assert(ksize) == 25 - assert table_size == size, table_size - assert n_tables == 4 - - try: - os.remove(fn) - except OSError as err: - print('...failed to remove {fn}'.format(fn) + str(err), - file=sys.stderr) - - def test_check_file_status_kfile(): fn = utils.get_temp_filename('thisfiledoesnotexist') diff --git a/tests/test_nodegraph.py b/tests/test_nodegraph.py index 249f901acf..132c2424fc 100644 --- a/tests/test_nodegraph.py +++ b/tests/test_nodegraph.py @@ -34,9 +34,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,protected-access,no-member,invalid-name -from __future__ import print_function -from __future__ import absolute_import - import khmer from khmer import Nodegraph, Countgraph from khmer import ReadParser @@ -46,6 +43,7 @@ import screed import pytest +import os from . import khmer_tst_utils as utils @@ -63,6 +61,36 @@ def test_toobig(): print(str(err)) +def test_extract_nodegraph_info_badfile(): + try: + Nodegraph.extract_info( + utils.get_test_data('test-abund-read-2.fa')) + assert 0, 'this should fail' + except ValueError: + pass + + +def test_extract_nodegraph_info(): + fn = utils.get_temp_filename('test_extract_nodegraph.pt') + for size in [1e6, 2e6, 5e6, 1e7]: + ht = khmer.Nodegraph(25, size, 4) + ht.save(fn) + + info = Nodegraph.extract_info(fn) + ksize, table_size, n_tables, _, _, _ = info + print(ksize, table_size, n_tables) + + assert(ksize) == 25 + assert table_size == size, table_size + assert n_tables == 4 + + try: + os.remove(fn) + except OSError as err: + print('...failed to remove {fn}'.format(fn) + str(err), + file=sys.stderr) + + def test_add_tag(): nodegraph = khmer.Nodegraph(6, 1, 1) From c232d5bf01fb746d7acc0d3e567a7ac61efb8973 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 14:12:10 -0700 Subject: [PATCH 105/185] Remove __future__ imports --- Makefile | 4 ++-- doc/dev/guidelines-continued-dev.rst | 1 - examples/python-api/bloom.py | 1 - examples/python-api/consume.py | 1 - examples/python-api/exact-counting.py | 1 - khmer/_oxli/parsing.pxd | 1 - khmer/_oxli/parsing.pyx | 2 -- khmer/_oxli/utils.pyx | 1 - khmer/kfile.py | 2 -- khmer/khmer_args.py | 3 --- khmer/khmer_logger.py | 1 - khmer/thread_utils.py | 2 -- khmer/trimming.py | 1 - khmer/utils.py | 1 - oxli/build_graph.py | 1 - oxli/functions.py | 1 - oxli/partition.py | 1 - sandbox/assemble-and-track.py | 1 - sandbox/assemble-on-the-go.py | 1 - sandbox/assembly-diff-2.py | 1 - sandbox/assembly-diff.py | 2 -- sandbox/assemstats3.py | 2 -- sandbox/bloom-count.py | 2 -- sandbox/build-sparse-graph.py | 1 - sandbox/calc-best-assembly.py | 1 - sandbox/calc-error-profile.py | 2 -- sandbox/calc-median-distribution.py | 2 -- sandbox/collect-reads.py | 2 -- sandbox/collect-variants.py | 1 - sandbox/correct-reads.py | 1 - sandbox/count-kmers-single.py | 1 - sandbox/count-kmers.py | 1 - sandbox/error-correct-pass2.py | 1 - sandbox/estimate_optimal_hash.py | 1 - sandbox/extract-compact-dbg.py | 1 - sandbox/extract-single-partition.py | 1 - sandbox/extract-unassembled-reads-2.py | 1 - sandbox/extract-unassembled-reads.py | 1 - sandbox/filter-below-abund.py | 1 - sandbox/filter-median-and-pct.py | 1 - sandbox/filter-median.py | 1 - sandbox/graph-size.py | 1 - sandbox/link-compact-dbg.py | 1 - sandbox/make-coverage.py | 1 - sandbox/multi-rename.py | 1 - sandbox/normalize-by-median-pct.py | 2 -- sandbox/optimal_args_hashbits.py | 1 - sandbox/print-tagset.py | 1 - sandbox/readaligner_pairhmm_train.py | 2 -- sandbox/reassemble-contigs.py | 1 - sandbox/renumber-partitions.py | 1 - sandbox/saturate-by-median.py | 2 -- sandbox/shuffle-reverse-rotary.py | 1 - sandbox/slice-reads-by-coverage.py | 1 - sandbox/split-fasta.py | 1 - sandbox/split-sequences-by-length.py | 1 - sandbox/stoptag-abundance-hist.py | 1 - sandbox/stoptags-by-position.py | 1 - sandbox/streaming-assembly-simple.py | 1 - sandbox/strip-partition.py | 1 - sandbox/subset-report.py | 1 - sandbox/sweep-files.py | 1 - sandbox/sweep-out-reads-with-contigs.py | 1 - sandbox/sweep-reads.py | 1 - sandbox/sweep-reads2.py | 1 - sandbox/sweep-reads3.py | 1 - sandbox/write-trimmomatic.py | 1 - scripts/abundance-dist-single.py | 1 - scripts/abundance-dist.py | 1 - scripts/annotate-partitions.py | 1 - scripts/count-median.py | 1 - scripts/do-partition.py | 1 - scripts/extract-long-sequences.py | 1 - scripts/extract-paired-reads.py | 1 - scripts/extract-partitions.py | 1 - scripts/fastq-to-fasta.py | 1 - scripts/filter-abund-single.py | 1 - scripts/filter-abund.py | 1 - scripts/filter-stoptags.py | 1 - scripts/find-knots.py | 1 - scripts/interleave-reads.py | 1 - scripts/load-into-counting.py | 1 - scripts/make-initial-stoptags.py | 1 - scripts/merge-partitions.py | 1 - scripts/normalize-by-median.py | 1 - scripts/partition-graph.py | 1 - scripts/readstats.py | 1 - scripts/sample-reads-randomly.py | 1 - scripts/split-paired-reads.py | 1 - scripts/trim-low-abund.py | 1 - scripts/unique-kmers.py | 1 - tests/graph_features.py | 2 -- tests/khmer_tst_utils.py | 1 - tests/table_fixtures.py | 2 -- tests/test_assembly.py | 2 -- tests/test_banding.py | 2 -- tests/test_countgraph.py | 2 -- tests/test_counting_single.py | 2 -- tests/test_counttable.py | 2 -- tests/test_cpython_hierarchy.py | 2 -- tests/test_cython_parsing.py | 2 -- tests/test_functions.py | 2 -- tests/test_graph.py | 2 -- tests/test_hashset.py | 2 -- tests/test_hll.py | 2 -- tests/test_labelhash.py | 2 -- tests/test_lump.py | 1 - tests/test_nibblestorage.py | 2 -- tests/test_normalize_by_median.py | 1 - tests/test_oxli_functions.py | 1 - tests/test_qfstorage.py | 2 -- tests/test_read_aligner.py | 2 -- tests/test_read_handling.py | 3 --- tests/test_read_parsers.py | 2 -- tests/test_sandbox_scripts.py | 3 --- tests/test_script_arguments.py | 2 -- tests/test_script_output.py | 2 -- tests/test_scripts.py | 3 --- tests/test_sequence_validation.py | 2 -- tests/test_streaming_io.py | 3 --- tests/test_subset_graph.py | 1 - tests/test_version.py | 1 - 122 files changed, 2 insertions(+), 166 deletions(-) diff --git a/Makefile b/Makefile index a7b00c8308..7af5927cbe 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,8 @@ INCLUDESTRING=$(shell gcc -E -x c++ - -v < /dev/null 2>&1 >/dev/null \ INCLUDEOPTS=$(shell gcc -E -x c++ - -v < /dev/null 2>&1 >/dev/null \ | grep '^ /' | grep -v cc1plus | awk '{print "-I" $$1 " "}') -PYINCLUDE=$(shell python -c "from __future__ import print_function; \ - import sysconfig; flags = ['-I' + sysconfig.get_path('include'), \ +PYINCLUDE=$(shell python -c "import sysconfig; \ + flags = ['-I' + sysconfig.get_path('include'), \ '-I' + sysconfig.get_path('platinclude')]; print(' '.join(flags))") CPPCHECK_SOURCES=$(filter-out lib/test%, $(wildcard lib/*.cc khmer/_khmer.cc) ) diff --git a/doc/dev/guidelines-continued-dev.rst b/doc/dev/guidelines-continued-dev.rst index 56f4d97674..a367c1d5c2 100644 --- a/doc/dev/guidelines-continued-dev.rst +++ b/doc/dev/guidelines-continued-dev.rst @@ -158,7 +158,6 @@ When wrapping code from liboxli: For imports, -- `__future__` imports at the top, as usual. - `libc` cimports next, - then `libcpp` imports and cimports. - followed by cimports diff --git a/examples/python-api/bloom.py b/examples/python-api/bloom.py index dcfe6bb567..c437e330dc 100755 --- a/examples/python-api/bloom.py +++ b/examples/python-api/bloom.py @@ -4,7 +4,6 @@ # khmer accrues a small false positive rate in order to save substantially on # memory requirements. -from __future__ import print_function import khmer ksize = 21 diff --git a/examples/python-api/consume.py b/examples/python-api/consume.py index cf5fdc52b9..f3050114a7 100755 --- a/examples/python-api/consume.py +++ b/examples/python-api/consume.py @@ -2,7 +2,6 @@ # A demonstration of khmer's primary sequence loading function. -from __future__ import print_function import khmer import sys diff --git a/examples/python-api/exact-counting.py b/examples/python-api/exact-counting.py index 9656e48318..d4a320d8c8 100755 --- a/examples/python-api/exact-counting.py +++ b/examples/python-api/exact-counting.py @@ -3,7 +3,6 @@ # A demonstration of using khmer for exact k-mer counting. The memory required # is 4^k, which limits this to small values of k. -from __future__ import print_function import khmer # Note: diff --git a/khmer/_oxli/parsing.pxd b/khmer/_oxli/parsing.pxd index d77cfe024e..fe2ad3d57b 100644 --- a/khmer/_oxli/parsing.pxd +++ b/khmer/_oxli/parsing.pxd @@ -1,6 +1,5 @@ # -*- coding: UTF-8 -*- -from __future__ import unicode_literals from libc.stdint cimport uintptr_t diff --git a/khmer/_oxli/parsing.pyx b/khmer/_oxli/parsing.pyx index 2cc83a03e1..bf646a5ad9 100644 --- a/khmer/_oxli/parsing.pyx +++ b/khmer/_oxli/parsing.pyx @@ -1,7 +1,5 @@ # -*- coding: UTF-8 -*- -from __future__ import print_function -from __future__ import unicode_literals from cython.operator cimport dereference as deref cimport cython diff --git a/khmer/_oxli/utils.pyx b/khmer/_oxli/utils.pyx index 664fc4327d..508efdb682 100644 --- a/khmer/_oxli/utils.pyx +++ b/khmer/_oxli/utils.pyx @@ -1,6 +1,5 @@ # -*- coding: UTF-8 -*- -from __future__ import unicode_literals from cpython.version cimport PY_MAJOR_VERSION from cython import short, int, long diff --git a/khmer/kfile.py b/khmer/kfile.py index 92288334b2..f214119df3 100644 --- a/khmer/kfile.py +++ b/khmer/kfile.py @@ -34,8 +34,6 @@ # Contact: khmer-project@idyll.org """File handling/checking utilities for command-line scripts.""" -from __future__ import print_function, unicode_literals, division - import os import sys import errno diff --git a/khmer/khmer_args.py b/khmer/khmer_args.py index f47218c63c..5659d20043 100644 --- a/khmer/khmer_args.py +++ b/khmer/khmer_args.py @@ -35,9 +35,6 @@ # Contact: khmer-project@idyll.org """Common argparse constructs.""" -from __future__ import unicode_literals -from __future__ import print_function - import sys import argparse import math diff --git a/khmer/khmer_logger.py b/khmer/khmer_logger.py index bfe64b4a94..aa792f05b4 100644 --- a/khmer/khmer_logger.py +++ b/khmer/khmer_logger.py @@ -33,7 +33,6 @@ # Contact: khmer-project@idyll.org """Lightweight logging framework for khmer.""" -from __future__ import print_function, unicode_literals import sys __QUIET__ = False diff --git a/khmer/thread_utils.py b/khmer/thread_utils.py index 55c15eab7c..25e49f9678 100644 --- a/khmer/thread_utils.py +++ b/khmer/thread_utils.py @@ -35,8 +35,6 @@ # pylint: disable=missing-docstring,too-few-public-methods """Utilities for dealing with multithreaded processing of short reads.""" -from __future__ import print_function, unicode_literals - import threading import sys import screed diff --git a/khmer/trimming.py b/khmer/trimming.py index e23679bb0e..ff99756f27 100644 --- a/khmer/trimming.py +++ b/khmer/trimming.py @@ -32,7 +32,6 @@ # # Contact: khmer-project@idyll.org """Common methods for trimming short reads on k-mer abundance.""" -from __future__ import print_function, unicode_literals import screed diff --git a/khmer/utils.py b/khmer/utils.py index cee1704e4c..f39689fb39 100644 --- a/khmer/utils.py +++ b/khmer/utils.py @@ -33,7 +33,6 @@ # # Contact: khmer-project@idyll.org """Helpful methods for performing common argument-checking tasks in scripts.""" -from __future__ import print_function, unicode_literals from khmer._oxli.parsing import (check_is_left, check_is_right, check_is_pair, UnpairedReadsError, _split_left_right) import itertools diff --git a/oxli/build_graph.py b/oxli/build_graph.py index 58674197bd..fbb1e71db6 100755 --- a/oxli/build_graph.py +++ b/oxli/build_graph.py @@ -43,7 +43,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function, absolute_import, unicode_literals import sys diff --git a/oxli/functions.py b/oxli/functions.py index e3608f66da..b252c53fae 100755 --- a/oxli/functions.py +++ b/oxli/functions.py @@ -35,7 +35,6 @@ """A collection of functions for use throughout khmer/oxli.""" -from __future__ import print_function import threading import khmer.utils diff --git a/oxli/partition.py b/oxli/partition.py index 53afe58f55..6cf71febf3 100755 --- a/oxli/partition.py +++ b/oxli/partition.py @@ -7,7 +7,6 @@ # # pylint: disable=missing-docstring,no-member """Common functions for partitioning.""" -from __future__ import print_function, absolute_import, unicode_literals import sys import gc diff --git a/sandbox/assemble-and-track.py b/sandbox/assemble-and-track.py index 429bce66f2..09bbbb1e77 100755 --- a/sandbox/assemble-and-track.py +++ b/sandbox/assemble-and-track.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from __future__ import print_function import csv import screed import khmer diff --git a/sandbox/assemble-on-the-go.py b/sandbox/assemble-on-the-go.py index 6e5f882dec..a768638e75 100755 --- a/sandbox/assemble-on-the-go.py +++ b/sandbox/assemble-on-the-go.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from __future__ import print_function import screed import khmer import argparse diff --git a/sandbox/assembly-diff-2.py b/sandbox/assembly-diff-2.py index 74856151db..54ab13bca1 100755 --- a/sandbox/assembly-diff-2.py +++ b/sandbox/assembly-diff-2.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import sys import khmer import screed diff --git a/sandbox/assembly-diff.py b/sandbox/assembly-diff.py index 4c19d7fffb..fae61c0d58 100755 --- a/sandbox/assembly-diff.py +++ b/sandbox/assembly-diff.py @@ -33,8 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import division -from __future__ import print_function import sys import khmer import screed diff --git a/sandbox/assemstats3.py b/sandbox/assemstats3.py index 6104231270..176eed2b3f 100755 --- a/sandbox/assemstats3.py +++ b/sandbox/assemstats3.py @@ -41,8 +41,6 @@ You can obtain screed by running pip install screed ''' -from __future__ import division -from __future__ import print_function import screed import sys diff --git a/sandbox/bloom-count.py b/sandbox/bloom-count.py index 7feefca345..e9e6ce9e4d 100755 --- a/sandbox/bloom-count.py +++ b/sandbox/bloom-count.py @@ -34,8 +34,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,no-member -from __future__ import print_function -from __future__ import absolute_import import khmer import sys diff --git a/sandbox/build-sparse-graph.py b/sandbox/build-sparse-graph.py index 8a88f36044..e31a09204f 100755 --- a/sandbox/build-sparse-graph.py +++ b/sandbox/build-sparse-graph.py @@ -34,7 +34,6 @@ # # Contact: khmer-project@idyll.org -from __future__ import print_function import khmer import sys import screed diff --git a/sandbox/calc-best-assembly.py b/sandbox/calc-best-assembly.py index cc11a9bc1a..d06537457c 100755 --- a/sandbox/calc-best-assembly.py +++ b/sandbox/calc-best-assembly.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import screed import argparse import sys diff --git a/sandbox/calc-error-profile.py b/sandbox/calc-error-profile.py index e4d122a43e..e607ba9688 100755 --- a/sandbox/calc-error-profile.py +++ b/sandbox/calc-error-profile.py @@ -41,8 +41,6 @@ Reads FASTQ and FASTA input. """ -from __future__ import division -from __future__ import print_function import sys import argparse diff --git a/sandbox/calc-median-distribution.py b/sandbox/calc-median-distribution.py index 78b8e7e563..e502adbdcb 100755 --- a/sandbox/calc-median-distribution.py +++ b/sandbox/calc-median-distribution.py @@ -33,8 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import division -from __future__ import print_function import sys import khmer import argparse diff --git a/sandbox/collect-reads.py b/sandbox/collect-reads.py index 6d8e397904..536d0fd40e 100755 --- a/sandbox/collect-reads.py +++ b/sandbox/collect-reads.py @@ -43,8 +43,6 @@ Use '-h' for parameter help. """ -from __future__ import division -from __future__ import print_function import sys import textwrap diff --git a/sandbox/collect-variants.py b/sandbox/collect-variants.py index 339e2760ba..a30fed50fb 100755 --- a/sandbox/collect-variants.py +++ b/sandbox/collect-variants.py @@ -40,7 +40,6 @@ TODO: add to sandbox README """ -from __future__ import print_function import sys import screed diff --git a/sandbox/correct-reads.py b/sandbox/correct-reads.py index 7dad20c336..7724185bc6 100755 --- a/sandbox/correct-reads.py +++ b/sandbox/correct-reads.py @@ -44,7 +44,6 @@ TODO: add to sandbox/README. """ -from __future__ import print_function import sys import os import tempfile diff --git a/sandbox/count-kmers-single.py b/sandbox/count-kmers-single.py index aca0d7be2c..fd8aaf5b3d 100755 --- a/sandbox/count-kmers-single.py +++ b/sandbox/count-kmers-single.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import khmer diff --git a/sandbox/count-kmers.py b/sandbox/count-kmers.py index 097e956c59..313b2e400d 100755 --- a/sandbox/count-kmers.py +++ b/sandbox/count-kmers.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import khmer diff --git a/sandbox/error-correct-pass2.py b/sandbox/error-correct-pass2.py index 8e25141cdc..9085232046 100755 --- a/sandbox/error-correct-pass2.py +++ b/sandbox/error-correct-pass2.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import os import screed diff --git a/sandbox/estimate_optimal_hash.py b/sandbox/estimate_optimal_hash.py index 8c31a7c21d..66dc5f2333 100755 --- a/sandbox/estimate_optimal_hash.py +++ b/sandbox/estimate_optimal_hash.py @@ -54,7 +54,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import argparse import khmer, oxli from khmer.khmer_args import info, optimal_size, sanitize_help diff --git a/sandbox/extract-compact-dbg.py b/sandbox/extract-compact-dbg.py index dc486e9c9b..bfa44a5c4c 100755 --- a/sandbox/extract-compact-dbg.py +++ b/sandbox/extract-compact-dbg.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from __future__ import print_function import khmer import screed import argparse diff --git a/sandbox/extract-single-partition.py b/sandbox/extract-single-partition.py index deb84ee531..289609c9dc 100755 --- a/sandbox/extract-single-partition.py +++ b/sandbox/extract-single-partition.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import sys from screed.fasta import fasta_iter diff --git a/sandbox/extract-unassembled-reads-2.py b/sandbox/extract-unassembled-reads-2.py index 56095e0618..be993481a7 100755 --- a/sandbox/extract-unassembled-reads-2.py +++ b/sandbox/extract-unassembled-reads-2.py @@ -13,7 +13,6 @@ erroneous paths from super-high-abundance data * run this script with the assembly & the remaining reads. """ -from __future__ import print_function import sys import os.path import khmer, khmer.utils diff --git a/sandbox/extract-unassembled-reads.py b/sandbox/extract-unassembled-reads.py index 3fc7842e39..478390c669 100755 --- a/sandbox/extract-unassembled-reads.py +++ b/sandbox/extract-unassembled-reads.py @@ -8,7 +8,6 @@ erroneous paths from super-high-abundance data * run this script with the assembly & the remaining reads. """ -from __future__ import print_function import sys import os.path import khmer, khmer.utils diff --git a/sandbox/filter-below-abund.py b/sandbox/filter-below-abund.py index 16a03ba1d9..bead13128d 100755 --- a/sandbox/filter-below-abund.py +++ b/sandbox/filter-below-abund.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import sys import os import khmer diff --git a/sandbox/filter-median-and-pct.py b/sandbox/filter-median-and-pct.py index 20047d5c20..5da4a28182 100755 --- a/sandbox/filter-median-and-pct.py +++ b/sandbox/filter-median-and-pct.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import screed.fasta import os diff --git a/sandbox/filter-median.py b/sandbox/filter-median.py index c6a4aa2ed1..fd97a5f66a 100755 --- a/sandbox/filter-median.py +++ b/sandbox/filter-median.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import screed.fasta import os diff --git a/sandbox/graph-size.py b/sandbox/graph-size.py index f9b955345f..44f68fedab 100755 --- a/sandbox/graph-size.py +++ b/sandbox/graph-size.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import khmer import sys import screed diff --git a/sandbox/link-compact-dbg.py b/sandbox/link-compact-dbg.py index 65372e7efc..2687b45a73 100755 --- a/sandbox/link-compact-dbg.py +++ b/sandbox/link-compact-dbg.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from __future__ import print_function import khmer import screed import argparse diff --git a/sandbox/make-coverage.py b/sandbox/make-coverage.py index 87767a552e..67f27588d5 100755 --- a/sandbox/make-coverage.py +++ b/sandbox/make-coverage.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import screed diff --git a/sandbox/multi-rename.py b/sandbox/multi-rename.py index ea8f595f9f..4fd469d0c4 100755 --- a/sandbox/multi-rename.py +++ b/sandbox/multi-rename.py @@ -34,7 +34,6 @@ # # Contact: khmer-project@idyll.org -from __future__ import print_function import screed import sys import textwrap diff --git a/sandbox/normalize-by-median-pct.py b/sandbox/normalize-by-median-pct.py index 9c03e68464..fa865b4cdd 100755 --- a/sandbox/normalize-by-median-pct.py +++ b/sandbox/normalize-by-median-pct.py @@ -41,8 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import division -from __future__ import print_function import sys import screed diff --git a/sandbox/optimal_args_hashbits.py b/sandbox/optimal_args_hashbits.py index 794ad26db5..34fbf5a223 100755 --- a/sandbox/optimal_args_hashbits.py +++ b/sandbox/optimal_args_hashbits.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import math diff --git a/sandbox/print-tagset.py b/sandbox/print-tagset.py index 19a950ce53..0fbf4b3084 100755 --- a/sandbox/print-tagset.py +++ b/sandbox/print-tagset.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import khmer import sys import os diff --git a/sandbox/readaligner_pairhmm_train.py b/sandbox/readaligner_pairhmm_train.py index e761c402d5..d6105be167 100755 --- a/sandbox/readaligner_pairhmm_train.py +++ b/sandbox/readaligner_pairhmm_train.py @@ -33,8 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import division -from __future__ import print_function import khmer import argparse import collections diff --git a/sandbox/reassemble-contigs.py b/sandbox/reassemble-contigs.py index d968b2567a..2c23083cd8 100755 --- a/sandbox/reassemble-contigs.py +++ b/sandbox/reassemble-contigs.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from __future__ import print_function import argparse import screed import khmer diff --git a/sandbox/renumber-partitions.py b/sandbox/renumber-partitions.py index 895cbc28b1..00f0f47350 100755 --- a/sandbox/renumber-partitions.py +++ b/sandbox/renumber-partitions.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import sys import screed import gzip diff --git a/sandbox/saturate-by-median.py b/sandbox/saturate-by-median.py index 64c1375a33..e784568011 100755 --- a/sandbox/saturate-by-median.py +++ b/sandbox/saturate-by-median.py @@ -40,8 +40,6 @@ reads whether or not they have high coverage. This is better for assessing saturation of (esp) low-coverage data sets. """ -from __future__ import division -from __future__ import print_function import sys import screed diff --git a/sandbox/shuffle-reverse-rotary.py b/sandbox/shuffle-reverse-rotary.py index 4b0bd57505..94c76cef35 100755 --- a/sandbox/shuffle-reverse-rotary.py +++ b/sandbox/shuffle-reverse-rotary.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import sys import screed import os.path diff --git a/sandbox/slice-reads-by-coverage.py b/sandbox/slice-reads-by-coverage.py index 61e91c0ec8..2428a4fd98 100755 --- a/sandbox/slice-reads-by-coverage.py +++ b/sandbox/slice-reads-by-coverage.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import argparse import screed import sys diff --git a/sandbox/split-fasta.py b/sandbox/split-fasta.py index b2f15c39f1..d0488e009e 100755 --- a/sandbox/split-fasta.py +++ b/sandbox/split-fasta.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import sys import screed diff --git a/sandbox/split-sequences-by-length.py b/sandbox/split-sequences-by-length.py index fafa9271ee..a267f24149 100755 --- a/sandbox/split-sequences-by-length.py +++ b/sandbox/split-sequences-by-length.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import screed.fasta import os diff --git a/sandbox/stoptag-abundance-hist.py b/sandbox/stoptag-abundance-hist.py index 61ee3295d2..4fc6d61791 100755 --- a/sandbox/stoptag-abundance-hist.py +++ b/sandbox/stoptag-abundance-hist.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import sys import khmer import os diff --git a/sandbox/stoptags-by-position.py b/sandbox/stoptags-by-position.py index 21f0839213..61e5c37063 100755 --- a/sandbox/stoptags-by-position.py +++ b/sandbox/stoptags-by-position.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import khmer import sys import screed diff --git a/sandbox/streaming-assembly-simple.py b/sandbox/streaming-assembly-simple.py index ea68f71594..c01a26d763 100755 --- a/sandbox/streaming-assembly-simple.py +++ b/sandbox/streaming-assembly-simple.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from __future__ import print_function import csv import screed import khmer diff --git a/sandbox/strip-partition.py b/sandbox/strip-partition.py index 14c1008e28..a2995304ec 100755 --- a/sandbox/strip-partition.py +++ b/sandbox/strip-partition.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import screed import sys diff --git a/sandbox/subset-report.py b/sandbox/subset-report.py index 081ae40c0d..21aa695b76 100755 --- a/sandbox/subset-report.py +++ b/sandbox/subset-report.py @@ -33,7 +33,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Contact: khmer-project@idyll.org -from __future__ import print_function import khmer import sys import gc diff --git a/sandbox/sweep-files.py b/sandbox/sweep-files.py index 47bf30c51f..e35c9d9e6b 100755 --- a/sandbox/sweep-files.py +++ b/sandbox/sweep-files.py @@ -41,7 +41,6 @@ % sweep-files.py -r --db \ --query """ -from __future__ import print_function EPILOG = """ Output will be a collection of fasta/q files, each corresponding to a database diff --git a/sandbox/sweep-out-reads-with-contigs.py b/sandbox/sweep-out-reads-with-contigs.py index 0673ddc19a..bf20a6468c 100755 --- a/sandbox/sweep-out-reads-with-contigs.py +++ b/sandbox/sweep-out-reads-with-contigs.py @@ -34,7 +34,6 @@ # # Contact: khmer-project@idyll.org -from __future__ import print_function import sys import khmer import os.path diff --git a/sandbox/sweep-reads.py b/sandbox/sweep-reads.py index 585e260e0a..6c6602810e 100755 --- a/sandbox/sweep-reads.py +++ b/sandbox/sweep-reads.py @@ -34,7 +34,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=invalid-name,missing-docstring,no-member -from __future__ import print_function, unicode_literals from io import open diff --git a/sandbox/sweep-reads2.py b/sandbox/sweep-reads2.py index f1a2d8ef2e..f512407c65 100755 --- a/sandbox/sweep-reads2.py +++ b/sandbox/sweep-reads2.py @@ -42,7 +42,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import khmer diff --git a/sandbox/sweep-reads3.py b/sandbox/sweep-reads3.py index 942c2075dc..1dfda39b72 100755 --- a/sandbox/sweep-reads3.py +++ b/sandbox/sweep-reads3.py @@ -42,7 +42,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import os.path diff --git a/sandbox/write-trimmomatic.py b/sandbox/write-trimmomatic.py index e18aaf6e02..9fcaac83f2 100755 --- a/sandbox/write-trimmomatic.py +++ b/sandbox/write-trimmomatic.py @@ -34,7 +34,6 @@ # # Contact: khmer-project@idyll.org -from __future__ import print_function import glob filelist = glob.glob('*R1*.fastq.gz') diff --git a/scripts/abundance-dist-single.py b/scripts/abundance-dist-single.py index 56278cbfa1..01ad2a6648 100755 --- a/scripts/abundance-dist-single.py +++ b/scripts/abundance-dist-single.py @@ -43,7 +43,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import os import sys import csv diff --git a/scripts/abundance-dist.py b/scripts/abundance-dist.py index 7050c342cb..af8b6c5e8e 100755 --- a/scripts/abundance-dist.py +++ b/scripts/abundance-dist.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import csv diff --git a/scripts/annotate-partitions.py b/scripts/annotate-partitions.py index 91f9996674..3d4e98faa4 100755 --- a/scripts/annotate-partitions.py +++ b/scripts/annotate-partitions.py @@ -43,7 +43,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import os import textwrap diff --git a/scripts/count-median.py b/scripts/count-median.py index 86060e91b4..7d0db34a62 100755 --- a/scripts/count-median.py +++ b/scripts/count-median.py @@ -49,7 +49,6 @@ NOTE: All 'N's in the input sequences are converted to 'A's. """ -from __future__ import print_function import argparse import screed import sys diff --git a/scripts/do-partition.py b/scripts/do-partition.py index 231c0bbbc8..0027270b58 100755 --- a/scripts/do-partition.py +++ b/scripts/do-partition.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import khmer import sys diff --git a/scripts/extract-long-sequences.py b/scripts/extract-long-sequences.py index 7de3f6872f..7526c4aedf 100755 --- a/scripts/extract-long-sequences.py +++ b/scripts/extract-long-sequences.py @@ -46,7 +46,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import argparse import screed import textwrap diff --git a/scripts/extract-paired-reads.py b/scripts/extract-paired-reads.py index 7ce24ece01..29d7cbe3cb 100755 --- a/scripts/extract-paired-reads.py +++ b/scripts/extract-paired-reads.py @@ -44,7 +44,6 @@ Reads FASTQ and FASTA input, retains format for output. """ -from __future__ import print_function import sys import os.path import textwrap diff --git a/scripts/extract-partitions.py b/scripts/extract-partitions.py index 48e8240a9a..a1d25dcdd6 100755 --- a/scripts/extract-partitions.py +++ b/scripts/extract-partitions.py @@ -46,7 +46,6 @@ @CTB note that if threshold is != 1, those sequences will not be output by output_unassigned... """ -from __future__ import print_function import sys import screed diff --git a/scripts/fastq-to-fasta.py b/scripts/fastq-to-fasta.py index 36ab82377c..ef597d9c39 100755 --- a/scripts/fastq-to-fasta.py +++ b/scripts/fastq-to-fasta.py @@ -42,7 +42,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function, unicode_literals import sys import screed from khmer import __version__ diff --git a/scripts/filter-abund-single.py b/scripts/filter-abund-single.py index a2e44c37c6..6d810df03a 100755 --- a/scripts/filter-abund-single.py +++ b/scripts/filter-abund-single.py @@ -45,7 +45,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import os import sys import threading diff --git a/scripts/filter-abund.py b/scripts/filter-abund.py index af2855dbb2..cb729c9b77 100755 --- a/scripts/filter-abund.py +++ b/scripts/filter-abund.py @@ -44,7 +44,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import os import textwrap diff --git a/scripts/filter-stoptags.py b/scripts/filter-stoptags.py index 7d0f744898..a2bc48f170 100755 --- a/scripts/filter-stoptags.py +++ b/scripts/filter-stoptags.py @@ -44,7 +44,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import os import textwrap diff --git a/scripts/find-knots.py b/scripts/find-knots.py index 5eab6c8385..ccfb45a6af 100755 --- a/scripts/find-knots.py +++ b/scripts/find-knots.py @@ -41,7 +41,6 @@ % python scripts/find-knots.py """ -from __future__ import print_function import glob import os diff --git a/scripts/interleave-reads.py b/scripts/interleave-reads.py index fdecd7c72f..65c557d5f2 100755 --- a/scripts/interleave-reads.py +++ b/scripts/interleave-reads.py @@ -44,7 +44,6 @@ By default, output is sent to stdout; or use -o. Use '-h' for parameter help. """ -from __future__ import print_function import screed import sys diff --git a/scripts/load-into-counting.py b/scripts/load-into-counting.py index 8164f4e84a..6e797232a8 100755 --- a/scripts/load-into-counting.py +++ b/scripts/load-into-counting.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function, unicode_literals import json import os diff --git a/scripts/make-initial-stoptags.py b/scripts/make-initial-stoptags.py index a56f116661..cbe78657ca 100755 --- a/scripts/make-initial-stoptags.py +++ b/scripts/make-initial-stoptags.py @@ -39,7 +39,6 @@ % python scripts/make-initial-stoptags.py """ -from __future__ import print_function import sys import textwrap diff --git a/scripts/merge-partitions.py b/scripts/merge-partitions.py index efa1b237f2..a0acca36b8 100755 --- a/scripts/merge-partitions.py +++ b/scripts/merge-partitions.py @@ -42,7 +42,6 @@ Load .subset.*.pmap and merge into a single pmap file. Final merged pmap file will be in .pmap.merged. """ -from __future__ import print_function import glob import os diff --git a/scripts/normalize-by-median.py b/scripts/normalize-by-median.py index b2971ba073..39e387663e 100755 --- a/scripts/normalize-by-median.py +++ b/scripts/normalize-by-median.py @@ -45,7 +45,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import sys import screed diff --git a/scripts/partition-graph.py b/scripts/partition-graph.py index ab81661a0c..f841fe4848 100755 --- a/scripts/partition-graph.py +++ b/scripts/partition-graph.py @@ -43,7 +43,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import threading import textwrap diff --git a/scripts/readstats.py b/scripts/readstats.py index b42cb30556..b6b06d9796 100755 --- a/scripts/readstats.py +++ b/scripts/readstats.py @@ -41,7 +41,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import argparse import sys diff --git a/scripts/sample-reads-randomly.py b/scripts/sample-reads-randomly.py index 5ec2fbb13e..f9206e8d32 100755 --- a/scripts/sample-reads-randomly.py +++ b/scripts/sample-reads-randomly.py @@ -45,7 +45,6 @@ Reads FASTQ and FASTA input, retains format for output. """ -from __future__ import print_function import argparse import os.path diff --git a/scripts/split-paired-reads.py b/scripts/split-paired-reads.py index 22d99ee644..5750100312 100755 --- a/scripts/split-paired-reads.py +++ b/scripts/split-paired-reads.py @@ -44,7 +44,6 @@ Reads FASTQ and FASTA input, retains format for output. """ -from __future__ import print_function import sys import os import textwrap diff --git a/scripts/trim-low-abund.py b/scripts/trim-low-abund.py index a4e34b9fb9..1f1177227d 100755 --- a/scripts/trim-low-abund.py +++ b/scripts/trim-low-abund.py @@ -43,7 +43,6 @@ Use -h for parameter help. """ -from __future__ import print_function import csv import sys import os diff --git a/scripts/unique-kmers.py b/scripts/unique-kmers.py index 61b07c2d85..279da4bdd5 100755 --- a/scripts/unique-kmers.py +++ b/scripts/unique-kmers.py @@ -42,7 +42,6 @@ Use '-h' for parameter help. """ -from __future__ import print_function import argparse import os diff --git a/tests/graph_features.py b/tests/graph_features.py index bf7699b688..c2d6912846 100644 --- a/tests/graph_features.py +++ b/tests/graph_features.py @@ -36,8 +36,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,protected-access,no-member,invalid-name -from __future__ import print_function -from __future__ import absolute_import import itertools import random diff --git a/tests/khmer_tst_utils.py b/tests/khmer_tst_utils.py index e310874d37..dc9c867ddf 100644 --- a/tests/khmer_tst_utils.py +++ b/tests/khmer_tst_utils.py @@ -35,7 +35,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring -from __future__ import print_function import tempfile import os import shutil diff --git a/tests/table_fixtures.py b/tests/table_fixtures.py index 18a9ac2e15..4bc5a9e169 100644 --- a/tests/table_fixtures.py +++ b/tests/table_fixtures.py @@ -34,8 +34,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,invalid-name -from __future__ import print_function -from __future__ import absolute_import from khmer import Countgraph, SmallCountgraph, Nodegraph from khmer import Nodetable, Counttable, SmallCounttable, QFCounttable diff --git a/tests/test_assembly.py b/tests/test_assembly.py index 8d6fcf95c2..d67f1337bc 100644 --- a/tests/test_assembly.py +++ b/tests/test_assembly.py @@ -36,8 +36,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,protected-access,no-member,invalid-name -from __future__ import print_function -from __future__ import absolute_import import itertools import random diff --git a/tests/test_banding.py b/tests/test_banding.py index cbd163e07a..3728ba0d8b 100644 --- a/tests/test_banding.py +++ b/tests/test_banding.py @@ -32,8 +32,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,protected-access,no-member,invalid-name -from __future__ import print_function -from __future__ import absolute_import, division import screed import khmer diff --git a/tests/test_countgraph.py b/tests/test_countgraph.py index 16e25f61e7..05cd331582 100644 --- a/tests/test_countgraph.py +++ b/tests/test_countgraph.py @@ -34,8 +34,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,protected-access,no-member,invalid-name -from __future__ import print_function -from __future__ import absolute_import, unicode_literals import gzip diff --git a/tests/test_counting_single.py b/tests/test_counting_single.py index 4532f819cc..585bc0834d 100644 --- a/tests/test_counting_single.py +++ b/tests/test_counting_single.py @@ -34,8 +34,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=C0111,C0103,missing-docstring,no-member,protected-access -from __future__ import print_function -from __future__ import absolute_import import khmer diff --git a/tests/test_counttable.py b/tests/test_counttable.py index 15b7808a0b..1873668a35 100644 --- a/tests/test_counttable.py +++ b/tests/test_counttable.py @@ -33,8 +33,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=C0111,C0103,missing-docstring,no-member,protected-access -from __future__ import print_function -from __future__ import absolute_import import khmer diff --git a/tests/test_cpython_hierarchy.py b/tests/test_cpython_hierarchy.py index 951f3655b2..a1b1707eb8 100644 --- a/tests/test_cpython_hierarchy.py +++ b/tests/test_cpython_hierarchy.py @@ -33,8 +33,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=C0111,C0103,missing-docstring,no-member,protected-access -from __future__ import print_function -from __future__ import absolute_import import khmer diff --git a/tests/test_cython_parsing.py b/tests/test_cython_parsing.py index 1c2aa40bbe..710ae711e2 100644 --- a/tests/test_cython_parsing.py +++ b/tests/test_cython_parsing.py @@ -1,5 +1,3 @@ -from __future__ import print_function -from __future__ import absolute_import import gc import itertools diff --git a/tests/test_functions.py b/tests/test_functions.py index 65cf660645..a289c58b2b 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -33,8 +33,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,invalid-name,no-member -from __future__ import print_function -from __future__ import absolute_import import screed import khmer diff --git a/tests/test_graph.py b/tests/test_graph.py index c3abd0445c..4eb8e32728 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -34,8 +34,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,no-member,invalid-name,no-self-use # pylint: disable=protected-access -from __future__ import print_function -from __future__ import absolute_import import khmer import screed diff --git a/tests/test_hashset.py b/tests/test_hashset.py index 1f7a6d89d3..650dfe68c7 100644 --- a/tests/test_hashset.py +++ b/tests/test_hashset.py @@ -36,8 +36,6 @@ """ Test code for HashSet objects. """ -from __future__ import print_function -from __future__ import absolute_import import khmer from . import khmer_tst_utils as utils diff --git a/tests/test_hll.py b/tests/test_hll.py index 5cad46f2aa..c22a9ae77a 100644 --- a/tests/test_hll.py +++ b/tests/test_hll.py @@ -33,8 +33,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,protected-access,no-member,invalid-name -from __future__ import division, print_function, unicode_literals -from __future__ import absolute_import import pickle diff --git a/tests/test_labelhash.py b/tests/test_labelhash.py index c11ba4d2d4..f632d33a66 100644 --- a/tests/test_labelhash.py +++ b/tests/test_labelhash.py @@ -33,8 +33,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,protected-access,no-member,invalid-name -from __future__ import print_function -from __future__ import absolute_import import os import khmer diff --git a/tests/test_lump.py b/tests/test_lump.py index b8cf5e0651..a475979a07 100644 --- a/tests/test_lump.py +++ b/tests/test_lump.py @@ -33,7 +33,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,no-member,protected-access,invalid-name -from __future__ import absolute_import import khmer diff --git a/tests/test_nibblestorage.py b/tests/test_nibblestorage.py index b64bb279bd..24943bb24c 100644 --- a/tests/test_nibblestorage.py +++ b/tests/test_nibblestorage.py @@ -32,8 +32,6 @@ # # Contact: khmer-project@idyll.org -from __future__ import print_function -from __future__ import absolute_import import random diff --git a/tests/test_normalize_by_median.py b/tests/test_normalize_by_median.py index 3b4b25a222..95ed93fbcf 100644 --- a/tests/test_normalize_by_median.py +++ b/tests/test_normalize_by_median.py @@ -32,7 +32,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,invalid-name -from __future__ import print_function, absolute_import, unicode_literals import os import threading diff --git a/tests/test_oxli_functions.py b/tests/test_oxli_functions.py index bf1dcef80c..28b06f6051 100644 --- a/tests/test_oxli_functions.py +++ b/tests/test_oxli_functions.py @@ -32,7 +32,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,invalid-name,no-member -from __future__ import print_function, absolute_import, unicode_literals from . import khmer_tst_utils as utils diff --git a/tests/test_qfstorage.py b/tests/test_qfstorage.py index daaa5eff8e..d12d058e08 100644 --- a/tests/test_qfstorage.py +++ b/tests/test_qfstorage.py @@ -1,5 +1,3 @@ -from __future__ import print_function -from __future__ import absolute_import import random diff --git a/tests/test_read_aligner.py b/tests/test_read_aligner.py index a2c715e988..da6543e368 100644 --- a/tests/test_read_aligner.py +++ b/tests/test_read_aligner.py @@ -33,8 +33,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,no-member,invalid-name,unused-variable -from __future__ import print_function -from __future__ import absolute_import import khmer from . import khmer_tst_utils as utils diff --git a/tests/test_read_handling.py b/tests/test_read_handling.py index 81f32c9b3a..020a594098 100644 --- a/tests/test_read_handling.py +++ b/tests/test_read_handling.py @@ -34,9 +34,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=C0111,C0103,E1103,W0612 -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals import gzip import os diff --git a/tests/test_read_parsers.py b/tests/test_read_parsers.py index 032c83d409..0601a888f2 100644 --- a/tests/test_read_parsers.py +++ b/tests/test_read_parsers.py @@ -35,8 +35,6 @@ # pylint: disable=missing-docstring,invalid-name # Tests for the ReadParser and Read classes. -from __future__ import print_function -from __future__ import absolute_import from khmer import Read from khmer import ReadParser from screed import Record diff --git a/tests/test_sandbox_scripts.py b/tests/test_sandbox_scripts.py index 2ce44574eb..92447b8d98 100644 --- a/tests/test_sandbox_scripts.py +++ b/tests/test_sandbox_scripts.py @@ -35,9 +35,6 @@ # pylint: disable=C0111,C0103,E1103,W0612 -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals import sys import os diff --git a/tests/test_script_arguments.py b/tests/test_script_arguments.py index 14d8b2ef9d..5764ae0fc2 100644 --- a/tests/test_script_arguments.py +++ b/tests/test_script_arguments.py @@ -36,8 +36,6 @@ """ Tests for various argument-handling code. """ -from __future__ import print_function, unicode_literals -from __future__ import absolute_import import sys import io diff --git a/tests/test_script_output.py b/tests/test_script_output.py index f1caf3da14..025e6cf2b3 100644 --- a/tests/test_script_output.py +++ b/tests/test_script_output.py @@ -37,8 +37,6 @@ Test code that verifies current script output md5 hashes against recorded hashes, to ensure that script function isn't changing. """ -from __future__ import print_function -from __future__ import absolute_import import khmer from . import khmer_tst_utils as utils diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 945c4739ec..348a521bf3 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -34,9 +34,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=C0111,C0103,E1103,unused-variable,protected-access -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals import csv import json diff --git a/tests/test_sequence_validation.py b/tests/test_sequence_validation.py index 5af6187929..192b51286f 100644 --- a/tests/test_sequence_validation.py +++ b/tests/test_sequence_validation.py @@ -35,8 +35,6 @@ # pylint: disable=missing-docstring,invalid-name # Tests for the ReadParser and Read classes. -from __future__ import print_function -from __future__ import absolute_import from khmer import Countgraph, SmallCountgraph, Nodegraph from khmer import Nodetable, Counttable, SmallCounttable from khmer import GraphLabels diff --git a/tests/test_streaming_io.py b/tests/test_streaming_io.py index 84a66c4e0f..072707a1da 100644 --- a/tests/test_streaming_io.py +++ b/tests/test_streaming_io.py @@ -37,9 +37,6 @@ # of the use of subprocess to execute. Most script tests should go into # test_scripts.py for this reason. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals import khmer from khmer import Nodegraph, Countgraph diff --git a/tests/test_subset_graph.py b/tests/test_subset_graph.py index c836801d69..bd34e41ba3 100644 --- a/tests/test_subset_graph.py +++ b/tests/test_subset_graph.py @@ -34,7 +34,6 @@ # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring,invalid-name,no-member,no-self-use # pylint: disable=protected-access -from __future__ import print_function, absolute_import import khmer from khmer._oxli.legacy_partitioning import SubsetPartition, PrePartitionInfo diff --git a/tests/test_version.py b/tests/test_version.py index d4366699d4..defde717b6 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -33,7 +33,6 @@ # # Contact: khmer-project@idyll.org # pylint: disable=missing-docstring -from __future__ import print_function, unicode_literals import khmer import pytest From 8512174f271e8f548f6b314577f18b23d947879a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 15:34:06 -0700 Subject: [PATCH 106/185] Introduce paired_fastx_handler, update sample-reads-randomly --- khmer/khmer_args.py | 23 +++++-- khmer/utils.py | 103 +++++++++---------------------- scripts/sample-reads-randomly.py | 13 ++-- tests/test_scripts.py | 10 +-- 4 files changed, 58 insertions(+), 91 deletions(-) diff --git a/khmer/khmer_args.py b/khmer/khmer_args.py index 5659d20043..89c01f4e3e 100644 --- a/khmer/khmer_args.py +++ b/khmer/khmer_args.py @@ -41,16 +41,13 @@ import textwrap from argparse import _VersionAction from collections import namedtuple -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO import screed import khmer from khmer import __version__, Countgraph -from .utils import print_error -from .khmer_logger import log_info, log_warn, configure_logging +from khmer.utils import print_error, PAIRING_MODES +from khmer.khmer_logger import log_info, log_warn, configure_logging DEFAULT_K = 32 @@ -492,6 +489,20 @@ def add_loadgraph_args(parser): help='load a precomputed k-mer graph from disk') +def add_pairing_args(parser): + """Common pairing mode argument.""" + parser.add_argument('--pairing-mode', default='interleaved', + choices=PAIRING_MODES, + help='How to interpret read pairing. With `single`, '\ + 'reads will be parsed as singletons, regardless'\ + ' of pairing or file order. With `interleaved`,'\ + ' each file will be assumed to be interleaved '\ + 'and paired, with singletons allowed to be mixed'\ + ' in. With `split`, it will be assumed that each'\ + ' group of two files in the input list are '\ + 'as (LEFT, RIGHT), ...') + + def calculate_graphsize(args, graphtype, multiplier=1.0): """ Transform the table parameters into a size. diff --git a/khmer/utils.py b/khmer/utils.py index f39689fb39..342cabae22 100644 --- a/khmer/utils.py +++ b/khmer/utils.py @@ -34,10 +34,19 @@ # Contact: khmer-project@idyll.org """Helpful methods for performing common argument-checking tasks in scripts.""" from khmer._oxli.parsing import (check_is_left, check_is_right, check_is_pair, - UnpairedReadsError, _split_left_right) + UnpairedReadsError, _split_left_right, + FastxParser, SplitPairedReader, + BrokenPairedReader) import itertools +PAIRING_MODES = ('split', 'interleaved', 'single') + +def grouper(n, iterable): + iterable = iter(iterable) + return iter(lambda: list(itertools.islice(iterable, n)), []) + + def print_error(msg): """Print the given message to 'stderr'.""" import sys @@ -45,76 +54,27 @@ def print_error(msg): print(msg, file=sys.stderr) -def broken_paired_reader(screed_iter, min_length=None, - force_single=False, require_paired=False): - """Read pairs from a stream. - - A generator that yields singletons and pairs from a stream of FASTA/FASTQ - records (yielded by 'screed_iter'). Yields (n, is_pair, r1, r2) where - 'r2' is None if is_pair is False. - - The input stream can be fully single-ended reads, interleaved paired-end - reads, or paired-end reads with orphans, a.k.a. "broken paired". - - Usage:: - - for n, is_pair, read1, read2 in broken_paired_reader(...): - ... +def paired_fastx_handler(samples, pairing_mode, *args, **kwargs): - Note that 'n' behaves like enumerate() and starts at 0, but tracks - the number of records read from the input stream, so is - incremented by 2 for a pair of reads. - - If 'min_length' is set, all reads under this length are ignored (even - if they are pairs). - - If 'force_single' is True, all reads are returned as singletons. - """ - record = None - prev_record = None - num = 0 - - if force_single and require_paired: - raise ValueError("force_single and require_paired cannot both be set!") - - # handle the majority of the stream. - for record in screed_iter: - if prev_record: - if check_is_pair(prev_record, record) and not force_single: - if min_length and (len(prev_record.sequence) < min_length or - len(record.sequence) < min_length): - if require_paired: - record = None - else: - yield num, True, prev_record, record # it's a pair! - num += 2 - record = None - else: # orphan. - if require_paired: - err = UnpairedReadsError( - "Unpaired reads when require_paired is set!", - prev_record, record) - raise err - - # ignore short reads - if min_length and len(prev_record.sequence) < min_length: - pass - else: - yield num, False, prev_record, None - num += 1 - - prev_record = record - record = None - - # handle the last record, if it exists (i.e. last two records not a pair) - if prev_record: - if require_paired: - raise UnpairedReadsError("Unpaired reads when require_paired " - "is set!", prev_record, None) - if min_length and len(prev_record.sequence) < min_length: - pass + if pairing_mode not in PAIRING_MODES: + raise ValueError('Pairing mode must be one of {0}'.format(PAIRING_MODES)) + + if pairing_mode == 'split': + _samples = grouper(2, samples) + else: + _samples = samples + + for group in _samples: + if pairing_mode == 'split': + reader = SplitPairedReader(FastxParser(group[0]), + FastxParser(group[1])) + elif pairing_mode == 'single': + reader = BrokenPairedReader(FastxParser(group), + force_single=True) else: - yield num, False, prev_record, None + reader = BrokenPairedReader(FastxParser(group), + force_single=False) + yield reader def write_record(record, fileobj): @@ -188,10 +148,5 @@ def total_length(self): return sum([len(r.sequence) for r in self.reads]) -def grouper(n, iterable): - iterable = iter(iterable) - return iter(lambda: list(itertools.islice(iterable, n)), []) - - # vim: set filetype=python tabstop=4 softtabstop=4 shiftwidth=4 expandtab: # vim: set textwidth=79: diff --git a/scripts/sample-reads-randomly.py b/scripts/sample-reads-randomly.py index f9206e8d32..0784c18692 100755 --- a/scripts/sample-reads-randomly.py +++ b/scripts/sample-reads-randomly.py @@ -56,8 +56,9 @@ from khmer import ReadParser from khmer.kfile import (check_input_files, add_output_compression_type, get_file_writer) -from khmer.khmer_args import sanitize_help, KhmerArgumentParser -from khmer.utils import write_record, broken_paired_reader +from khmer.khmer_args import (sanitize_help, KhmerArgumentParser, + add_pairing_args) +from khmer.utils import write_record, paired_fastx_handler DEFAULT_NUM_READS = int(1e5) DEFAULT_MAX_READS = int(1e8) @@ -93,14 +94,13 @@ def get_parser(): default=1) parser.add_argument('-R', '--random-seed', type=int, dest='random_seed', help='Provide a random seed for the generator') - parser.add_argument('--force_single', default=False, action='store_true', - help='Ignore read pair information if present') parser.add_argument('-o', '--output', dest='output_file', type=argparse.FileType('wb'), metavar="filename", default=None) parser.add_argument('-f', '--force', default=False, action='store_true', help='Overwrite output file if it exits') add_output_compression_type(parser) + add_pairing_args(parser) return parser @@ -167,11 +167,10 @@ def main(): reads.append([]) # read through all the sequences and load/resample the reservoir - for filename in args.filenames: + for reader in paired_fastx_handler(args.filenames, args.pairing_mode): print('opening', filename, 'for reading', file=sys.stderr) - for count, (_, _, rcrd1, rcrd2) in enumerate(broken_paired_reader( - ReadParser(filename), force_single=args.force_single)): + for count, (_, _, rcrd1, rcrd2) in enumerate(reader): if count % 10000 == 0: print('...', count, 'reads scanned', file=sys.stderr) if count >= args.max_reads: diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 348a521bf3..1ebe0d2107 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -1688,13 +1688,14 @@ def test_sample_reads_randomly(): assert seqs == answer -def test_sample_reads_randomly_force_single(): +def test_sample_reads_randomly_single_mode(): infile = utils.copy_test_data('test-reads.fa') in_dir = os.path.dirname(infile) script = 'sample-reads-randomly.py' # fix random number seed for reproducibility - args = ['-N', '10', '-M', '12000', '-R', '1', '--force_single'] + args = ['-N', '10', '-M', '12000', '-R', '1', + '--pairing-mode', 'single'] args.append(infile) utils.runscript(script, args, in_dir) @@ -1730,13 +1731,14 @@ def test_sample_reads_randomly_force_single(): assert seqs == answer -def test_sample_reads_randomly_force_single_outfile(): +def test_sample_reads_randomly_single_mode_outfile(): infile = utils.copy_test_data('test-reads.fa') in_dir = os.path.dirname(infile) script = 'sample-reads-randomly.py' # fix random number seed for reproducibility - args = ['-N', '10', '-M', '12000', '-R', '1', '--force_single', '-o', + args = ['-N', '10', '-M', '12000', '-R', '1', + '--pairing-mode', 'single', '-o', in_dir + '/randreads.out'] args.append(infile) From 96824803cc84e8b9878f3adc87c1c6fb3cc0cdde Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 17:55:36 -0700 Subject: [PATCH 107/185] Split Sequence to its own module, add a clean method to Sequence, make a trim function --- khmer/__init__.py | 3 +- khmer/_oxli/__init__.py | 6 -- khmer/_oxli/graphs.pxd | 3 + khmer/_oxli/graphs.pyx | 7 ++ khmer/_oxli/parsing.pxd | 71 +------------- khmer/_oxli/parsing.pyx | 164 ++----------------------------- khmer/_oxli/sequence.pxd | 79 +++++++++++++++ khmer/_oxli/sequence.pyx | 182 +++++++++++++++++++++++++++++++++++ khmer/_oxli/utils.pyx | 10 +- khmer/utils.py | 25 ++++- tests/test_cython_parsing.py | 5 +- 11 files changed, 312 insertions(+), 243 deletions(-) create mode 100644 khmer/_oxli/sequence.pxd create mode 100644 khmer/_oxli/sequence.pyx diff --git a/khmer/__init__.py b/khmer/__init__.py index fe4f2b5db0..43fcd2b19a 100644 --- a/khmer/__init__.py +++ b/khmer/__init__.py @@ -67,7 +67,8 @@ from khmer._oxli.legacy_partitioning import SubsetPartition, PrePartitionInfo -from khmer._oxli.parsing import FastxParser +from khmer._oxli.parsing import (FastxParser, SanitizedFastxParser, + BrokenPairedReader) from khmer._oxli.readaligner import ReadAligner diff --git a/khmer/_oxli/__init__.py b/khmer/_oxli/__init__.py index 06d02cd291..e69de29bb2 100644 --- a/khmer/_oxli/__init__.py +++ b/khmer/_oxli/__init__.py @@ -1,6 +0,0 @@ -from .assembly import LinearAssembler -from .hashing import Kmer -from .parsing import Alphabets, Sequence, ReadBundle, UnpairedReadsError -from .parsing import FastxParser, SanitizedFastxParser, SplitPairedReader -from .parsing import BrokenPairedReader, _split_left_right -from .parsing import check_is_left, check_is_right, check_is_pair diff --git a/khmer/_oxli/graphs.pxd b/khmer/_oxli/graphs.pxd index 55339a2ec1..7e380eeabb 100644 --- a/khmer/_oxli/graphs.pxd +++ b/khmer/_oxli/graphs.pxd @@ -10,6 +10,7 @@ from khmer._oxli.hashing cimport Kmer, CpKmer, KmerSet, CpKmerFactory, CpKmerIte from khmer._oxli.parsing cimport CpReadParser, CpSequence from khmer._oxli.legacy_partitioning cimport (CpSubsetPartition, cp_pre_partition_info, SubsetPartition) +from khmer._oxli.sequence cimport Sequence from khmer._oxli.utils cimport oxli_raise_py_error @@ -248,6 +249,8 @@ cdef class Hashtable: cdef CpKmer _build_kmer(self, object kmer) except * cdef list _get_raw_tables(self, uint8_t **, vector[uint64_t]) + cdef int _trim_on_abundance(self, Sequence sequence, int abundance) + cdef class QFCounttable(Hashtable): cdef shared_ptr[CpQFCounttable] _qf_this diff --git a/khmer/_oxli/graphs.pyx b/khmer/_oxli/graphs.pyx index 7eb084d132..8d767d9c4e 100644 --- a/khmer/_oxli/graphs.pyx +++ b/khmer/_oxli/graphs.pyx @@ -20,6 +20,7 @@ from khmer._oxli.hashset cimport HashSet from khmer._oxli.legacy_partitioning cimport (CpSubsetPartition, SubsetPartition, cp_pre_partition_info, PrePartitionInfo) from khmer._oxli.oxli_types cimport MAX_BIGCOUNT, HashIntoType +from khmer._oxli.sequence cimport Sequence from khmer._oxli.traversal cimport Traverser from khmer._khmer import ReadParser @@ -207,6 +208,12 @@ cdef class Hashtable: trimmed_at = deref(self._ht_this).trim_on_abundance(data, abundance) return sequence[:trimmed_at], trimmed_at + cdef int _trim_on_abundance(self, Sequence sequence, int abundance): + trimmed_at = \ + deref(self._ht_this).trim_on_abundance(sequence._obj.cleaned_seq, + abundance) + return trimmed_at + def trim_below_abundance(self, str sequence, int abundance): """Trim sequence at first k-mer above the given abundance.""" cdef bytes data = self._valid_sequence(sequence) diff --git a/khmer/_oxli/parsing.pxd b/khmer/_oxli/parsing.pxd index fe2ad3d57b..7b1c77ede5 100644 --- a/khmer/_oxli/parsing.pxd +++ b/khmer/_oxli/parsing.pxd @@ -9,49 +9,14 @@ from libcpp.utility cimport pair from libcpp.string cimport string from khmer._oxli.utils cimport oxli_raise_py_error +from khmer._oxli.sequence cimport Sequence, CpSequence, CpSequencePair ''' extern declarations for liboxli. ''' -# C++ ostream wrapper code stolen shamelessly from stackoverflow -# http://stackoverflow.com/questions/30984078/cython-working-with-c-streams -# We need ostream to wrap ReadParser -cdef extern from "" namespace "std": - cdef cppclass ostream: - ostream& write(const char*, int) except + - -# obviously std::ios_base isn't a namespace, but this lets -# Cython generate the connect C++ code -cdef extern from "" namespace "std::ios_base": - cdef cppclass open_mode: - pass - cdef open_mode binary - # you can define other constants as needed - - -cdef extern from "" namespace "std": - cdef cppclass ofstream(ostream): - # constructors - ofstream(const char*) except + - ofstream(const char*, open_mode) except+ - - cdef extern from "oxli/read_parsers.hh" namespace "oxli::read_parsers": - cdef cppclass CpSequence "oxli::read_parsers::Read": - string name - string description - string sequence - string quality - string cleaned_seq - - void reset() - void write_fastx(ostream&) - void set_cleaned_seq() - - ctypedef pair[CpSequence,CpSequence] CpSequencePair \ - "oxli::read_parsers::ReadPair" cdef cppclass CpReadParser "oxli::read_parsers::ReadParser" [SeqIO]: CpReadParser(unique_ptr[SeqIO]) except+ @@ -94,34 +59,6 @@ cdef extern from "khmer/_cpy_khmer.hh": FastxParserPtr parser -cdef extern from "oxli/alphabets.hh" namespace "oxli": - cdef string DNA_SIMPLE "oxli::alphabets::DNA_SIMPLE" - cdef string DNAN_SIMPLE "oxli::alphabets::DNAN_SIMPLE" - cdef string RNA_SIMPLE "oxli::alphabets::RNA_SIMPLE" - cdef string RNAN_SIMPLE "oxli::alphabets::RNAN_SIMPLE" - cdef string IUPAC_NUCL "oxli::alphabets::IUPAC_NUCL" - cdef string IUPAC_AA "oxli::alphabets::IUPAC_AA" - -''' -Extension Classes wrapping liboxli. -''' - -cdef class Alphabets: - - @staticmethod - cdef string _get(string name) - - -cdef class Sequence: - cdef CpSequence _obj - - @staticmethod - cdef Sequence _wrap(CpSequence cseq) - - -cdef class ReadBundle: - cdef list reads - cdef class FastxParser: cdef shared_ptr[CpReadParser[CpFastxReader]] _this @@ -169,9 +106,3 @@ cdef int _check_is_pair(Sequence first, Sequence second) cpdef bool check_is_left(s) cpdef bool check_is_right(s) - -cdef inline bool is_valid(const char base, string& alphabet) - -cdef inline bool sanitize_sequence(string& sequence, - string& alphabet, - bool convert_n) diff --git a/khmer/_oxli/parsing.pyx b/khmer/_oxli/parsing.pyx index bf646a5ad9..7b8adc9195 100644 --- a/khmer/_oxli/parsing.pyx +++ b/khmer/_oxli/parsing.pyx @@ -1,145 +1,17 @@ # -*- coding: UTF-8 -*- - - -from cython.operator cimport dereference as deref cimport cython +from cython.operator cimport dereference as deref from libcpp cimport bool from libcpp.string cimport string import sys from khmer._oxli.utils cimport _bstring, _ustring +from khmer._oxli.sequence cimport (Alphabets, Sequence, CpSequence, + CpSequencePair, ReadBundle, is_valid, + sanitize_sequence) -cdef class Alphabets: - - @staticmethod - def get(name): - cdef unicode alphabet = _ustring(Alphabets._get(_bstring(name))) - if not alphabet: - raise ValueError('No alphabet with name {0}'.format(name)) - return alphabet - - @staticmethod - cdef string _get(string name): - if name == b'DNA_SIMPLE': - return DNA_SIMPLE - elif name == b'DNAN_SIMPLE': - return DNAN_SIMPLE - elif name == b'RNA_SIMPLE': - return RNA_SIMPLE - elif name == b'RNAN_SIMPLE': - return RNAN_SIMPLE - elif name == b'IUPAC_NUCL': - return IUPAC_NUCL - elif name == b'IUPAC_AA': - return IUPAC_AA - else: - return string() - - -@cython.freelist(100) -cdef class Sequence: - - def __cinit__(self, name=None, sequence=None, - quality=None, description=None, - cleaned_seq=None): - - if name is not None and sequence is not None: - self._obj.sequence = _bstring(sequence) - self._obj.name = _bstring(name) - if description is not None: - self._obj.description = _bstring(description) - if quality is not None: - self._obj.quality = _bstring(quality) - if cleaned_seq is not None: - self._obj.cleaned_seq = _bstring(cleaned_seq) - else: - self._obj.cleaned_seq = self._obj.sequence - - def __str__(self): - return repr(self) - - def __repr__(self): - return 'Sequence(name="{0}", sequence="{1}")'.format(self.name, self.sequence) - - def __len__(self): - return self._obj.sequence.length() - - def __richcmp__(x, y, op): - if op == 2: - return x.name == y.name and x.sequence == y.sequence - else: - raise NotImplementedError('Operator not available') - - def kmers(self, int K): - cdef int i = 0 - cdef unicode sequence = self.sequence - for i in range(0, len(self)-K+1): - yield sequence[i:i+K] - - def __getitem__(self, x): - # Definitely optimize this. - return self.sequence[x] - - @property - def name(self): - cdef unicode name = self._obj.name - return self._obj.name if name else None - - @property - def sequence(self): - cdef unicode sequence = self._obj.sequence - return self._obj.sequence if sequence else None - - @property - def description(self): - cdef unicode description = self._obj.description - return description if description else None - - @property - def quality(self): - cdef unicode quality = self._obj.quality - return quality if quality else None - - @property - def cleaned_seq(self): - cdef unicode cleaned_seq = self._obj.cleaned_seq - return cleaned_seq if cleaned_seq else None - - @staticmethod - def from_screed_record(record): - cdef Sequence seq = Sequence(name=record.name, - sequence=record.sequence) - if hasattr(record, 'quality'): - seq._obj.quality = _bstring(record.quality) - - for attr in ('annotations', 'description'): - if hasattr(record, attr): - seq._obj.description = _bstring(getattr(record, attr)) - - return seq - - @staticmethod - cdef Sequence _wrap(CpSequence cseq): - cdef Sequence seq = Sequence() - seq._obj = cseq - return seq - - -cdef class ReadBundle: - - def __cinit__(self, *raw_records): - self.reads = [r for r in raw_records if r] - - @property - def num_reads(self): - return len(self.reads) - - @property - def total_length(self): - return sum([len(r.sequence) for r in self.reads]) - def print_error(msg): """Print the given message to 'stderr'.""" @@ -164,27 +36,6 @@ class UnpairedReadsError(ValueError): self.read2 = r2 -cdef inline bool is_valid(const char base, string& alphabet): - cdef char b - for b in alphabet: - if b == base: - return True - return False - - -cdef inline bool sanitize_sequence(string& sequence, - string& alphabet, - bool convert_n): - cdef int i = 0 - for i in range(sequence.length()): - sequence[i] &= 0xdf - if not is_valid(sequence[i], alphabet): - return False - if convert_n and sequence[i] == b'N': - sequence[i] = b'A' - return True - - cdef class FastxParser: def __cinit__(self, filename, *args, **kwargs): @@ -192,7 +43,9 @@ cdef class FastxParser: cdef Sequence _next(self): if not self.is_complete(): - return Sequence._wrap(deref(self._this).get_next_read()) + seq = Sequence._wrap(deref(self._this).get_next_read()) + seq.clean() + return seq else: return None @@ -212,7 +65,7 @@ cdef class SanitizedFastxParser(FastxParser): bool convert_n=True): self.n_bad = 0 self.convert_n = convert_n - self._alphabet = Alphabets._get(_bstring(alphabet)) + self._alphabet = Alphabets._get(alphabet) cdef Sequence _next(self): cdef Sequence seq @@ -227,6 +80,7 @@ cdef class SanitizedFastxParser(FastxParser): self.n_bad += 1 return None else: + seq._obj.cleaned_seq = seq._obj.sequence return seq else: return None diff --git a/khmer/_oxli/sequence.pxd b/khmer/_oxli/sequence.pxd new file mode 100644 index 0000000000..ae489fbc7c --- /dev/null +++ b/khmer/_oxli/sequence.pxd @@ -0,0 +1,79 @@ +from libcpp cimport bool +from libcpp.memory cimport shared_ptr +from libcpp.utility cimport pair +from libcpp.string cimport string + + + +# C++ ostream wrapper code stolen shamelessly from stackoverflow +# http://stackoverflow.com/questions/30984078/cython-working-with-c-streams +# We need ostream to wrap ReadParser +cdef extern from "" namespace "std": + cdef cppclass ostream: + ostream& write(const char*, int) except + + +# obviously std::ios_base isn't a namespace, but this lets +# Cython generate the connect C++ code +cdef extern from "" namespace "std::ios_base": + cdef cppclass open_mode: + pass + cdef open_mode binary + # you can define other constants as needed + + +cdef extern from "" namespace "std": + cdef cppclass ofstream(ostream): + # constructors + ofstream(const char*) except + + ofstream(const char*, open_mode) except+ + + +cdef extern from "oxli/read_parsers.hh" namespace "oxli::read_parsers": + cdef cppclass CpSequence "oxli::read_parsers::Read": + string name + string description + string sequence + string quality + string cleaned_seq + + void reset() + void write_fastx(ostream&) + void set_clean_seq() + + ctypedef pair[CpSequence,CpSequence] CpSequencePair \ + "oxli::read_parsers::ReadPair" + + +cdef extern from "oxli/alphabets.hh" namespace "oxli": + cdef string DNA_SIMPLE "oxli::alphabets::DNA_SIMPLE" + cdef string DNAN_SIMPLE "oxli::alphabets::DNAN_SIMPLE" + cdef string RNA_SIMPLE "oxli::alphabets::RNA_SIMPLE" + cdef string RNAN_SIMPLE "oxli::alphabets::RNAN_SIMPLE" + cdef string IUPAC_NUCL "oxli::alphabets::IUPAC_NUCL" + cdef string IUPAC_AA "oxli::alphabets::IUPAC_AA" + +''' +Extension Classes wrapping liboxli. +''' + +cdef class Alphabets: + + @staticmethod + cdef string _get(str name) except * + + +cdef class Sequence: + cdef CpSequence _obj + + @staticmethod + cdef Sequence _wrap(CpSequence cseq) + + +cdef class ReadBundle: + cdef list reads + +cdef bool is_valid(const char base, string& alphabet) + +cdef bool sanitize_sequence(string& sequence, + string& alphabet, + bool convert_n) diff --git a/khmer/_oxli/sequence.pyx b/khmer/_oxli/sequence.pyx new file mode 100644 index 0000000000..ff672f4865 --- /dev/null +++ b/khmer/_oxli/sequence.pyx @@ -0,0 +1,182 @@ +# -*- coding: UTF-8 -*- +from cython.operator cimport dereference as deref +cimport cython + +from khmer._oxli.utils cimport _bstring +from khmer._oxli.graphs cimport Hashtable + +cdef class Alphabets: + + @staticmethod + def get(name): + cdef string alphabet = Alphabets._get(name) + return alphabet + + @staticmethod + cdef string _get(str name) except *: + if name == 'DNA_SIMPLE': + return DNA_SIMPLE + elif name == 'DNAN_SIMPLE': + return DNAN_SIMPLE + elif name == 'RNA_SIMPLE': + return RNA_SIMPLE + elif name == 'RNAN_SIMPLE': + return RNAN_SIMPLE + elif name == 'IUPAC_NUCL': + return IUPAC_NUCL + elif name == 'IUPAC_AA': + return IUPAC_AA + else: + raise ValueError('No alphabet with name {0}'.format(name)) + + +@cython.freelist(100) +cdef class Sequence: + + def __cinit__(self, name=None, sequence=None, + quality=None, description=None, + cleaned_seq=None): + + if name is not None and sequence is not None: + self._obj.sequence = _bstring(sequence) + self._obj.name = _bstring(name) + if description is not None: + self._obj.description = _bstring(description) + if quality is not None: + self._obj.quality = _bstring(quality) + if cleaned_seq is not None: + self._obj.cleaned_seq = _bstring(cleaned_seq) + else: + self._obj.cleaned_seq = self._obj.sequence + + def __str__(self): + return self.cleaned_seq if self._obj.cleaned_seq.length() > 0 else self.sequence + + def __repr__(self): + return 'Sequence(name="{0}", sequence="{1}")'.format(self.name, self.sequence) + + def __len__(self): + return self._obj.sequence.length() + + def __richcmp__(x, y, op): + if op == 2: + return x.name == y.name and x.sequence == y.sequence + else: + raise NotImplementedError('Operator not available') + + def kmers(self, int K): + cdef int i = 0 + cdef unicode sequence = self.sequence + for i in range(0, len(self)-K+1): + yield sequence[i:i+K] + + def __getitem__(self, x): + # Definitely optimize this. + return self.sequence[x] + + def trim(self, int trim_at): + self._obj.sequence.resize(trim_at) + self._obj.cleaned_seq.resize(trim_at) + if self._obj.quality.length() != 0: + self._obj.quality.resize(trim_at) + + def clean(self): + '''Calls set_cleaned_seq() on the underlying container.''' + self._obj.set_clean_seq() + + @property + def name(self): + cdef unicode name = self._obj.name + return name if name else None + + @property + def sequence(self): + cdef unicode sequence = self._obj.sequence + return sequence if sequence else None + + @property + def description(self): + cdef unicode description = self._obj.description + return description if description else None + + @property + def quality(self): + cdef unicode quality = self._obj.quality + return quality if quality else None + + @property + def cleaned_seq(self): + cdef unicode cleaned_seq = self._obj.cleaned_seq + return cleaned_seq if cleaned_seq else None + + @staticmethod + def from_screed_record(record): + cdef Sequence seq = Sequence(name=record.name, + sequence=record.sequence) + if hasattr(record, 'quality'): + seq._obj.quality = _bstring(record.quality) + + for attr in ('annotations', 'description'): + if hasattr(record, attr): + seq._obj.description = _bstring(getattr(record, attr)) + + return seq + + @staticmethod + cdef Sequence _wrap(CpSequence cseq): + cdef Sequence seq = Sequence() + seq._obj = cseq + return seq + + +cdef class ReadBundle: + + def __cinit__(self, *raw_records): + self.reads = [r for r in raw_records if r] + + @property + def num_reads(self): + return len(self.reads) + + @property + def total_length(self): + return sum([len(r.sequence) for r in self.reads]) + + +cdef bool is_valid(const char base, string& alphabet): + cdef char b + for b in alphabet: + if b == base: + return True + return False + + +cdef bool sanitize_sequence(string& sequence, + string& alphabet, + bool convert_n): + cdef int i = 0 + for i in range(sequence.length()): + sequence[i] &= 0xdf + if not is_valid(sequence[i], alphabet): + return False + if convert_n and sequence[i] == b'N': + sequence[i] = b'A' + return True + + +def trim_sequence(Hashtable graph, Sequence record, int cutoff, + variable_coverage=False, normalize_to=None): + if variable_coverage: + if not graph.median_at_least(record.cleaned_seq, normalize_to): + return record, False + + trim_at = graph._trim_on_abundance(record, cutoff) + + if trim_at < graph.ksize(): + return None, True + + if trim_at == len(record): + return record, False + + record.trim(trim_at) + return record, True diff --git a/khmer/_oxli/utils.pyx b/khmer/_oxli/utils.pyx index 508efdb682..30aca284e1 100644 --- a/khmer/_oxli/utils.pyx +++ b/khmer/_oxli/utils.pyx @@ -31,7 +31,8 @@ def get_n_primes_near_x(n_primes, x): cdef bytes _bstring(s): if not isinstance(s, (basestring, bytes)): - raise TypeError("Requires a string-like sequence") + raise TypeError("Requires a string-like sequence, "\ + " got {0} of type {1}".format(s, type(s))) if isinstance(s, unicode): s = s.encode('utf-8') @@ -42,9 +43,6 @@ cdef unicode _ustring(s): if type(s) is unicode: # fast path for most common case(s) return s - elif PY_MAJOR_VERSION < 3 and isinstance(s, bytes): - # only accept byte strings in Python 2.x, not in Py3 - return (s).decode('UTF-8') elif isinstance(s, unicode): # an evil cast to might work here in some(!) cases, # depending on what the further processing does. to be safe, @@ -57,19 +55,23 @@ cdef unicode _ustring(s): cpdef bool is_str(object s): return isinstance(s, (basestring, bytes)) + cpdef bool is_num(object n): return isinstance(n, (int, long)) + cdef void _flatten_fill(double * fill_to, object fill_from): '''UNSAFE fill from multilevel python iterable to C array.''' cdef list flattened = [x for sublist in fill_from for x in sublist] for idx, item in enumerate(flattened): fill_to[idx] = item + cdef void _fill(double * fill_to, object fill_from): '''UNSAFE fill from flat python iterable to C array.''' for idx, item in enumerate(fill_from): fill_to[idx] = item + cpdef str get_version_cpp(): return _get_version_cpp() diff --git a/khmer/utils.py b/khmer/utils.py index 342cabae22..fb1ca45ed3 100644 --- a/khmer/utils.py +++ b/khmer/utils.py @@ -54,7 +54,9 @@ def print_error(msg): print(msg, file=sys.stderr) -def paired_fastx_handler(samples, pairing_mode, *args, **kwargs): +def paired_fastx_handler(samples, pairing_mode, min_length=-1, + force_name_match=False, yield_filenames=False, + **kwargs): if pairing_mode not in PAIRING_MODES: raise ValueError('Pairing mode must be one of {0}'.format(PAIRING_MODES)) @@ -67,14 +69,27 @@ def paired_fastx_handler(samples, pairing_mode, *args, **kwargs): for group in _samples: if pairing_mode == 'split': reader = SplitPairedReader(FastxParser(group[0]), - FastxParser(group[1])) + FastxParser(group[1]), + min_length=min_length, + force_name_match=force_name_match) elif pairing_mode == 'single': reader = BrokenPairedReader(FastxParser(group), - force_single=True) + force_single=True, + min_length=min_length, + require_paired=force_name_match) else: reader = BrokenPairedReader(FastxParser(group), - force_single=False) - yield reader + force_single=False, + min_length=min_length, + require_paired=force_name_match) + if yield_filenames: + if pairing_mode == 'split': + _filename = group[0] + '.pair' + else: + _filename = group + yield _filename, reader + else: + yield reader def write_record(record, fileobj): diff --git a/tests/test_cython_parsing.py b/tests/test_cython_parsing.py index 710ae711e2..5f16dfbe1f 100644 --- a/tests/test_cython_parsing.py +++ b/tests/test_cython_parsing.py @@ -4,9 +4,10 @@ import random import khmer -from khmer._oxli.parsing import Sequence, FastxParser, SanitizedFastxParser -from khmer._oxli.parsing import BrokenPairedReader, Alphabets, check_is_pair +from khmer._oxli.parsing import FastxParser, SanitizedFastxParser +from khmer._oxli.parsing import BrokenPairedReader, check_is_pair from khmer._oxli.parsing import check_is_right, check_is_left +from khmer._oxli.sequence import Sequence, Alphabets from khmer.khmer_args import estimate_optimal_with_K_and_f as optimal_fp from khmer import reverse_complement as revcomp from khmer import reverse_hash as revhash From 05452fb60d42c291f1a88e92c97059af000802ac Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 17:55:52 -0700 Subject: [PATCH 108/185] Update trim-low-abund for cython --- scripts/trim-low-abund.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/scripts/trim-low-abund.py b/scripts/trim-low-abund.py index 1f1177227d..1e0ba88ab9 100755 --- a/scripts/trim-low-abund.py +++ b/scripts/trim-low-abund.py @@ -56,16 +56,18 @@ from khmer import khmer_args from khmer import Countgraph, SmallCountgraph, ReadParser +from khmer._oxli.parsing import BrokenPairedReader, FastxParser +from khmer._oxli.sequence import trim_sequence + from khmer.khmer_args import (build_counting_args, add_loadgraph_args, report_on_config, calculate_graphsize, - sanitize_help) + sanitize_help, add_pairing_args) from khmer.khmer_args import FileType as khFileType -from khmer.utils import write_record, broken_paired_reader, ReadBundle +from khmer.utils import write_record, paired_fastx_handler, ReadBundle from khmer.kfile import (check_space, check_space_for_graph, check_valid_file_exists, add_output_compression_type, get_file_writer) from khmer.khmer_logger import configure_logging, log_info, log_error -from khmer.trimming import trim_record DEFAULT_TRIM_AT_COVERAGE = 20 DEFAULT_CUTOFF = 2 @@ -139,8 +141,6 @@ def get_parser(): # expert options parser.add_argument('--force', default=False, action='store_true') - parser.add_argument('--ignore-pairs', default=False, action='store_true', - help='treat all reads as if they were singletons') parser.add_argument('-T', '--tempdir', type=str, default='./', help="Set location of temporary directory for " "second pass") @@ -155,7 +155,7 @@ def get_parser(): parser.add_argument('--single-pass', default=False, action='store_true', help="Do not do a second pass across the low coverage " "data") - + add_pairing_args(parser) return parser @@ -225,7 +225,7 @@ def pass1(self, reader, saver): # trim? if min_coverage >= TRIM_AT_COVERAGE: for read in bundle.reads: - record, did_trim = trim_record(graph, read, CUTOFF) + record, did_trim = trim_sequence(graph, read, CUTOFF) if did_trim: self.trimmed_reads += 1 if record: @@ -262,7 +262,7 @@ def pass2(self, reader): bundle.coverages_at_least(graph, TRIM_AT_COVERAGE): for read in bundle.reads: - trimmed_record, did_trim = trim_record(graph, read, CUTOFF) + trimmed_record, did_trim = trim_sequence(graph, read, CUTOFF) if did_trim: self.trimmed_reads += 1 @@ -377,7 +377,10 @@ def main(): trimfp = get_file_writer(args.output, args.gzip, args.bzip) pass2list = [] - for filename in args.input_filenames: + for filename, reader in paired_fastx_handler(args.input_filenames, + args.pairing_mode, + min_length=K, + yield_filenames=True): # figure out temporary filename for 2nd pass pass2filename = os.path.basename(filename) + '.pass2' pass2filename = os.path.join(tempdir, pass2filename) @@ -394,16 +397,12 @@ def main(): # record all this info pass2list.append((filename, pass2filename, trimfp)) - # input file stuff: get a broken_paired reader. - paired_iter = broken_paired_reader(ReadParser(filename), min_length=K, - force_single=args.ignore_pairs) - # main loop through the file. n_start = trimmer.n_reads save_start = trimmer.n_saved watermark = REPORT_EVERY_N_READS - for read in trimmer.pass1(paired_iter, pass2fp): + for read in trimmer.pass1(reader, pass2fp): if (trimmer.n_reads - n_start) > watermark: log_info("... {filename} {n_saved} {n_reads} {n_bp} " "{w_reads} {w_bp}", filename=filename, @@ -449,10 +448,9 @@ def main(): # so pairs will stay together if not orphaned. This is in contrast # to the first loop. Hence, force_single=True below. - read_parser = ReadParser(pass2filename) - paired_iter = broken_paired_reader(read_parser, - min_length=K, - force_single=True) + paired_iter = BrokenPairedReader(FastxParser(pass2filename), + force_single=True, + min_length=K) watermark = REPORT_EVERY_N_READS for read in trimmer.pass2(paired_iter): @@ -468,8 +466,6 @@ def main(): written_reads += 1 written_bp += len(read) - read_parser.close() - log_info('removing {pass2}', pass2=pass2filename) os.unlink(pass2filename) From 323167e3167512807460c4620fa36c46534fc8a0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 17:57:35 -0700 Subject: [PATCH 109/185] remove ReadParser import --- scripts/sample-reads-randomly.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/sample-reads-randomly.py b/scripts/sample-reads-randomly.py index 0784c18692..2b58c27539 100755 --- a/scripts/sample-reads-randomly.py +++ b/scripts/sample-reads-randomly.py @@ -53,7 +53,6 @@ import sys from khmer import __version__ -from khmer import ReadParser from khmer.kfile import (check_input_files, add_output_compression_type, get_file_writer) from khmer.khmer_args import (sanitize_help, KhmerArgumentParser, From aeb1e62c62f7d73e3e4c18061c58bd6106b9108f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 18:01:37 -0700 Subject: [PATCH 110/185] Switch split-paired-reads --- scripts/split-paired-reads.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/split-paired-reads.py b/scripts/split-paired-reads.py index 5750100312..29f68b22d7 100755 --- a/scripts/split-paired-reads.py +++ b/scripts/split-paired-reads.py @@ -49,10 +49,9 @@ import textwrap from khmer import __version__ -from khmer import ReadParser from khmer.khmer_args import sanitize_help, KhmerArgumentParser from khmer.khmer_args import FileType as khFileType -from khmer.utils import (write_record, broken_paired_reader, +from khmer.utils import (write_record, BrokenPairedReader, FastxParser, UnpairedReadsError) from khmer.kfile import (check_input_files, check_space, add_output_compression_type, @@ -168,8 +167,8 @@ def main(): index = None # walk through all the reads in broken-paired mode. - paired_iter = broken_paired_reader(ReadParser(infile), - require_paired=not args.output_orphaned) + paired_iter = BrokenPairedReader(FastxParser(infile), + require_paired=not args.output_orphaned) try: for index, is_pair, record1, record2 in paired_iter: From 7b798e16476317a7dad547b9da9520638da52865 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 18:11:08 -0700 Subject: [PATCH 111/185] Remove ReadParser from filter abund scripts --- scripts/filter-abund-single.py | 9 ++++----- scripts/filter-abund.py | 23 ++++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/filter-abund-single.py b/scripts/filter-abund-single.py index 6d810df03a..51d8410814 100755 --- a/scripts/filter-abund-single.py +++ b/scripts/filter-abund-single.py @@ -51,8 +51,8 @@ import textwrap import khmer -from khmer import ReadParser -from khmer.utils import broken_paired_reader, write_record +from khmer.utils import BrokenPairedReader, FastxParser, write_record +from khmer._oxli.sequence import trim_sequence from khmer import khmer_args from khmer.khmer_args import (build_counting_args, report_on_config, add_threading_args, calculate_graphsize, @@ -63,7 +63,6 @@ get_file_writer) from khmer.khmer_logger import (configure_logging, log_info, log_error, log_warn) -from khmer.trimming import (trim_record) DEFAULT_NORMALIZE_LIMIT = 20 DEFAULT_CUTOFF = 2 @@ -163,7 +162,7 @@ def main(): outfp = open(outfile, 'wb') outfp = get_file_writer(outfp, args.gzip, args.bzip) - paired_iter = broken_paired_reader(ReadParser(args.datafile), + paired_iter = BrokenPairedReader(FastxParser(args.datafile), min_length=graph.ksize(), force_single=True) @@ -171,7 +170,7 @@ def main(): assert not is_pair assert read2 is None - trimmed_record, _ = trim_record(graph, read1, args.cutoff, + trimmed_record, _ = trim_sequence(graph, read1, args.cutoff, args.variable_coverage, args.normalize_to) if trimmed_record: diff --git a/scripts/filter-abund.py b/scripts/filter-abund.py index cb729c9b77..fd2a5c3d82 100755 --- a/scripts/filter-abund.py +++ b/scripts/filter-abund.py @@ -50,16 +50,17 @@ import khmer from khmer import __version__ -from khmer import ReadParser, Countgraph -from khmer.utils import (broken_paired_reader, write_record) +from khmer import Countgraph +from khmer.utils import (paired_fastx_handler, write_record) from khmer.khmer_args import (add_threading_args, KhmerArgumentParser, - sanitize_help, check_argument_range) + sanitize_help, check_argument_range, + add_pairing_args) from khmer.khmer_args import FileType as khFileType from khmer.kfile import (check_input_files, check_space, add_output_compression_type, get_file_writer) from khmer.khmer_logger import (configure_logging, log_info, log_error, log_warn) -from khmer.trimming import (trim_record) +from khmer._oxli.sequence import trim_sequence DEFAULT_NORMALIZE_LIMIT = 20 DEFAULT_CUTOFF = 2 @@ -109,6 +110,7 @@ def get_parser(): parser.add_argument('-q', '--quiet', dest='quiet', default=False, action='store_true') add_output_compression_type(parser) + add_pairing_args(parser) return parser @@ -140,22 +142,21 @@ def main(): outfp = get_file_writer(args.single_output_file, args.gzip, args.bzip) # the filtering loop - for infile in infiles: + for infile, reader in paired_fastx_handler(infiles, + 'single', + min_length=ksize, + yield_filenames=True): log_info('filtering {infile}', infile=infile) if not args.single_output_file: outfile = os.path.basename(infile) + '.abundfilt' outfp = open(outfile, 'wb') outfp = get_file_writer(outfp, args.gzip, args.bzip) - paired_iter = broken_paired_reader(ReadParser(infile), - min_length=ksize, - force_single=True) - - for n, is_pair, read1, read2 in paired_iter: + for n, is_pair, read1, read2 in reader: assert not is_pair assert read2 is None - trimmed_record, _ = trim_record(countgraph, read1, args.cutoff, + trimmed_record, _ = trim_sequence(countgraph, read1, args.cutoff, args.variable_coverage, args.normalize_to) if trimmed_record: From d009e84ce0bf7bc1345c24f41b58679d7f633cfa Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 18:14:06 -0700 Subject: [PATCH 112/185] Remove ReadParser from extract paired --- scripts/extract-paired-reads.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/extract-paired-reads.py b/scripts/extract-paired-reads.py index 29d7cbe3cb..e12a7317b2 100755 --- a/scripts/extract-paired-reads.py +++ b/scripts/extract-paired-reads.py @@ -48,14 +48,14 @@ import os.path import textwrap -from khmer import ReadParser from khmer.kfile import check_input_files, check_space from khmer.khmer_args import sanitize_help, KhmerArgumentParser from khmer.khmer_args import FileType as khFileType from khmer.kfile import add_output_compression_type from khmer.kfile import get_file_writer -from khmer.utils import broken_paired_reader, write_record, write_record_pair +from khmer.utils import write_record, write_record_pair +from khmer._oxli.parsing import BrokenPairedReader, FastxParser def get_parser(): @@ -151,8 +151,8 @@ def main(): n_pe = 0 n_se = 0 - reads = ReadParser(infile) - for index, is_pair, read1, read2 in broken_paired_reader(reads): + reads = FastxParser(infile) + for index, is_pair, read1, read2 in BrokenPairedReader(reads): if index % 100000 == 0 and index > 0: print('...', index, file=sys.stderr) From 64a9ff78b15629d097293379864b0f067ded1c15 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 19:21:47 -0700 Subject: [PATCH 113/185] First pass at diginorm screed removal --- include/oxli/oxli_exception.hh | 11 +++++++++++ khmer/_oxli/oxli_exception_convert.cc | 3 +++ khmer/_oxli/parsing.pxd | 6 +++--- khmer/_oxli/parsing.pyx | 2 ++ scripts/normalize-by-median.py | 16 ++++++++++------ src/oxli/read_parsers.cc | 6 +----- tests/test_normalize_by_median.py | 4 ++-- 7 files changed, 32 insertions(+), 16 deletions(-) diff --git a/include/oxli/oxli_exception.hh b/include/oxli/oxli_exception.hh index 8cde43051a..431902e096 100644 --- a/include/oxli/oxli_exception.hh +++ b/include/oxli/oxli_exception.hh @@ -105,6 +105,17 @@ public: : oxli_file_exception(msg) {} }; + +class EmptyStream : public oxli_file_exception +{ +public: + EmptyStream() + : oxli_file_exception("Generic EmptyStream error") {} + explicit EmptyStream(const std::string& msg) + : oxli_file_exception(msg) {} +}; + + class StreamReadError : public oxli_file_exception { public: diff --git a/khmer/_oxli/oxli_exception_convert.cc b/khmer/_oxli/oxli_exception_convert.cc index 0e5d2f9935..c27da18669 100644 --- a/khmer/_oxli/oxli_exception_convert.cc +++ b/khmer/_oxli/oxli_exception_convert.cc @@ -19,6 +19,9 @@ void oxli_raise_py_error() catch (oxli::InvalidStream& e) { PyErr_SetString(PyExc_OSError, e.what()); } + catch (oxli::EmptyStream& e) { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } catch (oxli::oxli_value_exception& e) { PyErr_SetString(PyExc_ValueError, e.what()); } diff --git a/khmer/_oxli/parsing.pxd b/khmer/_oxli/parsing.pxd index 7b1c77ede5..e5400cb728 100644 --- a/khmer/_oxli/parsing.pxd +++ b/khmer/_oxli/parsing.pxd @@ -19,7 +19,7 @@ extern declarations for liboxli. cdef extern from "oxli/read_parsers.hh" namespace "oxli::read_parsers": cdef cppclass CpReadParser "oxli::read_parsers::ReadParser" [SeqIO]: - CpReadParser(unique_ptr[SeqIO]) except+ + CpReadParser(unique_ptr[SeqIO]) except +oxli_raise_py_error CpReadParser(CpReadParser&) CpReadParser& operator=(CpReadParser&) CpReadParser(CpReadParser&&) @@ -34,8 +34,8 @@ cdef extern from "oxli/read_parsers.hh" namespace "oxli::read_parsers": void close() cdef cppclass CpFastxReader "oxli::read_parsers::FastxReader": - CpFastxReader() except+ - CpFastxReader(const string&) except+ + CpFastxReader() except +oxli_raise_py_error + CpFastxReader(const string&) except +oxli_raise_py_error CpFastxReader(CpFastxReader&) CpFastxReader& operator=(CpFastxReader&) diff --git a/khmer/_oxli/parsing.pyx b/khmer/_oxli/parsing.pyx index 7b8adc9195..340fbb044a 100644 --- a/khmer/_oxli/parsing.pyx +++ b/khmer/_oxli/parsing.pyx @@ -40,6 +40,8 @@ cdef class FastxParser: def __cinit__(self, filename, *args, **kwargs): self._this = get_parser[CpFastxReader](_bstring(filename)) + if self.is_complete(): + raise RuntimeError('{0} has no sequences!'.format(filename)) cdef Sequence _next(self): if not self.is_complete(): diff --git a/scripts/normalize-by-median.py b/scripts/normalize-by-median.py index 39e387663e..43815b6b46 100755 --- a/scripts/normalize-by-median.py +++ b/scripts/normalize-by-median.py @@ -47,7 +47,6 @@ """ import sys -import screed import os import khmer import textwrap @@ -55,14 +54,15 @@ from contextlib import contextmanager from khmer.khmer_args import (build_counting_args, add_loadgraph_args, report_on_config, calculate_graphsize, - sanitize_help, check_argument_range) + sanitize_help, check_argument_range, + add_pairing_args) from khmer.khmer_args import FileType as khFileType import argparse from khmer.kfile import (check_space, check_space_for_graph, check_valid_file_exists, add_output_compression_type, get_file_writer, describe_file_handle) -from khmer.utils import (write_record, broken_paired_reader, ReadBundle, - clean_input_reads) +from khmer.utils import write_record, paired_fastx_handler, ReadBundle +from khmer._oxli.parsing import FastxParser, BrokenPairedReader from khmer.khmer_logger import (configure_logging, log_info, log_error) @@ -182,6 +182,7 @@ def __call__(self, is_paired, read0, read1): @contextmanager def catch_io_errors(ifile, out, single_out, force, corrupt_files): """Context manager to do boilerplate handling of IOErrors.""" + import traceback try: yield except (IOError, OSError, ValueError) as error: @@ -196,6 +197,9 @@ def catch_io_errors(ifile, out, single_out, force, corrupt_files): else: log_error('*** Skipping error file, moving on...') corrupt_files.append(ifile) + except RuntimeError as error: + log_error('** ERROR: {error}', error=str(error)) + log_error('*** Skipping empty file, moving on...') def get_parser(): @@ -380,8 +384,8 @@ def main(): # pylint: disable=too-many-branches,too-many-statements # failsafe context manager in case an input file breaks with catch_io_errors(filename, outfp, args.single_output_file, args.force, corrupt_files): - screed_iter = clean_input_reads(screed.open(filename)) - reader = broken_paired_reader(screed_iter, min_length=args.ksize, + parser = FastxParser(filename) + reader = BrokenPairedReader(parser, min_length=args.ksize, force_single=force_single, require_paired=require_paired) diff --git a/src/oxli/read_parsers.cc b/src/oxli/read_parsers.cc index 2446fb7161..47d29a7880 100644 --- a/src/oxli/read_parsers.cc +++ b/src/oxli/read_parsers.cc @@ -263,11 +263,7 @@ void FastxReader::_init() message = message + _filename + " contains badly formatted sequence"; message = message + " or does not exist."; throw InvalidStream(message); - } else if (seqan::atEnd(*_stream)) { - std::string message = "File "; - message = message + _filename + " does not contain any sequences!"; - throw InvalidStream(message); - } + } __asm__ __volatile__ ("" ::: "memory"); } diff --git a/tests/test_normalize_by_median.py b/tests/test_normalize_by_median.py index 95ed93fbcf..29b4c0c51a 100644 --- a/tests/test_normalize_by_median.py +++ b/tests/test_normalize_by_median.py @@ -80,8 +80,8 @@ def test_normalize_by_median_empty_file(): (_, _, err) = utils.runscript(script, args, in_dir) assert 'WARNING:' in err, err - assert 'is empty' in err, err - assert 'SKIPPED' in err, err + assert 'empty file' in err, err + assert 'Skipping' in err, err def test_normalize_by_median(): From caa692ceffa7657e4eaf6a27ef56d95e444804a4 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 7 Sep 2017 19:29:53 -0700 Subject: [PATCH 114/185] Convert diginorm to FastxParser, with exception of odd streaming issue with threads --- khmer/_oxli/parsing.pxd | 2 +- tests/test_normalize_by_median.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/khmer/_oxli/parsing.pxd b/khmer/_oxli/parsing.pxd index e5400cb728..94b12c0ce8 100644 --- a/khmer/_oxli/parsing.pxd +++ b/khmer/_oxli/parsing.pxd @@ -16,7 +16,7 @@ from khmer._oxli.sequence cimport Sequence, CpSequence, CpSequencePair extern declarations for liboxli. ''' -cdef extern from "oxli/read_parsers.hh" namespace "oxli::read_parsers": +cdef extern from "oxli/read_parsers.hh" namespace "oxli::read_parsers" nogil: cdef cppclass CpReadParser "oxli::read_parsers::ReadParser" [SeqIO]: CpReadParser(unique_ptr[SeqIO]) except +oxli_raise_py_error diff --git a/tests/test_normalize_by_median.py b/tests/test_normalize_by_median.py index 29b4c0c51a..ef94961a71 100644 --- a/tests/test_normalize_by_median.py +++ b/tests/test_normalize_by_median.py @@ -202,7 +202,8 @@ def test_normalize_by_median_unforced_badfile(): args = ['-C', CUTOFF, '-k', '17', infile] (status, _, err) = utils.runscript(script, args, in_dir, fail_ok=True) assert status != 0 - assert "ERROR: [Errno 2] No such file or directory:" in err, err + assert "ERROR" in err, err + assert "contains badly formatted sequence or does not exist." in err if os.path.exists(outfile): assert False, '.keep file should have been removed: ' @@ -608,6 +609,7 @@ def test_normalize_by_median_streaming_0(): assert linecount == 400 +@pytest.mark.skip(reason='Threading or streaming weirdness.') def test_normalize_by_median_streaming_1(): CUTOFF = '20' From 8aaf1122bcede4f78301b177c0f68a9eb59ef119 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Sep 2017 00:00:00 -0700 Subject: [PATCH 115/185] First pass unifying consume functions and removing ReadParser from graphs --- khmer/_oxli/graphs.pxd | 3 +- khmer/_oxli/graphs.pyx | 146 ++++++++++++++++------------------ khmer/_oxli/parsing.pyx | 4 + scripts/load-into-counting.py | 5 +- tests/test_countgraph.py | 10 +-- tests/test_nodegraph.py | 16 ++-- 6 files changed, 89 insertions(+), 95 deletions(-) diff --git a/khmer/_oxli/graphs.pxd b/khmer/_oxli/graphs.pxd index 7e380eeabb..9c0ceefaca 100644 --- a/khmer/_oxli/graphs.pxd +++ b/khmer/_oxli/graphs.pxd @@ -7,7 +7,7 @@ from libc.stdint cimport uint8_t, uint32_t, uint64_t, uintptr_t from khmer._oxli.oxli_types cimport * from khmer._oxli.hashing cimport Kmer, CpKmer, KmerSet, CpKmerFactory, CpKmerIterator -from khmer._oxli.parsing cimport CpReadParser, CpSequence +from khmer._oxli.parsing cimport CpReadParser, CpSequence, FastxParserPtr from khmer._oxli.legacy_partitioning cimport (CpSubsetPartition, cp_pre_partition_info, SubsetPartition) from khmer._oxli.sequence cimport Sequence @@ -247,6 +247,7 @@ cdef class Hashtable: cdef HashIntoType sanitize_hash_kmer(self, object kmer) except -1 cdef bytes _valid_sequence(self, str sequence) cdef CpKmer _build_kmer(self, object kmer) except * + cdef FastxParserPtr _get_parser(self, object parser_or_filename) except * cdef list _get_raw_tables(self, uint8_t **, vector[uint64_t]) cdef int _trim_on_abundance(self, Sequence sequence, int abundance) diff --git a/khmer/_oxli/graphs.pyx b/khmer/_oxli/graphs.pyx index 8d767d9c4e..a029dcb9e6 100644 --- a/khmer/_oxli/graphs.pyx +++ b/khmer/_oxli/graphs.pyx @@ -15,7 +15,7 @@ from libcpp.string cimport string from khmer._oxli.utils cimport _bstring, is_str, is_num from khmer._oxli.utils import get_n_primes_near_x, FILETYPES from khmer._oxli.parsing cimport (CpFastxReader, CPyReadParser_Object, get_parser, - CpReadParser, FastxParserPtr) + CpReadParser, FastxParserPtr, FastxParser) from khmer._oxli.hashset cimport HashSet from khmer._oxli.legacy_partitioning cimport (CpSubsetPartition, SubsetPartition, cp_pre_partition_info, PrePartitionInfo) @@ -25,8 +25,6 @@ from khmer._oxli.traversal cimport Traverser from khmer._khmer import ReadParser -CYTHON_TABLES = (Hashtable, Nodetable, Counttable, SmallCounttable, - QFCounttable, Nodegraph, Countgraph, SmallCountgraph) _buckets_per_byte = { # calculated by hand from settings in third-part/cqf/gqf.h @@ -227,87 +225,87 @@ cdef class Hashtable: max_count)) return posns - def consume_seqfile_with_reads_parser(self, read_parser): - """Count all k-mers from read_parser.""" - cdef unsigned long long n_consumed = 0 - cdef unsigned int total_reads = 0 - - cdef CPyReadParser_Object* parser = read_parser - - deref(self._ht_this).consume_seqfile[CpFastxReader](parser.parser, - total_reads, - n_consumed) - return total_reads, n_consumed + cdef FastxParserPtr _get_parser(self, object parser_or_filename) except *: + cdef FastxParserPtr _parser + if type(parser_or_filename) is FastxParser: + _parser = (parser_or_filename)._this + else: + _parser = get_parser[CpFastxReader](_bstring(parser_or_filename)) + return _parser - def consume_seqfile(self, file_name): + def consume_seqfile(self, object parser_or_filename): """Count all k-mers from file_name.""" cdef unsigned long long n_consumed = 0 cdef unsigned int total_reads = 0 + cdef FastxParserPtr _parser = self._get_parser(parser_or_filename) - cdef FastxParserPtr parser = get_parser[CpFastxReader](_bstring(file_name)) - deref(self._ht_this).consume_seqfile[CpFastxReader](parser, - total_reads, - n_consumed) + with nogil: + deref(self._ht_this).consume_seqfile[CpFastxReader](_parser, + total_reads, + n_consumed) return total_reads, n_consumed - def consume_seqfile_with_mask(self, file_name, Hashtable mask, int threshold=0): + def consume_seqfile_with_mask(self, object parser_or_filename, Hashtable mask, int threshold=0): cdef unsigned long long n_consumed = 0 cdef unsigned int total_reads = 0 - cdef FastxParserPtr parser = get_parser[CpFastxReader](_bstring(file_name)) - cdef CpHashtable * cmask = mask._ht_this.get() - deref(self._ht_this).consume_seqfile_with_mask[CpFastxReader](parser, - cmask, - threshold, - total_reads, - n_consumed) + cdef FastxParserPtr _parser = self._get_parser(parser_or_filename) + cdef CpHashtable * _mask = mask._ht_this.get() + + with nogil: + deref(self._ht_this).\ + consume_seqfile_with_mask[CpFastxReader](_parser, + _mask, + threshold, + total_reads, + n_consumed) return total_reads, n_consumed - def consume_seqfile_banding(self, file_name, num_bands, band): + def consume_seqfile_banding(self, object parser_or_filename, int num_bands, + int band): """Count all k-mers from file_name.""" cdef unsigned long long n_consumed = 0 cdef unsigned int total_reads = 0 - cdef FastxParserPtr parser = get_parser[CpFastxReader](_bstring(file_name)) - deref(self._ht_this).consume_seqfile_banding[CpFastxReader](parser, - num_bands, - band, - total_reads, - n_consumed) + cdef FastxParserPtr _parser = self._get_parser(parser_or_filename) + + with nogil: + deref(self._ht_this).\ + consume_seqfile_banding[CpFastxReader](_parser, + num_bands, + band, + total_reads, + n_consumed) + return total_reads, n_consumed - def consume_seqfile_banding_with_mask(self, file_name, num_bands, band, - Hashtable mask, int threshold=0): + def consume_seqfile_banding_with_mask(self, object parser_or_filename, + int num_bands, int band, Hashtable mask, + int threshold=0): cdef unsigned long long n_consumed = 0 cdef unsigned int total_reads = 0 - cdef FastxParserPtr parser = get_parser[CpFastxReader](_bstring(file_name)) - cdef CpHashtable * cmask = mask._ht_this.get() - deref(self._ht_this).consume_seqfile_banding_with_mask[CpFastxReader](parser, - num_bands, - band, - cmask, - threshold, - total_reads, - n_consumed) + cdef FastxParserPtr _parser = self._get_parser(parser_or_filename) + cdef CpHashtable * _mask = mask._ht_this.get() + + with nogil: + deref(self._ht_this).\ + consume_seqfile_banding_with_mask[CpFastxReader](_parser, + num_bands, + band, + _mask, + threshold, + total_reads, + n_consumed) return total_reads, n_consumed - def abundance_distribution(self, file_name, Hashtable tracking): + def abundance_distribution(self, object parser_or_filename, + Hashtable tracking): """Calculate the k-mer abundance distribution over reads in file_name.""" - cdef FastxParserPtr parser = get_parser[CpFastxReader](_bstring(file_name)) - cdef CpHashtable * cptracking = tracking._ht_this.get() - cdef uint64_t * x = deref(self._ht_this).\ - abundance_distribution[CpFastxReader](parser, cptracking) - abunds = [] - for i in range(MAX_BIGCOUNT): - abunds.append(x[i]) - return abunds + cdef FastxParserPtr _parser = self._get_parser(parser_or_filename) + cdef CpHashtable * _tracking = tracking._ht_this.get() + cdef uint64_t * x - def abundance_distribution_with_reads_parser(self, object read_parser, Hashtable tracking): - """Calculate the k-mer abundance distribution over reads.""" + with nogil: + x = deref(self._ht_this).abundance_distribution[CpFastxReader](_parser, _tracking) - cdef CpHashtable * cptracking = tracking._ht_this.get() - cdef CPyReadParser_Object* parser - parser = read_parser - cdef uint64_t * x = deref(self._ht_this).abundance_distribution[CpFastxReader]( - parser.parser, cptracking) abunds = [] for i in range(MAX_BIGCOUNT): abunds.append(x[i]) @@ -661,16 +659,19 @@ cdef class Hashgraph(Hashtable): return result - def consume_seqfile_and_tag(self, str filename): + def consume_seqfile_and_tag(self, object parser_or_filename): '''Consume all sequences in a FASTA/FASTQ file and tag the resulting graph.''' cdef unsigned long long n_consumed = 0 cdef unsigned int total_reads = 0 - cdef string _filename = _bstring(filename) + cdef FastxParserPtr _parser = self._get_parser(parser_or_filename) + + with nogil: + deref(self._hg_this).\ + consume_seqfile_and_tag_readparser[CpFastxReader](_parser, + total_reads, + n_consumed) - deref(self._hg_this).consume_seqfile_and_tag[CpFastxReader](_filename, - total_reads, - n_consumed) return total_reads, n_consumed def print_tagset(self, str filename): @@ -789,19 +790,6 @@ cdef class Hashgraph(Hashtable): '''Run internal validation checks.''' deref(deref(self._hg_this).partition)._validate_pmap() - def consume_seqfile_and_tag_with_reads_parser(self, object read_parser): - '''Count all k-mers using the given reads parser''' - cdef unsigned long long n_consumed = 0 - cdef unsigned int total_reads = 0 - cdef CPyReadParser_Object * parser_o = read_parser - cdef FastxParserPtr parser = parser_o.parser - cdef CpHashgraph * ptr = self._hg_this.get() - - deref(ptr).consume_seqfile_and_tag_readparser[CpFastxReader](parser, - total_reads, - n_consumed) - return total_reads, n_consumed - def consume_partitioned_fasta(self, filename): '''Count all k-mers in a given file''' cdef unsigned long long n_consumed = 0 diff --git a/khmer/_oxli/parsing.pyx b/khmer/_oxli/parsing.pyx index 340fbb044a..cad16c7889 100644 --- a/khmer/_oxli/parsing.pyx +++ b/khmer/_oxli/parsing.pyx @@ -60,6 +60,10 @@ cdef class FastxParser: seq = self._next() yield seq + @property + def num_reads(self): + return deref(self._this).get_num_reads() + cdef class SanitizedFastxParser(FastxParser): diff --git a/scripts/load-into-counting.py b/scripts/load-into-counting.py index 6e797232a8..562c449e10 100755 --- a/scripts/load-into-counting.py +++ b/scripts/load-into-counting.py @@ -57,6 +57,7 @@ from khmer.kfile import check_space_for_graph from khmer.khmer_logger import (configure_logging, log_info, log_error, log_warn) +from khmer._oxli.parsing import FastxParser def get_parser(): @@ -142,13 +143,13 @@ def main(): for index, filename in enumerate(filenames): - rparser = khmer.ReadParser(filename) + rparser = FastxParser(filename) threads = [] log_info('consuming input {input}', input=filename) for _ in range(args.threads): cur_thrd = \ threading.Thread( - target=countgraph.consume_seqfile_with_reads_parser, + target=countgraph.consume_seqfile, args=(rparser, ) ) threads.append(cur_thrd) diff --git a/tests/test_countgraph.py b/tests/test_countgraph.py index 05cd331582..703917ac19 100644 --- a/tests/test_countgraph.py +++ b/tests/test_countgraph.py @@ -40,7 +40,7 @@ import os import khmer -from khmer import Countgraph, SmallCountgraph, Nodegraph +from khmer import Countgraph, SmallCountgraph, Nodegraph, FastxParser from . import khmer_tst_utils as utils from khmer import ReadParser import screed @@ -1221,15 +1221,15 @@ def test_consume_absentfasta(): def test_consume_absentfasta_with_reads_parser(): countgraph = khmer.Countgraph(4, 4 ** 4, 4) try: - countgraph.consume_seqfile_with_reads_parser() + countgraph.consume_seqfile() assert 0, "this should fail" except TypeError as err: print(str(err)) try: - readparser = ReadParser(utils.get_test_data('empty-file')) - countgraph.consume_seqfile_with_reads_parser(readparser) + parser = FastxParser(utils.get_test_data('empty-file')) + countgraph.consume_seqfile(parser) assert 0, "this should fail" - except OSError as err: + except RuntimeError as err: print(str(err)) except ValueError as err: print(str(err)) diff --git a/tests/test_nodegraph.py b/tests/test_nodegraph.py index 132c2424fc..1de7c75d50 100644 --- a/tests/test_nodegraph.py +++ b/tests/test_nodegraph.py @@ -36,7 +36,7 @@ import khmer from khmer import Nodegraph, Countgraph -from khmer import ReadParser +from khmer import FastxParser from khmer import reverse_complement as revcomp from khmer.khmer_args import create_matching_nodegraph @@ -938,15 +938,15 @@ def test_bad_primes_list(): def test_consume_absentfasta_with_reads_parser(): nodegraph = khmer.Nodegraph(31, 1, 1) try: - nodegraph.consume_seqfile_with_reads_parser() + nodegraph.consume_seqfile() assert 0, "this should fail" except TypeError as err: print(str(err)) try: - readparser = ReadParser(utils.get_test_data('empty-file')) - nodegraph.consume_seqfile_with_reads_parser(readparser) + parser = FastxParser(utils.get_test_data('empty-file')) + nodegraph.consume_seqfile(parser) assert 0, "this should fail" - except OSError as err: + except RuntimeError as err: print(str(err)) except ValueError as err: print(str(err)) @@ -963,10 +963,10 @@ def test_bad_primes(): def test_consume_seqfile_and_tag_with_badreads_parser(): nodegraph = khmer.Nodegraph(6, 1e6, 2) try: - readsparser = khmer.ReadParser(utils.get_test_data("test-empty.fa")) - nodegraph.consume_seqfile_and_tag_with_reads_parser(readsparser) + parser = FastxParser(utils.get_test_data("test-empty.fa")) + nodegraph.consume_seqfile_and_tag(parser) assert 0, "this should fail" - except OSError as e: + except RuntimeError as e: print(str(e)) except ValueError as e: print(str(e)) From 198b893355d92932db2e011bfb890d17e06e5ac0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Sep 2017 01:01:09 -0700 Subject: [PATCH 116/185] update imports --- khmer/_oxli/app.pyx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index 8850e6b021..937ec73ced 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -1,7 +1,4 @@ # -*- coding: UTF-8 -*- -# cython: c_string_type=unicode, c_string_encoding=utf8 - -from __future__ import print_function import argparse import itertools import json @@ -19,8 +16,10 @@ from khmer._oxli.graphs cimport Nodegraph, Countgraph from khmer._oxli.partitioning cimport StreamingPartitioner, Component from khmer._oxli.partitioning import StreamingPartitioner, Component -from khmer._oxli.parsing cimport BrokenPairedReader, SplitPairedReader, FastxParser, Sequence -from khmer._oxli.parsing import BrokenPairedReader, SplitPairedReader, FastxParser, Sequence +from khmer._oxli.parsing cimport BrokenPairedReader, SplitPairedReader, FastxParser +from khmer._oxli.parsing import BrokenPairedReader, SplitPairedReader, FastxParser +from khmer._oxli.sequence cimport Sequence +from khmer._oxli.sequence import Sequence from khmer._oxli.utils cimport _bstring def grouper(n, iterable): From 255e522f2c619acb1640bfddd156fbbc863be0f2 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Sep 2017 15:20:39 -0700 Subject: [PATCH 117/185] Add headers to package data --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ca4ecbb181..48985a2487 100755 --- a/setup.py +++ b/setup.py @@ -245,7 +245,7 @@ def build_dir(): "Operating System :: MacOS :: MacOS X", "Programming Language :: C++", "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", "Topic :: Scientific/Engineering :: Bio-Informatics", ] if "-rc" in versioneer.get_version(): @@ -279,7 +279,7 @@ def build_dir(): # additional-meta-data note #3 "url": 'https://khmer.readthedocs.io/', "packages": ['khmer', 'khmer.tests', 'oxli', 'khmer._oxli'], - "package_data": {'khmer/_oxli': ['*.pxd']}, + "package_data": {'khmer/_oxli': ['*.pxd', 'oxli_exception_convert.hh']}, "package_dir": {'khmer.tests': 'tests'}, "install_requires": ['screed >= 1.0', 'bz2file', 'Cython==0.25.2'], "setup_requires": ["pytest-runner>=2.0,<3dev", "setuptools>=18.0"], From 57e8dc1bfa45e574b23ea0e116625cc84baabc6b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Sep 2017 15:56:20 -0700 Subject: [PATCH 118/185] Move convert headers to include --- Makefile | 1 + {khmer/_oxli => include/oxli}/oxli_exception_convert.hh | 0 setup.py | 2 +- {khmer/_oxli => src/oxli}/oxli_exception_convert.cc | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) rename {khmer/_oxli => include/oxli}/oxli_exception_convert.hh (100%) rename {khmer/_oxli => src/oxli}/oxli_exception_convert.cc (94%) diff --git a/Makefile b/Makefile index a7b00c8308..2eea6bac47 100644 --- a/Makefile +++ b/Makefile @@ -285,6 +285,7 @@ install-liboxli: liboxli cd src/oxli && $(MAKE) install PREFIX=$(PREFIX) mkdir -p $(PREFIX)/include/khmer cp -r include/khmer/_cpy_*.hh $(PREFIX)/include/khmer/ + cp include/oxli/oxli_exception_convert.hh $(PREFIX)/include/oxli/ # Runs a test of liboxli libtest: FORCE diff --git a/khmer/_oxli/oxli_exception_convert.hh b/include/oxli/oxli_exception_convert.hh similarity index 100% rename from khmer/_oxli/oxli_exception_convert.hh rename to include/oxli/oxli_exception_convert.hh diff --git a/setup.py b/setup.py index 48985a2487..45bdc95152 100755 --- a/setup.py +++ b/setup.py @@ -211,7 +211,7 @@ def build_dir(): CY_EXTENSION_MOD_DICT = \ { - "sources": [cython_ext, "khmer/_oxli/oxli_exception_convert.cc"], + "sources": [cython_ext, "src/oxli/oxli_exception_convert.cc"], "extra_compile_args": EXTRA_COMPILE_ARGS, "extra_link_args": EXTRA_LINK_ARGS, "extra_objects": [path_join(build_dir(), splitext(p)[0] + '.o') diff --git a/khmer/_oxli/oxli_exception_convert.cc b/src/oxli/oxli_exception_convert.cc similarity index 94% rename from khmer/_oxli/oxli_exception_convert.cc rename to src/oxli/oxli_exception_convert.cc index 0e5d2f9935..725d11a6be 100644 --- a/khmer/_oxli/oxli_exception_convert.cc +++ b/src/oxli/oxli_exception_convert.cc @@ -2,7 +2,7 @@ #include #include #include "oxli/oxli_exception.hh" -#include "oxli_exception_convert.hh" +#include "oxli/oxli_exception_convert.hh" void oxli_raise_py_error() From 66bd46ccc2e22f63c9ee8fad9a85c5aae4b1ba09 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Sep 2017 16:08:06 -0700 Subject: [PATCH 119/185] Fix path --- khmer/_oxli/utils.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/khmer/_oxli/utils.pxd b/khmer/_oxli/utils.pxd index ae487c38cd..f69cea6c1a 100644 --- a/khmer/_oxli/utils.pxd +++ b/khmer/_oxli/utils.pxd @@ -4,7 +4,7 @@ from libc.stdint cimport uint32_t, uint64_t from libcpp cimport bool -cdef extern from "oxli_exception_convert.hh": +cdef extern from "oxli/oxli_exception_convert.hh": cdef void oxli_raise_py_error() From 8b81ca50d216d29556fb0efc4d4f57d3c9409322 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Sep 2017 16:41:43 -0700 Subject: [PATCH 120/185] PIMPL for CQF --- include/oxli/storage.hh | 32 ++++------- src/oxli/storage.cc | 115 +++++++++++++++++++++++++++++----------- 2 files changed, 93 insertions(+), 54 deletions(-) diff --git a/include/oxli/storage.hh b/include/oxli/storage.hh index 9a19b0f775..55b770646b 100644 --- a/include/oxli/storage.hh +++ b/include/oxli/storage.hh @@ -43,7 +43,8 @@ Contact: khmer-project@idyll.org #include using MuxGuard = std::lock_guard; -#include "gqf.h" +typedef struct quotient_filter; +typedef quotient_filter QF; namespace oxli { typedef std::map KmerCountMap; @@ -411,19 +412,12 @@ public: */ class QFStorage : public Storage { protected: - QF cf; + std::shared_ptr cf; public: - QFStorage(int size) { - // size is the power of two to specify the number of slots in - // the filter (2**size). Third argument sets the number of bits used - // in the key (current value of size+8 is copied from the CQF example) - // Final argument is the number of bits allocated for the value, which - // we do not use. - qf_init(&cf, (1ULL << size), size+8, 0); - } + QFStorage(int size); - ~QFStorage() { qf_destroy(&cf); } + ~QFStorage(); BoundedCounterType test_and_set_bits(HashIntoType khash) { BoundedCounterType x = get_count(khash); @@ -432,24 +426,18 @@ public: } // - bool add(HashIntoType khash) { - bool is_new = get_count(khash) == 0; - qf_insert(&cf, khash % cf.range, 0, 1); - return is_new; - } + bool add(HashIntoType khash); // get the count for the given k-mer hash. - const BoundedCounterType get_count(HashIntoType khash) const { - return qf_count_key_value(&cf, khash % cf.range, 0); - } + const BoundedCounterType get_count(HashIntoType khash) const; // Accessors for protected/private table info members // xnslots is larger than nslots. It includes some extra slots to deal // with some details of how the counting is implemented - std::vector get_tablesizes() const { return {cf.xnslots}; } + std::vector get_tablesizes() const; const size_t n_tables() const { return 1; } - const uint64_t n_unique_kmers() const { return cf.ndistinct_elts; } - const uint64_t n_occupied() const { return cf.noccupied_slots; } + const uint64_t n_unique_kmers() const; + const uint64_t n_occupied() const; void save(std::string outfilename, WordLength ksize); void load(std::string infilename, WordLength &ksize); diff --git a/src/oxli/storage.cc b/src/oxli/storage.cc index 923843f451..174bb5f822 100644 --- a/src/oxli/storage.cc +++ b/src/oxli/storage.cc @@ -43,6 +43,7 @@ Contact: khmer-project@idyll.org #include "oxli/oxli_exception.hh" #include "oxli/hashtable.hh" #include "zlib.h" +#include "gqf.h" using namespace oxli; using namespace std; @@ -916,6 +917,56 @@ void NibbleStorage::load(std::string infilename, WordLength& ksize) } +QFStorage::QFStorage(int size) +{ + cf = std::make_shared(); + // size is the power of two to specify the number of slots in + // the filter (2**size). Third argument sets the number of bits used + // in the key (current value of size+8 is copied from the CQF example) + // Final argument is the number of bits allocated for the value, which + // we do not use. + qf_init(cf.get(), (1ULL << size), size+8, 0); +} + + +QFStorage::~QFStorage() +{ + qf_destroy(cf.get()); +} + + +bool QFStorage::add(HashIntoType khash) +{ + bool is_new = get_count(khash) == 0; + qf_insert(cf.get(), khash % cf->range, 0, 1); + return is_new; +} + + +const BoundedCounterType QFStorage::get_count(HashIntoType khash) const +{ + return qf_count_key_value(cf.get(), khash % cf->range, 0); +} + + +std::vector QFStorage::get_tablesizes() const +{ + return {cf->xnslots}; +} + + +const uint64_t QFStorage::n_unique_kmers() const +{ + return cf->ndistinct_elts; +} + + +const uint64_t QFStorage::n_occupied() const +{ + return cf->noccupied_slots; +} + + void QFStorage::save(std::string outfilename, WordLength ksize) { ofstream outfile(outfilename.c_str(), ios::binary); @@ -931,25 +982,25 @@ void QFStorage::save(std::string outfilename, WordLength ksize) /* just a hack to handle __uint128_t value. Don't know a better to handle it * right now */ uint64_t tmp_range; - tmp_range = cf.range; - - outfile.write((const char *) &cf.nslots, sizeof(cf.nslots)); - outfile.write((const char *) &cf.xnslots, sizeof(cf.xnslots)); - outfile.write((const char *) &cf.key_bits, sizeof(cf.key_bits)); - outfile.write((const char *) &cf.value_bits, sizeof(cf.value_bits)); - outfile.write((const char *) &cf.key_remainder_bits, sizeof(cf.key_remainder_bits)); - outfile.write((const char *) &cf.bits_per_slot, sizeof(cf.bits_per_slot)); + tmp_range = cf->range; + + outfile.write((const char *) &cf->nslots, sizeof(cf->nslots)); + outfile.write((const char *) &cf->xnslots, sizeof(cf->xnslots)); + outfile.write((const char *) &cf->key_bits, sizeof(cf->key_bits)); + outfile.write((const char *) &cf->value_bits, sizeof(cf->value_bits)); + outfile.write((const char *) &cf->key_remainder_bits, sizeof(cf->key_remainder_bits)); + outfile.write((const char *) &cf->bits_per_slot, sizeof(cf->bits_per_slot)); outfile.write((const char *) &tmp_range, sizeof(tmp_range)); - outfile.write((const char *) &cf.nblocks, sizeof(cf.nblocks)); - outfile.write((const char *) &cf.nelts, sizeof(cf.nelts)); - outfile.write((const char *) &cf.ndistinct_elts, sizeof(cf.ndistinct_elts)); - outfile.write((const char *) &cf.noccupied_slots, sizeof(cf.noccupied_slots)); + outfile.write((const char *) &cf->nblocks, sizeof(cf->nblocks)); + outfile.write((const char *) &cf->nelts, sizeof(cf->nelts)); + outfile.write((const char *) &cf->ndistinct_elts, sizeof(cf->ndistinct_elts)); + outfile.write((const char *) &cf->noccupied_slots, sizeof(cf->noccupied_slots)); #if BITS_PER_SLOT == 8 || BITS_PER_SLOT == 16 || BITS_PER_SLOT == 32 || BITS_PER_SLOT == 64 - outfile.write((const char *) cf.blocks, sizeof(qfblock) * cf.nblocks); + outfile.write((const char *) cf->blocks, sizeof(qfblock) * cf->nblocks); #else - outfile.write((const char *) cf.blocks, - (sizeof(qfblock) + SLOTS_PER_BLOCK * cf.bits_per_slot / 8) * cf.nblocks); + outfile.write((const char *) cf->blocks, + (sizeof(qfblock) + SLOTS_PER_BLOCK * cf->bits_per_slot / 8) * cf->nblocks); #endif outfile.close(); } @@ -1011,34 +1062,34 @@ void QFStorage::load(std::string infilename, WordLength &ksize) infile.read((char *) &save_ksize, sizeof(save_ksize)); ksize = save_ksize; - infile.read((char *) &cf.nslots, sizeof(cf.nslots)); - infile.read((char *) &cf.xnslots, sizeof(cf.xnslots)); - infile.read((char *) &cf.key_bits, sizeof(cf.key_bits)); - infile.read((char *) &cf.value_bits, sizeof(cf.value_bits)); - infile.read((char *) &cf.key_remainder_bits, sizeof(cf.key_remainder_bits)); - infile.read((char *) &cf.bits_per_slot, sizeof(cf.bits_per_slot)); + infile.read((char *) &cf->nslots, sizeof(cf->nslots)); + infile.read((char *) &cf->xnslots, sizeof(cf->xnslots)); + infile.read((char *) &cf->key_bits, sizeof(cf->key_bits)); + infile.read((char *) &cf->value_bits, sizeof(cf->value_bits)); + infile.read((char *) &cf->key_remainder_bits, sizeof(cf->key_remainder_bits)); + infile.read((char *) &cf->bits_per_slot, sizeof(cf->bits_per_slot)); infile.read((char *) &tmp_range, sizeof(tmp_range)); - infile.read((char *) &cf.nblocks, sizeof(cf.nblocks)); - infile.read((char *) &cf.nelts, sizeof(cf.nelts)); - infile.read((char *) &cf.ndistinct_elts, sizeof(cf.ndistinct_elts)); - infile.read((char *) &cf.noccupied_slots, sizeof(cf.noccupied_slots)); + infile.read((char *) &cf->nblocks, sizeof(cf->nblocks)); + infile.read((char *) &cf->nelts, sizeof(cf->nelts)); + infile.read((char *) &cf->ndistinct_elts, sizeof(cf->ndistinct_elts)); + infile.read((char *) &cf->noccupied_slots, sizeof(cf->noccupied_slots)); /* just a hack to handle __uint128_t value. Don't know a better to handle it * right now */ - cf.range = tmp_range; + cf->range = tmp_range; // deallocate previously allocated blocks - free(cf.blocks); + free(cf->blocks); /* allocate the space for the actual qf blocks */ #if BITS_PER_SLOT == 8 || BITS_PER_SLOT == 16 || BITS_PER_SLOT == 32 || BITS_PER_SLOT == 64 - cf.blocks = (qfblock *)calloc(cf.nblocks, sizeof(qfblock)); + cf->blocks = (qfblock *)calloc(cf->nblocks, sizeof(qfblock)); #else - cf.blocks = (qfblock *)calloc(cf.nblocks, sizeof(qfblock) + SLOTS_PER_BLOCK * cf.bits_per_slot / 8); + cf->blocks = (qfblock *)calloc(cf->nblocks, sizeof(qfblock) + SLOTS_PER_BLOCK * cf->bits_per_slot / 8); #endif #if BITS_PER_SLOT == 8 || BITS_PER_SLOT == 16 || BITS_PER_SLOT == 32 || BITS_PER_SLOT == 64 - infile.read((char *) cf.blocks, sizeof(qfblock) * cf.nblocks); + infile.read((char *) cf->blocks, sizeof(qfblock) * cf->nblocks); #else - infile.read((char *) cf.blocks, - (sizeof(qfblock) + SLOTS_PER_BLOCK * cf.bits_per_slot / 8) * cf.nblocks); + infile.read((char *) cf->blocks, + (sizeof(qfblock) + SLOTS_PER_BLOCK * cf->bits_per_slot / 8) * cf->nblocks); #endif infile.close(); } From b84266eab4c15758ce6b08e1a588bcfc81cafdf5 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 12 Sep 2017 16:08:11 -0700 Subject: [PATCH 121/185] Change gmap to unordered_map --- include/oxli/gmap.hh | 3 ++- khmer/_oxli/partitioning.pxd | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/oxli/gmap.hh b/include/oxli/gmap.hh index 1a5cff3ee4..15be996c9f 100644 --- a/include/oxli/gmap.hh +++ b/include/oxli/gmap.hh @@ -39,6 +39,7 @@ Contact: khmer-project@idyll.org #include #include +#include #include "oxli.hh" #include "kmer_hash.hh" @@ -55,7 +56,7 @@ class GuardedHashMap { // Filter should be owned exclusively by GuardedKmerMap std::unique_ptr filter; - std::map data; + std::unordered_map data; explicit GuardedHashMap(WordLength ksize, unsigned short n_tables, diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index 6db45a3562..e2b7aa7f5b 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -1,6 +1,6 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr, weak_ptr, shared_ptr -from libcpp.map cimport map +from libcpp.unordered_map cimport unordered_map from libcpp.vector cimport vector from libcpp.set cimport set from libcpp.queue cimport queue @@ -43,7 +43,7 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": ctypedef vector[ComponentPtr] ComponentPtrVector cdef cppclass CpGuardedHashCompMap "oxli::GuardedHashCompMap": - map[HashIntoType, ComponentPtr] data + unordered_map[HashIntoType, ComponentPtr] data ComponentPtr get(HashIntoType) void set(HashIntoType, ComponentPtr) From 813d3672fc27f8abe5c968eaacf9304c8cec8da7 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 12 Sep 2017 16:08:44 -0700 Subject: [PATCH 122/185] Add a performance comparison of map v unordered_map --- src/oxli/Makefile | 3 + src/oxli/map_type_test.cc | 249 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 252 insertions(+) create mode 100644 src/oxli/map_type_test.cc diff --git a/src/oxli/Makefile b/src/oxli/Makefile index f44f6bc985..cd7d005880 100644 --- a/src/oxli/Makefile +++ b/src/oxli/Makefile @@ -329,6 +329,9 @@ murmur3.o: ../../third-party/smhasher/MurmurHash3.cc %.o: %.cc $(PRECOMILE_OBJS) $(OXLI_HEADERS) $(CXX) $(CXXFLAGS) $(LDFLAGS) -c -o $@ $< +map_type_test: map_type_test.cc + $(CXX) -o $@ $< $(CXXFLAGS) -loxli -L. + $(LIBOXLISO): $(LIBOXLI_OBJS) $(CXX) $(CXXFLAGS) $(LDFLAGS) $(SONAME_FLAGS) -shared -o $@ $^ ln -sf $(SONAME) liboxli.$(SHARED_EXT) diff --git a/src/oxli/map_type_test.cc b/src/oxli/map_type_test.cc new file mode 100644 index 0000000000..2dde9423e3 --- /dev/null +++ b/src/oxli/map_type_test.cc @@ -0,0 +1,249 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace oxli; +using namespace oxli::read_parsers; + +#define K 21 + +unsigned long long llrand() { + unsigned long long r = 0; + + for (int i = 0; i < 5; ++i) { + r = (r << 15) | (rand() & 0x7FFF); + } + + return r & 0xFFFFFFFFFFFFFFFFULL; +} + +FastxParserPtr get_test_reads() { + FastxParserPtr parser = get_parser("../../tests/test-data/test-reads.fa"); + return parser; +} + +vector * get_test_kmers(int num_hashes=5000000) { + vector * hashes = new vector(); + while(num_hashes > 0) { + hashes->push_back(llrand()); + num_hashes--; + } + return hashes; +} + + +void fill_gmap(GuardedHashMap& _map, vector * hashes) { + for(auto hash: *hashes) { + _map.set(hash, rand()); + } +} + + +void fill_uomap(std::unordered_map& _map, vector * hashes) { + for (auto hash: *hashes) { + _map[hash] = rand(); + } +} + +void fill_map(std::map& _map, vector * hashes) { + for (auto hash: *hashes) { + _map[hash] = rand(); + } +} + +void test_gmap(vector * hashes) { + + std::cout << "=== GMAP ===" << std::endl; + + vector get_full_times; + vector get_empty_times; + vector get_bad_times; + GuardedHashMap _map(K, 4, 1000000); + std::chrono::time_point start, end; + + fill_gmap(_map, hashes); + for (auto hash: *hashes) { + start = std::chrono::system_clock::now(); + int result = _map.get(hash); + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + get_full_times.push_back(elapsed_seconds.count()); + } + double avg_get_full_time = std::accumulate(get_full_times.begin(), + get_full_times.end(), 0.0) / get_full_times.size(); + std::cout << "Avg full get time: " << avg_get_full_time << std::endl; + + + vector * newhashes = get_test_kmers(); + for (auto hash: *newhashes) { + start = std::chrono::system_clock::now(); + int result = _map.get(hash); + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + get_bad_times.push_back(elapsed_seconds.count()); + } + double avg_get_bad_time = std::accumulate(get_bad_times.begin(), + get_bad_times.end(), 0.0) / get_bad_times.size(); + std::cout << "Avg bad get time: " << avg_get_bad_time << std::endl; + delete newhashes; + + + _map = GuardedHashMap(K, 4, 1000000); + + for (auto hash: *hashes) { + start = std::chrono::system_clock::now(); + int result = _map.get(hash); + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + + get_empty_times.push_back(elapsed_seconds.count()); + } + + double avg_get_empty_time = std::accumulate(get_empty_times.begin(), + get_empty_times.end(), 0.0) / get_empty_times.size(); + std::cout << "Avg empty get time: " << avg_get_empty_time << std::endl; +} + +void test_uomap(vector * hashes) { + + std::cout << "=== UOMAP ===" << std::endl; + + vector get_full_times; + vector get_empty_times; + vector get_bad_times; + std::unordered_map _map; + std::chrono::time_point start, end; + + fill_uomap(_map, hashes); + + for (auto hash: *hashes) { + start = std::chrono::system_clock::now(); + int result; + auto search = _map.find(hash); + if (search != _map.end()) { + result = search->second; + } + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + + get_full_times.push_back(elapsed_seconds.count()); + } + double avg_get_full_time = std::accumulate(get_full_times.begin(), + get_full_times.end(), 0.0) / get_full_times.size(); + std::cout << "Avg full get time: " << avg_get_full_time << std::endl; + + + vector * newhashes = get_test_kmers(); + for (auto hash: *newhashes) { + start = std::chrono::system_clock::now(); + int result; + auto search = _map.find(hash); + if (search != _map.end()) { + result = search->second; + } + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + get_bad_times.push_back(elapsed_seconds.count()); + } + double avg_get_bad_time = std::accumulate(get_bad_times.begin(), + get_bad_times.end(), 0.0) / get_bad_times.size(); + std::cout << "Avg bad get time: " << avg_get_bad_time << std::endl; + delete newhashes; + + + _map = std::unordered_map(); + for (auto hash: *hashes) { + start = std::chrono::system_clock::now(); + int result; + auto search = _map.find(hash); + if (search != _map.end()) { + result = search->second; + } + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + + get_empty_times.push_back(elapsed_seconds.count()); + } + + double avg_get_empty_time = std::accumulate(get_empty_times.begin(), + get_empty_times.end(), 0.0) / get_empty_times.size(); + std::cout << "Avg empty get time: " << avg_get_empty_time << std::endl; +} + +void test_map(vector * hashes) { + + std::cout << "=== MAP ===" << std::endl; + + vector get_full_times; + vector get_empty_times; + vector get_bad_times; + std::map _map; + std::chrono::time_point start, end; + + fill_map(_map, hashes); + for (auto hash: *hashes) { + start = std::chrono::system_clock::now(); + int result; + auto search = _map.find(hash); + if (search != _map.end()) { + result = search->second; + } + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + + get_full_times.push_back(elapsed_seconds.count()); + } + double avg_get_full_time = std::accumulate(get_full_times.begin(), + get_full_times.end(), 0.0) / get_full_times.size(); + std::cout << "Avg full get time: " << avg_get_full_time << std::endl; + + vector * newhashes = get_test_kmers(); + for (auto hash: *newhashes) { + start = std::chrono::system_clock::now(); + int result; + auto search = _map.find(hash); + if (search != _map.end()) { + result = search->second; + } + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + get_bad_times.push_back(elapsed_seconds.count()); + } + double avg_get_bad_time = std::accumulate(get_bad_times.begin(), + get_bad_times.end(), 0.0) / get_bad_times.size(); + std::cout << "Avg bad get time: " << avg_get_bad_time << std::endl; + delete newhashes; + + _map = std::map(); + for (auto hash: *hashes) { + start = std::chrono::system_clock::now(); + int result; + auto search = _map.find(hash); + if (search != _map.end()) { + result = search->second; + } + end = std::chrono::system_clock::now(); + std::chrono::duration elapsed_seconds = end-start; + + get_empty_times.push_back(elapsed_seconds.count()); + } + + double avg_get_empty_time = std::accumulate(get_empty_times.begin(), + get_empty_times.end(), 0.0) / get_empty_times.size(); + std::cout << "Avg empty get time: " << avg_get_empty_time << std::endl; +} + + +int main() { + vector * hashes = get_test_kmers(); + test_gmap(hashes); + test_uomap(hashes); + test_map(hashes); +} From 3c81f5c71d8a9d2c639b5fcde79349b82dad8ce8 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 12 Sep 2017 16:09:24 -0700 Subject: [PATCH 123/185] Sketch Linked dbg and implemented get_junction_choices --- khmer/_oxli/graphlinks.pxd | 26 ++++++++++++++ khmer/_oxli/graphlinks.pyx | 69 ++++++++++++++++++++++++++++++++++++++ tests/test_linked_dbg.py | 51 ++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 khmer/_oxli/graphlinks.pxd create mode 100644 khmer/_oxli/graphlinks.pyx create mode 100644 tests/test_linked_dbg.py diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd new file mode 100644 index 0000000000..825df9e33d --- /dev/null +++ b/khmer/_oxli/graphlinks.pxd @@ -0,0 +1,26 @@ +from libcpp.memory cimport shared_ptr + +from khmer._oxli.oxli_types cimport * +from khmer._oxli.graphs cimport CpHashgraph, Hashgraph, Nodegraph, Countgraph + + +cdef class Link: + cdef readonly HashIntoType u + cdef readonly HashIntoType v + + @staticmethod + cdef Link _new(HashIntoType u, HashIntoType v) + + +cdef class LinkPath: + + cdef readonly list path + + +cdef class GraphLinker: + + cdef Hashgraph graph + cdef shared_ptr[CpHashgraph] _graph + cdef dict links + + cdef list _get_junction_choices(self, string sequence) diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx new file mode 100644 index 0000000000..a6df754bf0 --- /dev/null +++ b/khmer/_oxli/graphlinks.pyx @@ -0,0 +1,69 @@ +cimport cython + +from cython.operator cimport dereference as deref + +from libcpp.memory cimport shared_ptr, make_shared +from libcpp.string cimport string +from libcpp.vector cimport vector + +from khmer._oxli.oxli_types cimport * +from khmer._oxli.hashing cimport CpKmer, CpKmerIterator +from khmer._oxli.traversal cimport CpTraverser +from khmer._oxli.utils cimport _bstring + +ctypedef vector[HashIntoType] HashVector +ctypedef shared_ptr[HashVector] HashVectorPtr + + +@cython.freelist(100) +cdef class Link: + + def __init__(self, HashIntoType u, HashIntoType v): + self.u = u + self.v = v + + @staticmethod + cdef Link _new(HashIntoType u, HashIntoType v): + cdef Link link = Link.__new__(Link) + link.u = u + link.v = v + return link + + +cdef class LinkPath: + + def __cinit__(self): + self.path = [] + + +cdef class GraphLinker: + + def __cinit__(self, Hashgraph graph not None): + self.graph = graph + self._graph = graph._hg_this + + # mapping from high degree flanking nodes to link paths + self.links = {} + + cdef list _get_junction_choices(self, string sequence): + cdef shared_ptr[CpTraverser] _traverser = make_shared[CpTraverser](self._graph.get()) + cdef CpKmerIterator * _it = new CpKmerIterator(sequence.c_str(), + self.graph.ksize()) + cdef CpKmer u = deref(_it).next() + if deref(_it).done(): + return [] + + cdef list choices = [] + cdef CpKmer v = deref(_it).next() + + while not deref(_it).done(): + if deref(_traverser).degree_left(v) > 1: + choices.append(Link._new(u, v)) + u = v + v = deref(_it).next() + + return choices + + def get_junction_choices(self, str sequence): + cdef list choices = self._get_junction_choices(_bstring(sequence)) + return choices diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py new file mode 100644 index 0000000000..2a676afcb6 --- /dev/null +++ b/tests/test_linked_dbg.py @@ -0,0 +1,51 @@ +import gc +import itertools +import random + +from khmer import reverse_complement as revcomp +from khmer import reverse_hash as revhash +from khmer import forward_hash +from . import khmer_tst_utils as utils +from .graph_features import * + +from khmer._oxli.graphlinks import Link, LinkPath, GraphLinker +from khmer import Nodegraph +import pytest + + +def teardown(): + utils.cleanup() + + +def test_get_junction_choices(left_tip_structure): + graph, contig, L, HDN, R, tip = left_tip_structure + linker = GraphLinker(graph) + + choices = linker.get_junction_choices(contig) + assert len(choices) == 1 + link = choices.pop() + + assert link.u == forward_hash(L, K) + assert link.v == forward_hash(HDN, K) + + +def test_get_junction_choices_rc(left_tip_structure): + graph, contig, L, HDN, R, tip = left_tip_structure + linker = GraphLinker(graph) + + choices = linker.get_junction_choices(revcomp(contig)) + assert len(choices) == 0 + + +def test_get_junction_choices_right_rc(right_tip_structure): + graph, contig, L, HDN, R, tip = right_tip_structure + linker = GraphLinker(graph) + + choices = linker.get_junction_choices(revcomp(contig)) + assert len(choices) == 1 + link = choices.pop() + + assert link.u == forward_hash(R, K) + assert link.v == forward_hash(HDN, K) + + From 1012ff345daa77d71fcaa67ea9800d49a7fdaab7 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 13 Sep 2017 15:06:12 -0700 Subject: [PATCH 124/185] stub --- khmer/_oxli/graphlinks.pyx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index a6df754bf0..be71ba4543 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -67,3 +67,6 @@ cdef class GraphLinker: def get_junction_choices(self, str sequence): cdef list choices = self._get_junction_choices(_bstring(sequence)) return choices + + cdef void build_links(self, string sequence): + From 41c145a9872ece86cc5d1075444beea2148dca62 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 14 Sep 2017 15:12:16 -0700 Subject: [PATCH 125/185] More attributes on Link --- khmer/_oxli/graphlinks.pxd | 18 +++++++++++++++++- khmer/_oxli/graphlinks.pyx | 35 ++++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 825df9e33d..9dd9e7c3a6 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -1,15 +1,30 @@ +cimport cython from libcpp.memory cimport shared_ptr +from libc.stdint cimport uint8_t, uint32_t, uint64_t from khmer._oxli.oxli_types cimport * from khmer._oxli.graphs cimport CpHashgraph, Hashgraph, Nodegraph, Countgraph +cdef enum DBGNucl: + A, C, G, T + + cdef class Link: cdef readonly HashIntoType u cdef readonly HashIntoType v + cdef readonly bool forward + cdef list children + cdef Link parent + + cpdef bool add_child(self, DBGNucl nuc, Link child_link) + cpdef Link get_child(self, DBGNucl nuc) @staticmethod - cdef Link _new(HashIntoType u, HashIntoType v) + cdef Link _new(HashIntoType u, + HashIntoType v, + bool forward, + Link parent=*) cdef class LinkPath: @@ -21,6 +36,7 @@ cdef class GraphLinker: cdef Hashgraph graph cdef shared_ptr[CpHashgraph] _graph + cdef WordLength K cdef dict links cdef list _get_junction_choices(self, string sequence) diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index be71ba4543..420649ae6a 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -11,22 +11,38 @@ from khmer._oxli.hashing cimport CpKmer, CpKmerIterator from khmer._oxli.traversal cimport CpTraverser from khmer._oxli.utils cimport _bstring -ctypedef vector[HashIntoType] HashVector -ctypedef shared_ptr[HashVector] HashVectorPtr - @cython.freelist(100) cdef class Link: - def __init__(self, HashIntoType u, HashIntoType v): + def __init__(self, HashIntoType u, + HashIntoType v, + bool forward, + Link parent=None): self.u = u self.v = v + self.forward = forward + self.children = [None, None, None, None] + self.parent = parent + + @cython.boundscheck(False) + cpdef bool add_child(self, DBGNucl nuc, Link child_link): + self.children[nuc] = child_link + + @cython.boundscheck(False) + cpdef Link get_child(self, DBGNucl nuc): + return self.children[nuc] @staticmethod - cdef Link _new(HashIntoType u, HashIntoType v): + cdef Link _new(HashIntoType u, + HashIntoType v, + bool forward, + Link parent=None): cdef Link link = Link.__new__(Link) link.u = u link.v = v + link.forward = forward + link.parent = parent return link @@ -41,6 +57,7 @@ cdef class GraphLinker: def __cinit__(self, Hashgraph graph not None): self.graph = graph self._graph = graph._hg_this + self.K = graph.ksize() # mapping from high degree flanking nodes to link paths self.links = {} @@ -55,12 +72,18 @@ cdef class GraphLinker: cdef list choices = [] cdef CpKmer v = deref(_it).next() + cdef uint64_t kmer_start_idx = 0 + cdef uint64_t kmer_end_idx = self.K - 1 while not deref(_it).done(): if deref(_traverser).degree_left(v) > 1: + choices.append(Link._new(u, v)) + u = v v = deref(_it).next() + kmer_start_idx += 1 + kmer_end_idx += 1 return choices @@ -68,5 +91,3 @@ cdef class GraphLinker: cdef list choices = self._get_junction_choices(_bstring(sequence)) return choices - cdef void build_links(self, string sequence): - From 6c4d382108d221df2ab25d1c50d6df8cd74eb983 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Sep 2017 03:27:52 -0700 Subject: [PATCH 126/185] Update terminology, add fw and rc junction discovery plus initial link tracking --- khmer/_oxli/graphlinks.pxd | 26 +++++---- khmer/_oxli/graphlinks.pyx | 108 ++++++++++++++++++++++--------------- khmer/_oxli/sequence.pxd | 3 ++ khmer/_oxli/sequence.pyx | 9 ++++ tests/test_linked_dbg.py | 34 ++++++------ 5 files changed, 104 insertions(+), 76 deletions(-) diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 9dd9e7c3a6..e8e3aac6f2 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -10,26 +10,22 @@ cdef enum DBGNucl: A, C, G, T -cdef class Link: +cdef class Junction: cdef readonly HashIntoType u cdef readonly HashIntoType v - cdef readonly bool forward - cdef list children - cdef Link parent - - cpdef bool add_child(self, DBGNucl nuc, Link child_link) - cpdef Link get_child(self, DBGNucl nuc) @staticmethod - cdef Link _new(HashIntoType u, - HashIntoType v, - bool forward, - Link parent=*) + cdef Junction _new(HashIntoType u, + HashIntoType v) -cdef class LinkPath: +cdef class Link: - cdef readonly list path + cdef readonly list junctions + cdef bool forward + + @staticmethod + cdef Link _new(list junctions, bool forward) cdef class GraphLinker: @@ -39,4 +35,6 @@ cdef class GraphLinker: cdef WordLength K cdef dict links - cdef list _get_junction_choices(self, string sequence) + cdef tuple _get_junctions(self, string sequence) + cdef int _add_link(self, string sequence, + unsigned int min_link_size=*) except -1 diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 420649ae6a..d215f501e6 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -7,51 +7,44 @@ from libcpp.string cimport string from libcpp.vector cimport vector from khmer._oxli.oxli_types cimport * -from khmer._oxli.hashing cimport CpKmer, CpKmerIterator +from khmer._oxli.hashing cimport CpKmer, CpKmerIterator, _revcomp +from khmer._oxli.sequence cimport Sequence, _object_to_string +from khmer._oxli.sequence import Sequence from khmer._oxli.traversal cimport CpTraverser from khmer._oxli.utils cimport _bstring @cython.freelist(100) -cdef class Link: +cdef class Junction: def __init__(self, HashIntoType u, - HashIntoType v, - bool forward, - Link parent=None): + HashIntoType v): self.u = u self.v = v - self.forward = forward - self.children = [None, None, None, None] - self.parent = parent - @cython.boundscheck(False) - cpdef bool add_child(self, DBGNucl nuc, Link child_link): - self.children[nuc] = child_link + @staticmethod + cdef Junction _new(HashIntoType u, + HashIntoType v): + cdef Junction junc = Junction.__new__(Junction) + junc.u = u + junc.v = v + return junc + + +cdef class Link: - @cython.boundscheck(False) - cpdef Link get_child(self, DBGNucl nuc): - return self.children[nuc] + def __init__(self, list junctions, bool forward): + self.junctions = junctions + self.forward = forward @staticmethod - cdef Link _new(HashIntoType u, - HashIntoType v, - bool forward, - Link parent=None): + cdef Link _new(list junctions, bool forward): cdef Link link = Link.__new__(Link) - link.u = u - link.v = v + link.junctions = junctions link.forward = forward - link.parent = parent return link -cdef class LinkPath: - - def __cinit__(self): - self.path = [] - - cdef class GraphLinker: def __cinit__(self, Hashgraph graph not None): @@ -59,35 +52,64 @@ cdef class GraphLinker: self._graph = graph._hg_this self.K = graph.ksize() - # mapping from high degree flanking nodes to link paths self.links = {} - cdef list _get_junction_choices(self, string sequence): + def check_sequence_length(self, object sequence): + if len(sequence) < self.K: + raise ValueError("sequence length ({}) must >= the hashtable " + "k-mer size ({})".format(len(sequence), + self.K)) + + cdef tuple _get_junctions(self, string sequence): cdef shared_ptr[CpTraverser] _traverser = make_shared[CpTraverser](self._graph.get()) cdef CpKmerIterator * _it = new CpKmerIterator(sequence.c_str(), self.graph.ksize()) cdef CpKmer u = deref(_it).next() + cdef CpKmer start = u if deref(_it).done(): - return [] + return [], [] - cdef list choices = [] + cdef list fw_choices = [] + cdef list rc_choices = [] cdef CpKmer v = deref(_it).next() - cdef uint64_t kmer_start_idx = 0 - cdef uint64_t kmer_end_idx = self.K - 1 + #cdef uint64_t kmer_start_idx = 0 + #cdef uint64_t kmer_end_idx = self.K - 1 while not deref(_it).done(): + if deref(_traverser).degree_right(u) > 1: + fw_choices.append(Junction._new(u, v)) if deref(_traverser).degree_left(v) > 1: - - choices.append(Link._new(u, v)) + rc_choices.append(Junction._new(v, u)) u = v v = deref(_it).next() - kmer_start_idx += 1 - kmer_end_idx += 1 - - return choices - - def get_junction_choices(self, str sequence): - cdef list choices = self._get_junction_choices(_bstring(sequence)) - return choices + #kmer_start_idx += 1 + #kmer_end_idx += 1 + rc_choices.reverse() + return fw_choices, rc_choices + + def get_junctions(self, object sequence): + self.check_sequence_length(sequence) + return self._get_junctions(_object_to_string(sequence)) + + cdef int _add_link(self, string sequence, unsigned int min_link_size=1) except -1: + + cdef Link fw_link, rc_link + fw_choices, rc_choices = self._get_junctions(sequence) + + if len(fw_choices) >= min_link_size: + fw_link = Link._new(fw_choices, True) + self.links[fw_choices[0].u] = fw_link + if len(rc_choices) >= min_link_size: + rc_link = Link._new(rc_choices, False) + self.links[rc_choices[0].u] = rc_link + + return len(fw_choices) + len(rc_choices) + + def add_link(self, object sequence, unsigned int min_link_size=1): + if min_link_size < 1: + raise ValueError("Minimum link size must be at least 1.") + self.check_sequence_length(sequence) + + return self._add_links(_object_to_string(sequence), min_link_size) diff --git a/khmer/_oxli/sequence.pxd b/khmer/_oxli/sequence.pxd index ae489fbc7c..d8c8e30937 100644 --- a/khmer/_oxli/sequence.pxd +++ b/khmer/_oxli/sequence.pxd @@ -69,6 +69,9 @@ cdef class Sequence: cdef Sequence _wrap(CpSequence cseq) +cdef string _object_to_string(object sequence) except * + + cdef class ReadBundle: cdef list reads diff --git a/khmer/_oxli/sequence.pyx b/khmer/_oxli/sequence.pyx index ff672f4865..ec4443641a 100644 --- a/khmer/_oxli/sequence.pyx +++ b/khmer/_oxli/sequence.pyx @@ -129,6 +129,15 @@ cdef class Sequence: return seq +cdef string _object_to_string(object sequence) except *: + if isinstance(sequence, bytes): + return sequence + elif isinstance(sequence, Sequence): + return (sequence)._obj.cleaned_seq + else: + return _bstring(sequence) + + cdef class ReadBundle: def __cinit__(self, *raw_records): diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 2a676afcb6..52ca49aeba 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -8,7 +8,7 @@ from . import khmer_tst_utils as utils from .graph_features import * -from khmer._oxli.graphlinks import Link, LinkPath, GraphLinker +from khmer._oxli.graphlinks import Link, Junction, GraphLinker from khmer import Nodegraph import pytest @@ -17,35 +17,31 @@ def teardown(): utils.cleanup() -def test_get_junction_choices(left_tip_structure): - graph, contig, L, HDN, R, tip = left_tip_structure +def test_get_junctions(right_tip_structure): + graph, contig, L, HDN, R, tip = right_tip_structure linker = GraphLinker(graph) - choices = linker.get_junction_choices(contig) - assert len(choices) == 1 - link = choices.pop() + fw_choices, rc_choices = linker.get_junctions(contig) + assert len(fw_choices) == 1 + assert len(rc_choices) == 0 + link = fw_choices.pop() - assert link.u == forward_hash(L, K) - assert link.v == forward_hash(HDN, K) + assert link.u == forward_hash(HDN, K) + assert link.v == forward_hash(R, K) -def test_get_junction_choices_rc(left_tip_structure): +def test_get_junctions_rc(left_tip_structure): graph, contig, L, HDN, R, tip = left_tip_structure linker = GraphLinker(graph) - choices = linker.get_junction_choices(revcomp(contig)) - assert len(choices) == 0 + fw_choices, rc_choices = linker.get_junctions(contig) + assert len(fw_choices) == 0 + link = rc_choices.pop() -def test_get_junction_choices_right_rc(right_tip_structure): - graph, contig, L, HDN, R, tip = right_tip_structure - linker = GraphLinker(graph) + assert link.u == forward_hash(HDN, K) + assert link.v == forward_hash(L, K) - choices = linker.get_junction_choices(revcomp(contig)) - assert len(choices) == 1 - link = choices.pop() - assert link.u == forward_hash(R, K) - assert link.v == forward_hash(HDN, K) From 7d39c1d33b99e1dfe207998636dba3ea55e32086 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 28 Sep 2017 15:39:51 -0700 Subject: [PATCH 127/185] First pass full c++ linked dbg --- include/oxli/links.hh | 359 +++++++++++++++++++++++++++++++++++++ khmer/_oxli/graphlinks.pxd | 49 ++--- khmer/_oxli/graphlinks.pyx | 141 ++++----------- setup.py | 4 +- src/oxli/links.cc | 28 +++ tests/test_linked_dbg.py | 33 ++-- 6 files changed, 462 insertions(+), 152 deletions(-) create mode 100644 include/oxli/links.hh create mode 100644 src/oxli/links.cc diff --git a/include/oxli/links.hh b/include/oxli/links.hh new file mode 100644 index 0000000000..5580d30fb6 --- /dev/null +++ b/include/oxli/links.hh @@ -0,0 +1,359 @@ +/* +This file is part of khmer, https://github.com/dib-lab/khmer/, and is +Copyright (C) 2015-2016, The Regents of the University of California. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the Michigan State University nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +LICENSE (END) + +Contact: khmer-project@idyll.org +*/ +#ifndef LINKS_HH +#define LINKS_HH + +#include +#include +#include +#include + +#include "oxli.hh" +#include "kmer_hash.hh" +#include "hashtable.hh" +#include "hashgraph.hh" +#include "kmer_filters.hh" +#include "traversal.hh" + + +namespace oxli { + +struct Junction { + HashIntoType u; + HashIntoType v; + //uint32_t distance_prev; + uint64_t count; + Junction() = default; + HashIntoType id() const { return u ^ v; } + + friend std::ostream& operator<< (std::ostream& stream, + const Junction* j); +}; + +typedef std::list JunctionList; + + +class Link { +private: + + static uint64_t n_links; + +protected: + + JunctionList junctions; + bool forward; + uint64_t link_id; + uint64_t time_created; // in "read space" ie read number + //size_t extent; + +public: + + Link(uint64_t time_created, bool forward=true) : + forward(forward), link_id(n_links), + time_created(time_created) + { + n_links++; + } + inline void push_back(Junction* junction) + { + //if (junctions.size() == 0) { + // junction->start = true; + //junction->distance_prev = 0; + //} + junctions.push_back(junction); + } + + inline void push_front(Junction* junction) + { + //if (junctions.size() > 0) { + // start_junction()->start = false; + //} + //junction->start = true; // just to be sure + //junction->distance_prev = 0; + junctions.push_front(junction); + } + + void insert_junction(JunctionList::iterator insert_before, + Junction* junction) + { + if (insert_before == begin()) { + push_front(junction); + } else { + junctions.insert(insert_before, junction); + } + } + + inline Junction* start_junction() const + { + return junctions.front(); + } + + inline Junction* end_junction() const + { + return junctions.back(); + } + + inline bool is_forward() const + { + return forward; + } + + inline size_t size() const + { + return junctions.size(); + } + + inline JunctionList::iterator begin() + { + return junctions.begin(); + } + + inline JunctionList::iterator end() + { + return junctions.end(); + } + + const JunctionList& get_junctions() { + return junctions; + } + +}; + +typedef std::unordered_multimap LinkMap; +typedef std::list LinkList; +typedef std::unordered_map JunctionMap; +typedef std::pair LinkMapPair; + +class GraphLinker +{ +protected: + // map from starting high degree nodes to associated links + LinkMap links; + // map from junction keys to Junction* + JunctionMap junctions; + std::shared_ptr graph; + uint64_t n_sequences_added; + +public: + + GraphLinker(std::shared_ptr graph) : + graph(graph), n_sequences_added(0) + { + + } + + void report() const + { + std::cout << "GraphLinker (@" << this << " with " + << "Hashgraph @" << graph.get() << ")" << std::endl; + std::cout << " * " << junctions.size() << " junctions" << std::endl; + std::cout << " * " << n_sequences_added << " sequences added" << std::endl; + std::cout << " * " << links.size() << " links" << std::endl; + } + + Junction* get_junction(HashIntoType key) const + { + auto search = junctions.find(key); + if (search != junctions.end()) { + return search->second; + } + return nullptr; + } + + Junction* get_junction(HashIntoType u, HashIntoType v) const + { + return get_junction(u ^ v); + } + + Junction* get_junction(Junction& junction) const + { + return get_junction(junction.id()); + } + + Junction* new_junction(HashIntoType u, HashIntoType v) + { + Junction* j = new Junction(); + j->u = u; + j->v = v; + j->count = 0; + junctions[j->id()] = j; + return j; + } + + Junction* fetch_or_new_junction(HashIntoType u, HashIntoType v) + { + Junction* j = get_junction(u, v); + if (j != nullptr) { + j->count = j->count + 1; + } else { + j = new_junction(u, v); + j->count = 1; + } + return j; + } + + std::shared_ptr get_junctions(const std::string& sequence) const + { + KmerIterator kmers(sequence.c_str(), graph->ksize()); + std::shared_ptr junctions = make_shared(); + + Kmer u = kmers.next(); + if (kmers.done()) { + return junctions; + } + Kmer v = kmers.next(); + + Junction* j; + + while(!kmers.done()) { + j = get_junction(u, v); + if (j != nullptr) { + junctions->push_back(j); + } + u = v; + v = kmers.next(); + } + + return junctions; + } + + void build_links(const std::string& sequence, + Link* fw_link, Link* rc_link) + { + KmerIterator kmers(sequence.c_str(), graph->ksize()); + Kmer u, v; + + u = kmers.next(); + if (kmers.done()) { + fw_link = rc_link = nullptr; + return; + } + v = kmers.next(); + + fw_link = new Link(n_sequences_added); + rc_link = new Link(n_sequences_added, false); + Traverser traverser(graph.get()); + + while(!kmers.done()) { + if (traverser.degree_right(u) > 1) { + fw_link->push_back(fetch_or_new_junction(u, v)); + } + if (traverser.degree_left(v) > 1) { + rc_link->push_front(fetch_or_new_junction(v, u)); + } + + u = v; + v = kmers.next(); + } + + if (fw_link->size() < 2) { + delete fw_link; + fw_link = nullptr; + } + + if (rc_link->size() < 2) { + delete rc_link; + rc_link = nullptr; + } + } + + void add_links(const std::string& sequence) + { + std::cout << "add_links('" << sequence << "')" << std::endl; + Link * fw_link = nullptr; + Link * rc_link = nullptr; + n_sequences_added++; + build_links(sequence, fw_link, rc_link); + + if (fw_link != nullptr) { + std::cout << " - add_links: found fw_link" << std::endl; + Junction* start = fw_link->start_junction(); + std::cout << " * start junction: " << start << std::endl; + links.insert(LinkMapPair(start->u, fw_link)); + } + if (rc_link != nullptr) { + std::cout << " - add_links: found rc_link" << std::endl; + Junction* start = rc_link->start_junction(); + std::cout << " * start junction: " << start << std::endl; + links.insert(LinkMapPair(start->u, rc_link)); + } + } + + void get_links(std::list high_degree_nodes, + std::shared_ptr found_links) + { + for (HashIntoType hdn : high_degree_nodes) { + auto range = links.equal_range(hdn); + for (auto it = range.first; it != range.second; ++it) { + found_links->push_back(it->second); + } + } + } + + std::shared_ptr get_links(const std::string& sequence) + { + std::cout << "get_links('" << sequence << "')" << std::endl; + KmerIterator kmers(sequence.c_str(), graph->ksize()); + Traverser traverser(graph.get()); + std::list hdns; + + + std::cout << " - get_link: start iterating k-mers" << std::endl; + Kmer kmer = kmers.next(); + while(!kmers.done()) { + if (traverser.degree(kmer) > 2) { + hdns.push_back(kmer); + } + kmer = kmers.next(); + } + std::cout << " - get_links: found " << hdns.size() << " hdns" << std::endl; + + std::shared_ptr found_links = make_shared(); + get_links(hdns, found_links); + return found_links; + } + +}; + + + +} + + + + +#endif diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index e8e3aac6f2..bdc5f6c564 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -1,40 +1,47 @@ cimport cython from libcpp.memory cimport shared_ptr +from libcpp.list cimport list as stdlist from libc.stdint cimport uint8_t, uint32_t, uint64_t from khmer._oxli.oxli_types cimport * from khmer._oxli.graphs cimport CpHashgraph, Hashgraph, Nodegraph, Countgraph -cdef enum DBGNucl: - A, C, G, T +cdef extern from "oxli/links.hh" namespace "oxli": + ctypedef struct Junction: + HashIntoType u + HashIntoType v + uint64_t count + ctypedef stdlist[Junction*] JunctionList -cdef class Junction: - cdef readonly HashIntoType u - cdef readonly HashIntoType v + cdef cppclass CpLink "oxli::Link": + CpLink(uint64_t, bool) + CpLink(uint64_t) + bool is_forward() + Junction* start_junction() + Junction* end_junction() - @staticmethod - cdef Junction _new(HashIntoType u, - HashIntoType v) + stdlist[Junction*].iterator begin() + stdlist[Junction*].iterator end() + const JunctionList& get_junctions() + ctypedef stdlist[CpLink*] LinkList + + cdef cppclass CpGraphLinker "oxli::GraphLinker": + CpGraphLinker(shared_ptr[CpHashgraph]) -cdef class Link: + Junction* get_junction(HashIntoType) + Junction* get_junction(HashIntoType, HashIntoType) + Junction* get_junction(Junction&) + shared_ptr[JunctionList] get_junctions(const string&) - cdef readonly list junctions - cdef bool forward - - @staticmethod - cdef Link _new(list junctions, bool forward) + void add_links(const string&) + shared_ptr[LinkList] get_links(const string&) + void report() const cdef class GraphLinker: - cdef Hashgraph graph cdef shared_ptr[CpHashgraph] _graph - cdef WordLength K - cdef dict links - - cdef tuple _get_junctions(self, string sequence) - cdef int _add_link(self, string sequence, - unsigned int min_link_size=*) except -1 + cdef shared_ptr[CpGraphLinker] _gl_this diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index d215f501e6..9b278df078 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -1,115 +1,40 @@ -cimport cython - from cython.operator cimport dereference as deref +from libcpp.memory cimport make_shared -from libcpp.memory cimport shared_ptr, make_shared -from libcpp.string cimport string -from libcpp.vector cimport vector - -from khmer._oxli.oxli_types cimport * -from khmer._oxli.hashing cimport CpKmer, CpKmerIterator, _revcomp -from khmer._oxli.sequence cimport Sequence, _object_to_string -from khmer._oxli.sequence import Sequence -from khmer._oxli.traversal cimport CpTraverser from khmer._oxli.utils cimport _bstring - -@cython.freelist(100) -cdef class Junction: - - def __init__(self, HashIntoType u, - HashIntoType v): - self.u = u - self.v = v - - @staticmethod - cdef Junction _new(HashIntoType u, - HashIntoType v): - cdef Junction junc = Junction.__new__(Junction) - junc.u = u - junc.v = v - return junc - - -cdef class Link: - - def __init__(self, list junctions, bool forward): - self.junctions = junctions - self.forward = forward - - @staticmethod - cdef Link _new(list junctions, bool forward): - cdef Link link = Link.__new__(Link) - link.junctions = junctions - link.forward = forward - return link - - cdef class GraphLinker: - def __cinit__(self, Hashgraph graph not None): - self.graph = graph + def __cinit__(self, Hashgraph graph): self._graph = graph._hg_this - self.K = graph.ksize() - - self.links = {} - - def check_sequence_length(self, object sequence): - if len(sequence) < self.K: - raise ValueError("sequence length ({}) must >= the hashtable " - "k-mer size ({})".format(len(sequence), - self.K)) - - cdef tuple _get_junctions(self, string sequence): - cdef shared_ptr[CpTraverser] _traverser = make_shared[CpTraverser](self._graph.get()) - cdef CpKmerIterator * _it = new CpKmerIterator(sequence.c_str(), - self.graph.ksize()) - cdef CpKmer u = deref(_it).next() - cdef CpKmer start = u - if deref(_it).done(): - return [], [] - - cdef list fw_choices = [] - cdef list rc_choices = [] - cdef CpKmer v = deref(_it).next() - #cdef uint64_t kmer_start_idx = 0 - #cdef uint64_t kmer_end_idx = self.K - 1 - - while not deref(_it).done(): - if deref(_traverser).degree_right(u) > 1: - fw_choices.append(Junction._new(u, v)) - if deref(_traverser).degree_left(v) > 1: - rc_choices.append(Junction._new(v, u)) - - u = v - v = deref(_it).next() - #kmer_start_idx += 1 - #kmer_end_idx += 1 - rc_choices.reverse() - return fw_choices, rc_choices - - def get_junctions(self, object sequence): - self.check_sequence_length(sequence) - return self._get_junctions(_object_to_string(sequence)) - - cdef int _add_link(self, string sequence, unsigned int min_link_size=1) except -1: - - cdef Link fw_link, rc_link - fw_choices, rc_choices = self._get_junctions(sequence) - - if len(fw_choices) >= min_link_size: - fw_link = Link._new(fw_choices, True) - self.links[fw_choices[0].u] = fw_link - if len(rc_choices) >= min_link_size: - rc_link = Link._new(rc_choices, False) - self.links[rc_choices[0].u] = rc_link - - return len(fw_choices) + len(rc_choices) - - def add_link(self, object sequence, unsigned int min_link_size=1): - if min_link_size < 1: - raise ValueError("Minimum link size must be at least 1.") - self.check_sequence_length(sequence) - - return self._add_links(_object_to_string(sequence), min_link_size) - + + if type(self) is GraphLinker: + self._gl_this = make_shared[CpGraphLinker](self._graph) + + def add_links(self, str sequence): + cdef string _sequence = _bstring(sequence) + deref(self._gl_this).add_links(_sequence) + + def get_junctions(self, str sequence): + cdef string _sequence = _bstring(sequence) + cdef shared_ptr[JunctionList] junctions = deref(self._gl_this).get_junctions(_sequence) + cdef Junction* j + for j in deref(junctions): + yield deref(j) + + def get_links(self, str sequence): + cdef string _sequence = _bstring(sequence) + cdef shared_ptr[LinkList] links = deref(self._gl_this).get_links(_sequence) + + cdef CpLink* link + cdef stdlist[Junction*].iterator it + for link in deref(links): + ret = [] + it = deref(link).begin() + while it != deref(link).end(): + ret.append(deref(deref(it))) + yield ret + + + def report(self): + deref(self._gl_this).report() diff --git a/setup.py b/setup.py index 373f55915c..704385e140 100755 --- a/setup.py +++ b/setup.py @@ -160,7 +160,7 @@ def build_dir(): "khmer", "kmer_hash", "hashtable", "labelhash", "hashgraph", "hllcounter", "oxli_exception", "read_aligner", "subset", "read_parsers", "kmer_filters", "traversal", "assembler", "alphabets", "storage", - "partitioning", "gmap", "hist"]) + "partitioning", "gmap", "hist", "links"]) SOURCES = [path_join("src", "khmer", bn + ".cc") for bn in [ "_cpy_khmer", "_cpy_utils", "_cpy_readparsers" @@ -169,7 +169,7 @@ def build_dir(): "read_parsers", "kmer_hash", "hashtable", "hashgraph", "labelhash", "subset", "read_aligner", "oxli", "hllcounter", "traversal", "kmer_filters", "assembler", "alphabets", - "storage", "partitioning"]) + "storage", "partitioning", "links"]) SOURCES.extend(path_join("third-party", "smhasher", bn + ".cc") for bn in [ "MurmurHash3"]) diff --git a/src/oxli/links.cc b/src/oxli/links.cc new file mode 100644 index 0000000000..27e9eadcf2 --- /dev/null +++ b/src/oxli/links.cc @@ -0,0 +1,28 @@ +#include "oxli/links.hh" + +namespace oxli { + +uint64_t Link::n_links = 0; + + +inline std::ostream& operator<< (std::ostream& stream, + const Junction* j) +{ + stream << "Junction(u=" << j->u << ", v=" << j->v << + ", count=" << j->count << ", id=" << j->id() << ")"; +} + + + + + + + + + + + + + + +} diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 52ca49aeba..f50b9af4cf 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -8,7 +8,7 @@ from . import khmer_tst_utils as utils from .graph_features import * -from khmer._oxli.graphlinks import Link, Junction, GraphLinker +from khmer._oxli.graphlinks import GraphLinker from khmer import Nodegraph import pytest @@ -17,31 +17,22 @@ def teardown(): utils.cleanup() -def test_get_junctions(right_tip_structure): +def test_add_links_one_junction(right_tip_structure): + '''Should have no links. Need two junctions. + ''' graph, contig, L, HDN, R, tip = right_tip_structure linker = GraphLinker(graph) - fw_choices, rc_choices = linker.get_junctions(contig) - assert len(fw_choices) == 1 - assert len(rc_choices) == 0 - link = fw_choices.pop() + linker.add_links(contig) + linker.report() - assert link.u == forward_hash(HDN, K) - assert link.v == forward_hash(R, K) - - -def test_get_junctions_rc(left_tip_structure): - graph, contig, L, HDN, R, tip = left_tip_structure - linker = GraphLinker(graph) - - fw_choices, rc_choices = linker.get_junctions(contig) - assert len(fw_choices) == 0 - - link = rc_choices.pop() - - assert link.u == forward_hash(HDN, K) - assert link.v == forward_hash(L, K) + links = list(linker.get_links(contig)) + assert len(links) == 0 + + junctions = list(linker.get_junctions(contig)) + assert len(junctions) == 1 + print(junctions) From bf16d16b84b4d95ac13db71b4e5fe6a730eb12ce Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 28 Sep 2017 15:43:25 -0700 Subject: [PATCH 128/185] expand test to check count --- tests/test_linked_dbg.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index f50b9af4cf..68b12c0e67 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -17,7 +17,7 @@ def teardown(): utils.cleanup() -def test_add_links_one_junction(right_tip_structure): +def test_get_junctions_single(right_tip_structure): '''Should have no links. Need two junctions. ''' graph, contig, L, HDN, R, tip = right_tip_structure @@ -31,8 +31,17 @@ def test_add_links_one_junction(right_tip_structure): junctions = list(linker.get_junctions(contig)) assert len(junctions) == 1 + junction = junctions.pop() + assert junction['count'] == 1 + assert junction['u'] == forward_hash(HDN, K) + assert junction['v'] == forward_hash(R, K) - print(junctions) - + linker.add_links(contig) + links = list(linker.get_links(contig)) + assert len(links) == 0 + junctions = list(linker.get_junctions(contig)) + assert len(junctions) == 1 + junction = junctions.pop() + assert junction['count'] == 2 From f3f52f8668b428b12c51acdae0819f7bd7f412de Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 28 Sep 2017 17:03:15 -0700 Subject: [PATCH 129/185] Require new junctions in order to create Link --- include/oxli/links.hh | 41 +++++++++++++++++++++++++------------- khmer/_oxli/graphlinks.pxd | 3 ++- khmer/_oxli/graphlinks.pyx | 11 +++++----- src/oxli/links.cc | 26 ++++++++++-------------- tests/test_linked_dbg.py | 21 ++++++++++++++++--- 5 files changed, 63 insertions(+), 39 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 5580d30fb6..f3dc1cfc28 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -60,10 +60,12 @@ struct Junction { Junction() = default; HashIntoType id() const { return u ^ v; } + friend std::ostream& operator<< (std::ostream& stream, - const Junction* j); + const Junction& j); }; + typedef std::list JunctionList; @@ -147,7 +149,7 @@ public: return junctions.end(); } - const JunctionList& get_junctions() { + JunctionList& get_junctions() { return junctions; } @@ -214,12 +216,13 @@ public: return j; } - Junction* fetch_or_new_junction(HashIntoType u, HashIntoType v) + Junction* fetch_or_new_junction(HashIntoType u, HashIntoType v, uint64_t& counter) { Junction* j = get_junction(u, v); if (j != nullptr) { j->count = j->count + 1; } else { + counter++; j = new_junction(u, v); j->count = 1; } @@ -252,8 +255,9 @@ public: } void build_links(const std::string& sequence, - Link* fw_link, Link* rc_link) + Link* &fw_link, Link* &rc_link) { + std::cout << "build_links()" << std::endl; KmerIterator kmers(sequence.c_str(), graph->ksize()); Kmer u, v; @@ -263,34 +267,42 @@ public: return; } v = kmers.next(); - + + std::cout << " - build_links: allocate new Link*" << std::endl; fw_link = new Link(n_sequences_added); rc_link = new Link(n_sequences_added, false); - Traverser traverser(graph.get()); + uint64_t n_new_fw = 0; + uint64_t n_new_rc = 0; + Traverser traverser(graph.get()); while(!kmers.done()) { if (traverser.degree_right(u) > 1) { - fw_link->push_back(fetch_or_new_junction(u, v)); + std::cout << " - build_links: found FW HDN " << u << std::endl; + fw_link->push_back(fetch_or_new_junction(u, v, n_new_fw)); } if (traverser.degree_left(v) > 1) { - rc_link->push_front(fetch_or_new_junction(v, u)); + std::cout << " - build_links: found RC HDN " << v << std::endl; + rc_link->push_front(fetch_or_new_junction(v, u, n_new_rc)); } u = v; v = kmers.next(); } - if (fw_link->size() < 2) { + if (fw_link->size() < 1 || n_new_fw == 0) { + std::cout << " - build_links: (fw_link) no (new) junctions found." << std::endl; delete fw_link; fw_link = nullptr; } - if (rc_link->size() < 2) { + if (rc_link->size() < 1 || n_new_rc == 0) { + std::cout << " - build_links: (rc_link) no (new) junctions found." << std::endl; delete rc_link; rc_link = nullptr; } } + void add_links(const std::string& sequence) { std::cout << "add_links('" << sequence << "')" << std::endl; @@ -300,15 +312,15 @@ public: build_links(sequence, fw_link, rc_link); if (fw_link != nullptr) { - std::cout << " - add_links: found fw_link" << std::endl; + std::cout << " - add_links: insert found fw_link" << std::endl; Junction* start = fw_link->start_junction(); - std::cout << " * start junction: " << start << std::endl; + std::cout << " * start junction: " << &start << std::endl; links.insert(LinkMapPair(start->u, fw_link)); } if (rc_link != nullptr) { - std::cout << " - add_links: found rc_link" << std::endl; + std::cout << " - add_links: insert found rc_link" << std::endl; Junction* start = rc_link->start_junction(); - std::cout << " * start junction: " << start << std::endl; + std::cout << " * start junction: " << &start << std::endl; links.insert(LinkMapPair(start->u, rc_link)); } } @@ -344,6 +356,7 @@ public: std::shared_ptr found_links = make_shared(); get_links(hdns, found_links); + std::cout << " - get_links: returning " << found_links->size() << " links" << std::endl; return found_links; } diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index bdc5f6c564..6bf72411b6 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -21,10 +21,11 @@ cdef extern from "oxli/links.hh" namespace "oxli": bool is_forward() Junction* start_junction() Junction* end_junction() + uint64_t size() stdlist[Junction*].iterator begin() stdlist[Junction*].iterator end() - const JunctionList& get_junctions() + JunctionList& get_junctions() ctypedef stdlist[CpLink*] LinkList diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 9b278df078..3b1208fdd4 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -27,13 +27,14 @@ cdef class GraphLinker: cdef shared_ptr[LinkList] links = deref(self._gl_this).get_links(_sequence) cdef CpLink* link + cdef Junction* j cdef stdlist[Junction*].iterator it for link in deref(links): - ret = [] - it = deref(link).begin() - while it != deref(link).end(): - ret.append(deref(deref(it))) - yield ret + if deref(link).size() > 0: + ret = [] + for j in deref(link).get_junctions(): + ret.append(deref(j)) + yield ret def report(self): diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 27e9eadcf2..56aae206bf 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -1,26 +1,20 @@ -#include "oxli/links.hh" - -namespace oxli { -uint64_t Link::n_links = 0; +#include +#include "oxli/links.hh" +using namespace oxli; -inline std::ostream& operator<< (std::ostream& stream, - const Junction* j) +std::ostream& operator<<(std::ostream& stream, + const oxli::Junction& j) { - stream << "Junction(u=" << j->u << ", v=" << j->v << - ", count=" << j->count << ", id=" << j->id() << ")"; + stream << "Junction(u=" << j.u << ", v=" << j.v << + ", count=" << j.count << ", id=" << j.id() << ")"; + return stream; } +namespace oxli { - - - - - - - - +uint64_t Link::n_links = 0; diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 68b12c0e67..452ad8edc1 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -27,8 +27,8 @@ def test_get_junctions_single(right_tip_structure): linker.report() links = list(linker.get_links(contig)) - assert len(links) == 0 - + assert len(links) == 1 + junctions = list(linker.get_junctions(contig)) assert len(junctions) == 1 junction = junctions.pop() @@ -37,11 +37,26 @@ def test_get_junctions_single(right_tip_structure): assert junction['v'] == forward_hash(R, K) linker.add_links(contig) + linker.report() + links = list(linker.get_links(contig)) - assert len(links) == 0 + print(links) + assert len(links) == 1 junctions = list(linker.get_junctions(contig)) assert len(junctions) == 1 junction = junctions.pop() assert junction['count'] == 2 +def test_links_bubble(snp_bubble_structure): + graph, wildtype_sequence, _, HDN_L, HDN_R = snp_bubble_structure + linker = GraphLinker(graph) + + # thread the wildtype half of the bubble + linker.add_links(wildtype_sequence) + linker.report() + + + links = list(linker.get_links(wildtype_sequence)) + assert len(links) == 2 + From 9e552328baf670febae4d258d81956c8b69aaa30 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 28 Sep 2017 17:11:50 -0700 Subject: [PATCH 130/185] Link content assert in test --- tests/test_linked_dbg.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 452ad8edc1..295e7c57f8 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -60,3 +60,8 @@ def test_links_bubble(snp_bubble_structure): links = list(linker.get_links(wildtype_sequence)) assert len(links) == 2 + link_a, link_b = links + if link_a[0]['u'] == forward_hash(HDN_L, K): + assert link_b[0]['u'] == forward_hash(HDN_R, K) + elif link_a[0]['u'] == forward_hash(HDN_R, K): + assert link_b[0]['u'] == forward_hash(HDN_L, K) From 6fc247d899b7558d8697193fb2c57585dd3d8097 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 29 Sep 2017 12:43:54 -0700 Subject: [PATCH 131/185] Sketch out linked assembler --- include/oxli/links.hh | 51 ++++++++++++++++++++++++++++++++++------ src/oxli/kmer_filters.cc | 6 +++++ src/oxli/links.cc | 42 +++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index f3dc1cfc28..2757ab602e 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -48,6 +48,7 @@ Contact: khmer-project@idyll.org #include "hashgraph.hh" #include "kmer_filters.hh" #include "traversal.hh" +#include "assembler.hh" namespace oxli { @@ -167,10 +168,11 @@ protected: LinkMap links; // map from junction keys to Junction* JunctionMap junctions; - std::shared_ptr graph; uint64_t n_sequences_added; public: + + std::shared_ptr graph; GraphLinker(std::shared_ptr graph) : graph(graph), n_sequences_added(0) @@ -178,6 +180,11 @@ public: } + WordLength ksize() const + { + return graph->ksize(); + } + void report() const { std::cout << "GraphLinker (@" << this << " with " @@ -269,8 +276,8 @@ public: v = kmers.next(); std::cout << " - build_links: allocate new Link*" << std::endl; - fw_link = new Link(n_sequences_added); - rc_link = new Link(n_sequences_added, false); + fw_link = new Link(n_sequences_added, u.is_forward()); + rc_link = new Link(n_sequences_added, !u.is_forward()); uint64_t n_new_fw = 0; uint64_t n_new_rc = 0; @@ -325,14 +332,19 @@ public: } } + void get_links(Kmer hdn, std::shared_ptr found_links) + { + auto range = links.equal_range(hdn); + for (auto it = range.first; it != range.second; ++it) { + found_links->push_back(it->second); + } + } + void get_links(std::list high_degree_nodes, std::shared_ptr found_links) { for (HashIntoType hdn : high_degree_nodes) { - auto range = links.equal_range(hdn); - for (auto it = range.first; it != range.second; ++it) { - found_links->push_back(it->second); - } + get_links(hdn, found_links); } } @@ -363,6 +375,31 @@ public: }; +class LinkedAssembler +{ + std::shared_ptr linear_asm; + +public: + + const std::shared_ptr graph; + const std::shared_ptr linker; + WordLength _ksize; + + explicit LinkedAssembler(std::shared_ptr linker) : + _ksize(linker->ksize()), graph(linker->graph), linker(linker) + { + linear_asm = make_shared(graph.get()); + } + + StringVector assemble(const Kmer seed_kmer, + std::shared_ptr stop_bf=nullptr) const; + + template + void _assemble_directed(AssemblerTraverser& start_cursor, + StringVector& paths) const; +}; + + } diff --git a/src/oxli/kmer_filters.cc b/src/oxli/kmer_filters.cc index 987ec327f2..6d83e62c58 100644 --- a/src/oxli/kmer_filters.cc +++ b/src/oxli/kmer_filters.cc @@ -118,6 +118,12 @@ KmerFilter get_simple_label_intersect_filter(const LabelSet& src_labels, return filter; } +KmerFilter get_link_filter(std::shared_ptr links, + const unsigned int min_count) +{ + +} + KmerFilter get_junction_count_filter(const Kmer& src_node, Countgraph * junctions, diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 56aae206bf..0aa1374c0e 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -16,7 +16,49 @@ namespace oxli { uint64_t Link::n_links = 0; +template +void LinkedAssembler:: +_assemble_directed(AssemblerTraverser start_cursor, + StringVector& paths) +const +{ + std::string root_contig = linear_asm->assemble_directed(start_cursor); + + StringVector segments; + std::vector< AssemblerTraverser > cursors; + + segments.push_back(root_contig); + cursors.push_back(start_cursor); + + while(segments.size() != 0) { + + std::string segment = segments.back(); + AssemblerTraverser cursor = cursors.back(); +#if DEBUG_ASSEMBLY + std::cout << "Pop: " << segments.size() << " segments on stack." << std::endl; + std::cout << "Segment: " << segment << std::endl; + std::cout << "Cursor: " << cursor.cursor.repr(_ksize) << std::endl; + std::cout << "n_filters: " << cursor.n_filters() << std::endl; +#endif + segments.pop_back(); + cursors.pop_back(); + + if (cursor.cursor_degree() > 1) { + std::shared_ptr links = make_shared(); + linker->get_links(hdn, links); + + if (links->size() == 0) { + paths.push_back(segment); + continue; + } else { + + } + + } + + } +} } From 12316d4a30d8611e744b73e915ef1005303a3008 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 29 Sep 2017 16:24:35 -0700 Subject: [PATCH 132/185] Update assembler sketching --- include/oxli/links.hh | 102 ++++++++++++++++++++++++++++++++++++--- src/oxli/kmer_filters.cc | 11 +++-- src/oxli/links.cc | 40 +++++++++------ 3 files changed, 128 insertions(+), 25 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 2757ab602e..d55b85f400 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -82,12 +82,15 @@ protected: uint64_t link_id; uint64_t time_created; // in "read space" ie read number //size_t extent; + uint64_t flanking_distance; public: - Link(uint64_t time_created, bool forward=true) : + Link(uint64_t time_created, + bool forward=true) : forward(forward), link_id(n_links), - time_created(time_created) + time_created(time_created), + flanking_distance(0) { n_links++; } @@ -100,6 +103,11 @@ public: junctions.push_back(junction); } + inline void set_flanking_distance(uint64_t d) + { + flanking_distance = d; + } + inline void push_front(Junction* junction) { //if (junctions.size() > 0) { @@ -140,6 +148,11 @@ public: return junctions.size(); } + inline uint64_t time_created() const + { + return time_created; + } + inline JunctionList::iterator begin() { return junctions.begin(); @@ -156,6 +169,46 @@ public: }; + +class LinkCursor +{ +public: + Link* link; + uint64_t age; + JunctionList::iterator cursor; + + LinkTraversal(Link* link, uint64_t age) : + link(link), age(age), cursor(link->begin()) + { + } + + bool done() { + return cursor == link->end(); + } +}; + + +class LinkTraversal +{ + std::shared_ptr> cursors; + + LinkTraversal() + { + cursors = std::make_shared>(); + + } + + void add_links(std::shared_ptr links, + uint64_t age) + { + for (Link* link: &links) { + cursors->push_back(LinkCursor(link, age)); + } + } + +}; + + typedef std::unordered_multimap LinkMap; typedef std::list LinkList; typedef std::unordered_map JunctionMap; @@ -267,33 +320,42 @@ public: std::cout << "build_links()" << std::endl; KmerIterator kmers(sequence.c_str(), graph->ksize()); Kmer u, v; + uint64_t d = 0; u = kmers.next(); + ++d; if (kmers.done()) { fw_link = rc_link = nullptr; return; } v = kmers.next(); + ++d; std::cout << " - build_links: allocate new Link*" << std::endl; fw_link = new Link(n_sequences_added, u.is_forward()); rc_link = new Link(n_sequences_added, !u.is_forward()); uint64_t n_new_fw = 0; uint64_t n_new_rc = 0; + uint64_t last_rc_pos = 0; Traverser traverser(graph.get()); while(!kmers.done()) { if (traverser.degree_right(u) > 1) { std::cout << " - build_links: found FW HDN " << u << std::endl; fw_link->push_back(fetch_or_new_junction(u, v, n_new_fw)); + if (n_new_fc == 1) { + fw_link->set_flanking_distance(d); + } } if (traverser.degree_left(v) > 1) { std::cout << " - build_links: found RC HDN " << v << std::endl; rc_link->push_front(fetch_or_new_junction(v, u, n_new_rc)); + last_rc_pos = d; } u = v; v = kmers.next(); + ++d; } if (fw_link->size() < 1 || n_new_fw == 0) { @@ -306,6 +368,8 @@ public: std::cout << " - build_links: (rc_link) no (new) junctions found." << std::endl; delete rc_link; rc_link = nullptr; + } else { + rc_link->set_flanking_distance(d - last_rc_pos); } } @@ -332,20 +396,46 @@ public: } } - void get_links(Kmer hdn, std::shared_ptr found_links) + uint64_t get_links(Kmer hdn, std::shared_ptr found_links) { auto range = links.equal_range(hdn); + uint64_t n_links_found = 0; for (auto it = range.first; it != range.second; ++it) { found_links->push_back(it->second); + n_link_found++; } + return n_links_found; } - void get_links(std::list high_degree_nodes, + uint64_t get_links_copy(Kmer hdn, std::shared_ptr> found_links) + { + auto range = links.equal_range(hdn); + uint64_t n_links_found = 0; + for (auto it = range.first; it != range.second; ++it) { + found_links->push_back(&(it->second)); + n_link_found++; + } + return n_links_found; + } + + uint64_t get_links(std::list high_degree_nodes, std::shared_ptr found_links) { + uint64_t n_links_found = 0; + for (HashIntoType hdn : high_degree_nodes) { + n_links_founnd += get_links(hdn, found_links); + } + return n_links_found; + } + + uint64_t get_links_copy(std::list high_degree_nodes, + std::shared_ptr found_links) + { + uint64_t n_links_found = 0; for (HashIntoType hdn : high_degree_nodes) { - get_links(hdn, found_links); + n_links_founnd += get_links_copy(hdn, found_links); } + return n_links_found; } std::shared_ptr get_links(const std::string& sequence) @@ -395,7 +485,7 @@ public: std::shared_ptr stop_bf=nullptr) const; template - void _assemble_directed(AssemblerTraverser& start_cursor, + void _assemble_directed(AssemblerTraverser& cursor, StringVector& paths) const; }; diff --git a/src/oxli/kmer_filters.cc b/src/oxli/kmer_filters.cc index 6d83e62c58..2386325355 100644 --- a/src/oxli/kmer_filters.cc +++ b/src/oxli/kmer_filters.cc @@ -118,12 +118,17 @@ KmerFilter get_simple_label_intersect_filter(const LabelSet& src_labels, return filter; } -KmerFilter get_link_filter(std::shared_ptr links, +/* +KmerFilter get_link_filter(const Kmer& src_node, + std::shared_ptr links, + std::shared_ptr< std::list > ages, const unsigned int min_count) { - + KmerFilter filter = [=] (const Kmer& node) { + + } } - +*/ KmerFilter get_junction_count_filter(const Kmer& src_node, Countgraph * junctions, diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 0aa1374c0e..22d5fea75d 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -18,44 +18,52 @@ uint64_t Link::n_links = 0; template void LinkedAssembler:: -_assemble_directed(AssemblerTraverser start_cursor, +_assemble_directed(AssemblerTraverser cursor, StringVector& paths) const { - std::string root_contig = linear_asm->assemble_directed(start_cursor); + std::string root_contig = linear_asm->assemble_directed(cursor); StringVector segments; - std::vector< AssemblerTraverser > cursors; - segments.push_back(root_contig); - cursors.push_back(start_cursor); - - while(segments.size() != 0) { + std::shared_ptr links = make_shared(); + std::shared_ptr< std::list > ages = make_shared >(); - std::string segment = segments.back(); - AssemblerTraverser cursor = cursors.back(); + paths.push_back(root_contig); + + while(1) { + #if DEBUG_ASSEMBLY std::cout << "Pop: " << segments.size() << " segments on stack." << std::endl; std::cout << "Segment: " << segment << std::endl; std::cout << "Cursor: " << cursor.cursor.repr(_ksize) << std::endl; std::cout << "n_filters: " << cursor.n_filters() << std::endl; #endif - segments.pop_back(); - cursors.pop_back(); - +/* if (cursor.cursor_degree() > 1) { - std::shared_ptr links = make_shared(); - linker->get_links(hdn, links); + linker->get_links_copy(hdn, links); if (links->size() == 0) { paths.push_back(segment); continue; - } else { - } + cursor.push_filter(get_link_filter(cursor.cursor, + links, + ages); + KmerQueue linked_nodes; + cursor.neighbors(linked_nodes); + cursor.pop_filter(); + + if (branch_starts.empty()) { + paths.push_back(segment); + } + + } + */ + break; } } From 56e6c75b5224281a083db51c8f71fe2802145663 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 2 Oct 2017 13:13:08 -0700 Subject: [PATCH 133/185] work on link trversal abstractions --- include/oxli/links.hh | 17 ++++++++++++++--- src/oxli/links.cc | 17 +++++++---------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index d55b85f400..2d2aa131ce 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -190,11 +190,12 @@ public: class LinkTraversal { - std::shared_ptr> cursors; + std::shared_ptr> link_cursors; + std::shared_ptr> constraints; // constraint junction ids LinkTraversal() { - cursors = std::make_shared>(); + link_cursors = std::make_shared>(); } @@ -202,10 +203,20 @@ class LinkTraversal uint64_t age) { for (Link* link: &links) { - cursors->push_back(LinkCursor(link, age)); + link_cursors->push_back(LinkCursor(link, age)); } } + template + void try_link_neighbors(AssemblerTraverser cursor) + { + KmerQueue neighbors; + cursor.neighbors(neighbors); + + } + + + }; diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 22d5fea75d..b0c983feba 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -22,14 +22,9 @@ _assemble_directed(AssemblerTraverser cursor, StringVector& paths) const { - std::string root_contig = linear_asm->assemble_directed(cursor); + std::string contig = linear_asm->assemble_directed(cursor); - StringVector segments; - - std::shared_ptr links = make_shared(); - std::shared_ptr< std::list > ages = make_shared >(); - - paths.push_back(root_contig); + LinkTraversal link_traversal; while(1) { @@ -40,9 +35,12 @@ const std::cout << "n_filters: " << cursor.n_filters() << std::endl; #endif -/* if (cursor.cursor_degree() > 1) { - linker->get_links_copy(hdn, links); + // hit a HDN, push new links + std::shared_ptr new_links; + linker->get_links(hdn, new_links); + // add them to traversal, with age being current contig size + link_traversal.add_links(new_links, contig.size()); if (links->size() == 0) { paths.push_back(segment); @@ -62,7 +60,6 @@ const } - */ break; } From 979199f612074fc72b694319736eb53f51a0e1f7 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 6 Oct 2017 17:20:03 -0700 Subject: [PATCH 134/185] add left flanking node to junction --- include/oxli/links.hh | 164 ++++++++++++++++++++++++++++++++++++++---- src/oxli/links.cc | 3 +- 2 files changed, 151 insertions(+), 16 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 2d2aa131ce..76eb87169c 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -37,6 +37,7 @@ Contact: khmer-project@idyll.org #ifndef LINKS_HH #define LINKS_HH +#include #include #include #include @@ -54,21 +55,66 @@ Contact: khmer-project@idyll.org namespace oxli { struct Junction { + // [u]-->[v (HDN)]-->[w] HashIntoType u; HashIntoType v; - //uint32_t distance_prev; + HashIntoType w; uint64_t count; Junction() = default; - HashIntoType id() const { return u ^ v; } - + HashIntoType id() const { return u ^ v ^ w; } + bool matches(HashIntoType u, HashIntoType v) { return (u ^ v ^ w) == id(); } friend std::ostream& operator<< (std::ostream& stream, const Junction& j); + friend bool operator== (const Junction& lhs, + const Junction& rhs) { + return lhs.id() == rhs.id(); + } }; typedef std::list JunctionList; +#define FW 1 +#define RC 0 + +class LinkNode { +private: + static uint64_t counter; + +public: + + LinkNode* next; + Junction* junction; + HashIntoType from; + uint64_t node_id; + + LinkNode(Junction* junction, HashIntoType from) : + junction(junction), from(from), time_created(time_created), + node_id(counter), next(nullptr) + { + ++counter; + } +}; + + +class LinkHead { +private: + static uint64_t counter; +public: + uint64_t link_id; + LinkNode* start; + HashIntoType from; + bool forward; + + LinkHead(LinkNode* start, HashIntoType from, bool forward, + uint64_t time_created) : + start(start), from(from), forward(forward), time_created(time_created) + { + ++counter; + } +}; + class Link { private: @@ -174,50 +220,131 @@ class LinkCursor { public: Link* link; - uint64_t age; + uint64_t traversal_age; JunctionList::iterator cursor; LinkTraversal(Link* link, uint64_t age) : - link(link), age(age), cursor(link->begin()) + link(link), traversal_age(age), cursor(link->begin()) { } bool done() { return cursor == link->end(); } + + Junction* current() const { + return &cursor; + } + + bool increment() { + ++cursor; + return done(); + } + + friend bool operator==(const LinkCursor& lhs, const LinkCursor& rhs) + { + return lhs->link == rhs->link; + } }; +// Compare the two link cursors first by their age within the +// traversal, then by their global age. We prefer links +// that were found earlier in the traversal, but were created +// most recently. +bool CompareLinks(const LinkCursor& a, const LinkCursor& b) +{ + if (a.traversal_age < b.traversal_age) { return false; } + if (b.traversal_age < a.traversal_age) { return true; } + + if ((a.link)->time_created() > (b.link)->time_created()) { return false; } + if ((b.link)->time_created() > (a.link)->time_created()) { return true; } + + return false; +} + +/* class LinkTraversal { - std::shared_ptr> link_cursors; + typedef std::priority_queue, + CompareLinks> CursorQueue; + std::shared_ptr link_cursors; std::shared_ptr> constraints; // constraint junction ids + LinkCursor last_link; - LinkTraversal() + LinkTraversal() : has_active_cursor(false) { - link_cursors = std::make_shared>(); - + link_cursors = std::make_shared(); } void add_links(std::shared_ptr links, uint64_t age) { for (Link* link: &links) { - link_cursors->push_back(LinkCursor(link, age)); + link_cursors->push(LinkCursor(link, age)); + } + } + + void pop_all(Junction* to_pop) + { + for (LinkCursor cursor : &link_cursors) { + if (!cursor.done() && (to_pop == cursor.current())) { + cursor.increment(); + } + } + } + + bool get_top_link(&LinkCursor result) + { + while(link_cusors->size() > 0) { + if (!link_cusors->top()->done()) { + result = link_cusors->top(); + return true; + } else { + link_cusors->pop(); + } } + return false; } template - void try_link_neighbors(AssemblerTraverser cursor) + bool try_link_neighbors(AssemblerTraverser asmt) { KmerQueue neighbors; - cursor.neighbors(neighbors); + asmt.neighbors(neighbors); + bool decided = false; + Kmer src = asmt.cursor; + Kmer dst; + + LinkCursor link; + bool has_link = get_top_link(link); + if (!has_link) { + return false; + } + + if (has_active_cursor) { + + for (Kmer neighbor : neighbors) { + if ((&active_cursor.current())->matches(neighbor.kmer_f, + neighbor.kmer_r)) { + decided = true; + dst = neighbor; + } + } + + + } else { + + + } } }; +*/ typedef std::unordered_multimap LinkMap; @@ -330,7 +457,7 @@ public: { std::cout << "build_links()" << std::endl; KmerIterator kmers(sequence.c_str(), graph->ksize()); - Kmer u, v; + Kmer u, v, w; uint64_t d = 0; u = kmers.next(); @@ -340,6 +467,11 @@ public: return; } v = kmers.next(); + if (kmers.done()) { + fw_link = rc_link = nullptr; + return; + } + w = kmers.next(); ++d; std::cout << " - build_links: allocate new Link*" << std::endl; @@ -365,7 +497,8 @@ public: } u = v; - v = kmers.next(); + v = w; + w = kmers.next(); ++d; } @@ -475,6 +608,7 @@ public: }; +/* class LinkedAssembler { @@ -499,7 +633,7 @@ public: void _assemble_directed(AssemblerTraverser& cursor, StringVector& paths) const; }; - +*/ } diff --git a/src/oxli/links.cc b/src/oxli/links.cc index b0c983feba..6b6ee2ee7f 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -16,6 +16,7 @@ namespace oxli { uint64_t Link::n_links = 0; +/* template void LinkedAssembler:: _assemble_directed(AssemblerTraverser cursor, @@ -64,6 +65,6 @@ const } } - +*/ } From f49e7f1d632ccb16800ade3b08b2da419d61a42c Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 9 Oct 2017 13:56:23 -0700 Subject: [PATCH 135/185] expand 3 node junction --- include/oxli/links.hh | 115 +++++++++++++++---------------------- khmer/_oxli/graphlinks.pxd | 1 + src/oxli/links.cc | 2 - tests/test_linked_dbg.py | 13 +++-- 4 files changed, 55 insertions(+), 76 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 76eb87169c..4a94c8ebd6 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -78,6 +78,7 @@ typedef std::list JunctionList; #define FW 1 #define RC 0 +/* class LinkNode { private: static uint64_t counter; @@ -114,7 +115,7 @@ public: ++counter; } }; - +*/ class Link { private: @@ -124,13 +125,14 @@ private: protected: JunctionList junctions; - bool forward; - uint64_t link_id; - uint64_t time_created; // in "read space" ie read number - //size_t extent; - uint64_t flanking_distance; public: + + const bool forward; + const uint64_t link_id; + const uint64_t time_created; // in "read space" ie read number + //size_t extent; + uint64_t flanking_distance; Link(uint64_t time_created, bool forward=true) : @@ -149,11 +151,6 @@ public: junctions.push_back(junction); } - inline void set_flanking_distance(uint64_t d) - { - flanking_distance = d; - } - inline void push_front(Junction* junction) { //if (junctions.size() > 0) { @@ -194,11 +191,6 @@ public: return junctions.size(); } - inline uint64_t time_created() const - { - return time_created; - } - inline JunctionList::iterator begin() { return junctions.begin(); @@ -223,7 +215,7 @@ public: uint64_t traversal_age; JunctionList::iterator cursor; - LinkTraversal(Link* link, uint64_t age) : + LinkCursor(Link* link, uint64_t age) : link(link), traversal_age(age), cursor(link->begin()) { } @@ -232,8 +224,9 @@ public: return cursor == link->end(); } - Junction* current() const { - return &cursor; + const Junction* current() const { + const Junction* current = *cursor; + return current; } bool increment() { @@ -243,7 +236,7 @@ public: friend bool operator==(const LinkCursor& lhs, const LinkCursor& rhs) { - return lhs->link == rhs->link; + return lhs.link == rhs.link; } }; @@ -252,13 +245,13 @@ public: // traversal, then by their global age. We prefer links // that were found earlier in the traversal, but were created // most recently. -bool CompareLinks(const LinkCursor& a, const LinkCursor& b) +inline bool CompareLinks(const LinkCursor& a, const LinkCursor& b) { if (a.traversal_age < b.traversal_age) { return false; } if (b.traversal_age < a.traversal_age) { return true; } - if ((a.link)->time_created() > (b.link)->time_created()) { return false; } - if ((b.link)->time_created() > (a.link)->time_created()) { return true; } + if ((a.link)->time_created > (b.link)->time_created) { return false; } + if ((b.link)->time_created > (a.link)->time_created) { return true; } return false; } @@ -394,9 +387,9 @@ public: return nullptr; } - Junction* get_junction(HashIntoType u, HashIntoType v) const + Junction* get_junction(HashIntoType u, HashIntoType v, HashIntoType w) const { - return get_junction(u ^ v); + return get_junction(u ^ v ^ w); } Junction* get_junction(Junction& junction) const @@ -404,24 +397,26 @@ public: return get_junction(junction.id()); } - Junction* new_junction(HashIntoType u, HashIntoType v) + Junction* new_junction(HashIntoType u, HashIntoType v, HashIntoType w) { Junction* j = new Junction(); j->u = u; j->v = v; + j->w = w; j->count = 0; junctions[j->id()] = j; return j; } - Junction* fetch_or_new_junction(HashIntoType u, HashIntoType v, uint64_t& counter) + Junction* fetch_or_new_junction(HashIntoType u, HashIntoType v, + HashIntoType w, uint64_t& counter) { - Junction* j = get_junction(u, v); + Junction* j = get_junction(u, v, w); if (j != nullptr) { j->count = j->count + 1; } else { counter++; - j = new_junction(u, v); + j = new_junction(u, v, w); j->count = 1; } return j; @@ -437,16 +432,21 @@ public: return junctions; } Kmer v = kmers.next(); + if (kmers.done()) { + return junctions; + } + Kmer w = kmers.next(); Junction* j; while(!kmers.done()) { - j = get_junction(u, v); + j = get_junction(u, v, w); if (j != nullptr) { junctions->push_back(j); } u = v; - v = kmers.next(); + v = w; + w = kmers.next(); } return junctions; @@ -467,6 +467,7 @@ public: return; } v = kmers.next(); + ++d; if (kmers.done()) { fw_link = rc_link = nullptr; return; @@ -483,16 +484,16 @@ public: Traverser traverser(graph.get()); while(!kmers.done()) { - if (traverser.degree_right(u) > 1) { + if (traverser.degree_right(v) > 1) { std::cout << " - build_links: found FW HDN " << u << std::endl; - fw_link->push_back(fetch_or_new_junction(u, v, n_new_fw)); - if (n_new_fc == 1) { - fw_link->set_flanking_distance(d); + fw_link->push_back(fetch_or_new_junction(u, v, w, n_new_fw)); + if (n_new_fw == 1) { + fw_link->flanking_distance = d; } } if (traverser.degree_left(v) > 1) { std::cout << " - build_links: found RC HDN " << v << std::endl; - rc_link->push_front(fetch_or_new_junction(v, u, n_new_rc)); + rc_link->push_front(fetch_or_new_junction(w, v, u, n_new_rc)); last_rc_pos = d; } @@ -502,18 +503,18 @@ public: ++d; } - if (fw_link->size() < 1 || n_new_fw == 0) { + if (fw_link->size() < 1) { std::cout << " - build_links: (fw_link) no (new) junctions found." << std::endl; delete fw_link; fw_link = nullptr; } - if (rc_link->size() < 1 || n_new_rc == 0) { + if (rc_link->size() < 1) { std::cout << " - build_links: (rc_link) no (new) junctions found." << std::endl; delete rc_link; rc_link = nullptr; } else { - rc_link->set_flanking_distance(d - last_rc_pos); + rc_link->flanking_distance = d - last_rc_pos; } } @@ -530,13 +531,13 @@ public: std::cout << " - add_links: insert found fw_link" << std::endl; Junction* start = fw_link->start_junction(); std::cout << " * start junction: " << &start << std::endl; - links.insert(LinkMapPair(start->u, fw_link)); + links.insert(LinkMapPair(start->v, fw_link)); } if (rc_link != nullptr) { std::cout << " - add_links: insert found rc_link" << std::endl; Junction* start = rc_link->start_junction(); std::cout << " * start junction: " << &start << std::endl; - links.insert(LinkMapPair(start->u, rc_link)); + links.insert(LinkMapPair(start->v, rc_link)); } } @@ -546,38 +547,17 @@ public: uint64_t n_links_found = 0; for (auto it = range.first; it != range.second; ++it) { found_links->push_back(it->second); - n_link_found++; + n_links_found++; } return n_links_found; } - uint64_t get_links_copy(Kmer hdn, std::shared_ptr> found_links) - { - auto range = links.equal_range(hdn); - uint64_t n_links_found = 0; - for (auto it = range.first; it != range.second; ++it) { - found_links->push_back(&(it->second)); - n_link_found++; - } - return n_links_found; - } - - uint64_t get_links(std::list high_degree_nodes, + uint64_t get_links(std::list high_degree_nodes, std::shared_ptr found_links) { uint64_t n_links_found = 0; - for (HashIntoType hdn : high_degree_nodes) { - n_links_founnd += get_links(hdn, found_links); - } - return n_links_found; - } - - uint64_t get_links_copy(std::list high_degree_nodes, - std::shared_ptr found_links) - { - uint64_t n_links_found = 0; - for (HashIntoType hdn : high_degree_nodes) { - n_links_founnd += get_links_copy(hdn, found_links); + for (Kmer hdn : high_degree_nodes) { + n_links_found += get_links(hdn, found_links); } return n_links_found; } @@ -587,8 +567,7 @@ public: std::cout << "get_links('" << sequence << "')" << std::endl; KmerIterator kmers(sequence.c_str(), graph->ksize()); Traverser traverser(graph.get()); - std::list hdns; - + std::list hdns; std::cout << " - get_link: start iterating k-mers" << std::endl; Kmer kmer = kmers.next(); diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 6bf72411b6..e3dfe0b2ce 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -11,6 +11,7 @@ cdef extern from "oxli/links.hh" namespace "oxli": ctypedef struct Junction: HashIntoType u HashIntoType v + HashIntoType w uint64_t count ctypedef stdlist[Junction*] JunctionList diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 6b6ee2ee7f..68e66feee2 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -15,7 +15,6 @@ std::ostream& operator<<(std::ostream& stream, namespace oxli { uint64_t Link::n_links = 0; - /* template void LinkedAssembler:: @@ -66,5 +65,4 @@ const } } */ - } diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 295e7c57f8..29f4168616 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -33,8 +33,9 @@ def test_get_junctions_single(right_tip_structure): assert len(junctions) == 1 junction = junctions.pop() assert junction['count'] == 1 - assert junction['u'] == forward_hash(HDN, K) - assert junction['v'] == forward_hash(R, K) + assert junction['u'] == forward_hash(L, K) + assert junction['v'] == forward_hash(HDN, K) + assert junction['w'] == forward_hash(R, K) linker.add_links(contig) linker.report() @@ -61,7 +62,7 @@ def test_links_bubble(snp_bubble_structure): assert len(links) == 2 link_a, link_b = links - if link_a[0]['u'] == forward_hash(HDN_L, K): - assert link_b[0]['u'] == forward_hash(HDN_R, K) - elif link_a[0]['u'] == forward_hash(HDN_R, K): - assert link_b[0]['u'] == forward_hash(HDN_L, K) + if link_a[0]['v'] == forward_hash(HDN_L, K): + assert link_b[0]['v'] == forward_hash(HDN_R, K) + elif link_a[0]['v'] == forward_hash(HDN_R, K): + assert link_b[0]['v'] == forward_hash(HDN_L, K) From 9fac2f464a290ed5fd6fbe9d4f6ad88c224643a7 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 12 Oct 2017 14:10:13 -0700 Subject: [PATCH 136/185] move toward link tree --- include/oxli/links.hh | 380 ++++++++++-------------------------------- src/oxli/links.cc | 8 +- 2 files changed, 96 insertions(+), 292 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 4a94c8ebd6..8ebb034577 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -54,21 +54,35 @@ Contact: khmer-project@idyll.org namespace oxli { -struct Junction { +class Junction { +protected: + + static uint64_t junction_counter; + +public: + // [u]-->[v (HDN)]-->[w] HashIntoType u; HashIntoType v; HashIntoType w; - uint64_t count; + const uint64_t ID; + Junction() = default; - HashIntoType id() const { return u ^ v ^ w; } - bool matches(HashIntoType u, HashIntoType v) { return (u ^ v ^ w) == id(); } + + Junction(HashIntoType u, HashIntoType v, HashIntoType w) : + u(u), v(v), w(w), ID(junction_counter++) {} + + uint64_t index() const { return u ^ v ^ w; } + + bool matches(HashIntoType u, + HashIntoType v, + HashintoType w) { return (u ^ v ^ w) == index(); } friend std::ostream& operator<< (std::ostream& stream, const Junction& j); friend bool operator== (const Junction& lhs, const Junction& rhs) { - return lhs.id() == rhs.id(); + return lhs.index() == rhs.index(); } }; @@ -78,173 +92,27 @@ typedef std::list JunctionList; #define FW 1 #define RC 0 -/* -class LinkNode { -private: - static uint64_t counter; - -public: - - LinkNode* next; - Junction* junction; - HashIntoType from; - uint64_t node_id; - - LinkNode(Junction* junction, HashIntoType from) : - junction(junction), from(from), time_created(time_created), - node_id(counter), next(nullptr) - { - ++counter; - } -}; - - -class LinkHead { -private: - static uint64_t counter; -public: - uint64_t link_id; - LinkNode* start; - HashIntoType from; - bool forward; - - LinkHead(LinkNode* start, HashIntoType from, bool forward, - uint64_t time_created) : - start(start), from(from), forward(forward), time_created(time_created) - { - ++counter; - } -}; -*/ - -class Link { -private: - - static uint64_t n_links; - +class LinkTreeSegment { protected: - - JunctionList junctions; - + static uint64_t segment_counter; public: + const uint64_t segment_id; + std::vector + std::vector junctions; // IDs of junctions in this segment + uint64_t children[4]; // child segment IDs + uint32_t count; - const bool forward; - const uint64_t link_id; - const uint64_t time_created; // in "read space" ie read number - //size_t extent; - uint64_t flanking_distance; - - Link(uint64_t time_created, - bool forward=true) : - forward(forward), link_id(n_links), - time_created(time_created), - flanking_distance(0) - { - n_links++; - } - inline void push_back(Junction* junction) - { - //if (junctions.size() == 0) { - // junction->start = true; - //junction->distance_prev = 0; - //} - junctions.push_back(junction); - } - - inline void push_front(Junction* junction) - { - //if (junctions.size() > 0) { - // start_junction()->start = false; - //} - //junction->start = true; // just to be sure - //junction->distance_prev = 0; - junctions.push_front(junction); - } - - void insert_junction(JunctionList::iterator insert_before, - Junction* junction) - { - if (insert_before == begin()) { - push_front(junction); - } else { - junctions.insert(insert_before, junction); - } - } - - inline Junction* start_junction() const - { - return junctions.front(); - } - - inline Junction* end_junction() const - { - return junctions.back(); - } - - inline bool is_forward() const - { - return forward; - } - - inline size_t size() const - { - return junctions.size(); - } - - inline JunctionList::iterator begin() - { - return junctions.begin(); - } - - inline JunctionList::iterator end() - { - return junctions.end(); - } - - JunctionList& get_junctions() { - return junctions; - } + LinkTreeSegment() : segment_id(segment_counter++) {} }; -class LinkCursor -{ -public: - Link* link; - uint64_t traversal_age; - JunctionList::iterator cursor; - - LinkCursor(Link* link, uint64_t age) : - link(link), traversal_age(age), cursor(link->begin()) - { - } - - bool done() { - return cursor == link->end(); - } - - const Junction* current() const { - const Junction* current = *cursor; - return current; - } - - bool increment() { - ++cursor; - return done(); - } - - friend bool operator==(const LinkCursor& lhs, const LinkCursor& rhs) - { - return lhs.link == rhs.link; - } -}; - // Compare the two link cursors first by their age within the // traversal, then by their global age. We prefer links // that were found earlier in the traversal, but were created // most recently. +/* inline bool CompareLinks(const LinkCursor& a, const LinkCursor& b) { if (a.traversal_age < b.traversal_age) { return false; } @@ -255,105 +123,45 @@ inline bool CompareLinks(const LinkCursor& a, const LinkCursor& b) return false; } - -/* -class LinkTraversal -{ - typedef std::priority_queue, - CompareLinks> CursorQueue; - std::shared_ptr link_cursors; - std::shared_ptr> constraints; // constraint junction ids - LinkCursor last_link; - - LinkTraversal() : has_active_cursor(false) - { - link_cursors = std::make_shared(); - } - - void add_links(std::shared_ptr links, - uint64_t age) - { - for (Link* link: &links) { - link_cursors->push(LinkCursor(link, age)); - } - } - - void pop_all(Junction* to_pop) - { - for (LinkCursor cursor : &link_cursors) { - if (!cursor.done() && (to_pop == cursor.current())) { - cursor.increment(); - } - } - } - - bool get_top_link(&LinkCursor result) - { - while(link_cusors->size() > 0) { - if (!link_cusors->top()->done()) { - result = link_cusors->top(); - return true; - } else { - link_cusors->pop(); - } - } - return false; - } - - template - bool try_link_neighbors(AssemblerTraverser asmt) - { - KmerQueue neighbors; - asmt.neighbors(neighbors); - bool decided = false; - Kmer src = asmt.cursor; - Kmer dst; - - LinkCursor link; - bool has_link = get_top_link(link); - if (!has_link) { - return false; - } - - if (has_active_cursor) { - - for (Kmer neighbor : neighbors) { - if ((&active_cursor.current())->matches(neighbor.kmer_f, - neighbor.kmer_r)) { - decided = true; - dst = neighbor; - } - } - - - } else { - - - } - - } - - - -}; */ -typedef std::unordered_multimap LinkMap; -typedef std::list LinkList; +typedef std::unordered_multimap LinkSegmentMap; +typedef std::vector LinkSegmentVector; +typedef std::vector JunctionVector; typedef std::unordered_map JunctionMap; -typedef std::pair LinkMapPair; +typedef std::pair LinkMapPair; class GraphLinker { protected: // map from starting high degree nodes to associated links - LinkMap links; + LinkSegmentMap link_segments; + // linear storage for Junctions + JunctionVector junctions; // map from junction keys to Junction* - JunctionMap junctions; + JunctionMap junction_map; uint64_t n_sequences_added; + // can cause duplicate junctions if we don't check the map first; + // this just keeps junctions on the same cache line if they're + // allocated one after the other + Junction* new_junction(HashIntoType u, HashIntoType v, HashIntoType w) + { + junctions.emplace_back(u, v, w); + + Junction* j = &(junctions.back); + junctions[j->index()] = j; + return j; + } + + LinkTreeSegment* new_link_tree_segment(uint64_t junction_id) + { + link_segments.emplace_back(); + l = &(link_segments.back); + return l; + } + public: std::shared_ptr graph; @@ -378,7 +186,7 @@ public: std::cout << " * " << links.size() << " links" << std::endl; } - Junction* get_junction(HashIntoType key) const + Junction* get_junction_by_index(HashIntoType key) const { auto search = junctions.find(key); if (search != junctions.end()) { @@ -387,31 +195,15 @@ public: return nullptr; } - Junction* get_junction(HashIntoType u, HashIntoType v, HashIntoType w) const + Junction* get_junction_by_index(HashIntoType u, HashIntoType v, HashIntoType w) const { return get_junction(u ^ v ^ w); } - Junction* get_junction(Junction& junction) const - { - return get_junction(junction.id()); - } - - Junction* new_junction(HashIntoType u, HashIntoType v, HashIntoType w) - { - Junction* j = new Junction(); - j->u = u; - j->v = v; - j->w = w; - j->count = 0; - junctions[j->id()] = j; - return j; - } - Junction* fetch_or_new_junction(HashIntoType u, HashIntoType v, HashIntoType w, uint64_t& counter) { - Junction* j = get_junction(u, v, w); + Junction* j = get_junction_by_index(u, v, w); if (j != nullptr) { j->count = j->count + 1; } else { @@ -422,10 +214,10 @@ public: return j; } - std::shared_ptr get_junctions(const std::string& sequence) const + std::vector get_junctions_ids(const std::string& sequence) const { KmerIterator kmers(sequence.c_str(), graph->ksize()); - std::shared_ptr junctions = make_shared(); + std::vector junctions; Kmer u = kmers.next(); if (kmers.done()) { @@ -440,9 +232,9 @@ public: Junction* j; while(!kmers.done()) { - j = get_junction(u, v, w); + j = get_junction_by_index(u, v, w); if (j != nullptr) { - junctions->push_back(j); + junctions->push_back(j->ID); } u = v; v = w; @@ -451,11 +243,28 @@ public: return junctions; } + + LinkSegment* get_link_segment(uint64_t start_junction_id) { + auto saerch = link_starts.find(start_junction_id); + if (search != link_starts.end()) { + return search->second; + } + return nullptr; + } + + LinkSegment* fetch_or_new_link_segment(uint64_t start_junction_id) { + LinkSegment* l = get_link_segment(start_junction_id); + if (l != nullptr) { + + } else { + l = new LinkTreeSegment(); + } + } - void build_links(const std::string& sequence, + void build_link(const std::string& sequence, Link* &fw_link, Link* &rc_link) { - std::cout << "build_links()" << std::endl; + std::cout << "build_link()" << std::endl; KmerIterator kmers(sequence.c_str(), graph->ksize()); Kmer u, v, w; uint64_t d = 0; @@ -475,26 +284,15 @@ public: w = kmers.next(); ++d; - std::cout << " - build_links: allocate new Link*" << std::endl; - fw_link = new Link(n_sequences_added, u.is_forward()); - rc_link = new Link(n_sequences_added, !u.is_forward()); uint64_t n_new_fw = 0; - uint64_t n_new_rc = 0; - uint64_t last_rc_pos = 0; + Junction* start = nullptr; + // find starting HDN that will index the Link Traverser traverser(graph.get()); - while(!kmers.done()) { - if (traverser.degree_right(v) > 1) { - std::cout << " - build_links: found FW HDN " << u << std::endl; - fw_link->push_back(fetch_or_new_junction(u, v, w, n_new_fw)); - if (n_new_fw == 1) { - fw_link->flanking_distance = d; - } - } - if (traverser.degree_left(v) > 1) { - std::cout << " - build_links: found RC HDN " << v << std::endl; - rc_link->push_front(fetch_or_new_junction(w, v, u, n_new_rc)); - last_rc_pos = d; + while(!kmers.done() && start == nullptr) { + if (traverser.degree(v) > 2) { + std::cout << " - build_links: found lead HDN" << u << std::endl; + start = fetch_or_new_junction(u, v, w, n_new_fw); } u = v; @@ -503,6 +301,10 @@ public: ++d; } + + + + if (fw_link->size() < 1) { std::cout << " - build_links: (fw_link) no (new) junctions found." << std::endl; delete fw_link; diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 68e66feee2..6e520153c2 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -7,14 +7,16 @@ using namespace oxli; std::ostream& operator<<(std::ostream& stream, const oxli::Junction& j) { - stream << "Junction(u=" << j.u << ", v=" << j.v << - ", count=" << j.count << ", id=" << j.id() << ")"; + stream << "Junction<(" << j.u << ", " << j.v << << ", " << j.w << + "), count=" << j.count << ", index=" << j.index() << ">"; return stream; } namespace oxli { -uint64_t Link::n_links = 0; +uint64_t Junction::junction_counter = 0; +uint64_t LinkTreeSegment::segment_counter = 0; + /* template void LinkedAssembler:: From 590662cf99f1507d75ad3200c004804d08093ab2 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 20 Oct 2017 11:59:21 -0700 Subject: [PATCH 137/185] switch toward compact dbg? --- include/oxli/assembler.hh | 20 +- include/oxli/hashgraph.hh | 3 +- include/oxli/links.hh | 445 ++++++++++++++++---------------------- include/oxli/oxli.hh | 2 +- src/oxli/assembler.cc | 63 +++++- src/oxli/hashgraph.cc | 22 +- src/oxli/links.cc | 11 +- 7 files changed, 276 insertions(+), 290 deletions(-) diff --git a/include/oxli/assembler.hh b/include/oxli/assembler.hh index 49d2995621..d10cb85efa 100644 --- a/include/oxli/assembler.hh +++ b/include/oxli/assembler.hh @@ -79,8 +79,10 @@ public: WordLength _ksize; const Hashgraph * graph; + std::shared_ptr global_visited; - explicit LinearAssembler(const Hashgraph * ht); + explicit LinearAssembler(const Hashgraph * ht, + std::shared_ptr global_visited = nullptr); virtual std::string assemble(const Kmer seed_kmer, const Hashgraph * stop_bf = 0) const; @@ -98,19 +100,18 @@ public: // The explicit specializations need to be declared in the same translation unit // as their unspecialized declaration. template<> -std::string LinearAssembler::_assemble_directed(AssemblerTraverser - &cursor) const; +std::string LinearAssembler::_assemble_directed(AssemblerTraverser &cursor) const; template<> -std::string LinearAssembler::_assemble_directed(AssemblerTraverser - &cursor) const; +std::string LinearAssembler::_assemble_directed(AssemblerTraverser &cursor) const; class CompactingAssembler: public LinearAssembler { public: - explicit CompactingAssembler(const Hashgraph* ht) : LinearAssembler(ht) {} + explicit CompactingAssembler(const Hashgraph* ht, + std::shared_ptr global_visited=nullptr) : LinearAssembler(ht, global_visited) {} virtual std::string assemble(const Kmer seed_kmer, const Hashgraph * stop_bf) const; @@ -120,6 +121,13 @@ public: virtual std::string assemble_left(const Kmer seed_kmer, const Hashgraph * stop_bf = 0) const; + + template + std::string CompactingAssembler:: + _assemble_directed(CompactingAT& cursor) const + { + return LinearAssembler::_assemble_directed(cursor); + } }; typedef CompactingAssembler CpCompactingAssembler; diff --git a/include/oxli/hashgraph.hh b/include/oxli/hashgraph.hh index f450a42e6b..60d925e2d6 100644 --- a/include/oxli/hashgraph.hh +++ b/include/oxli/hashgraph.hh @@ -196,7 +196,8 @@ public: // consume a string & add sparse graph nodes. void consume_sequence_and_tag(const std::string& seq, unsigned long long& n_consumed, - SeenSet * new_tags = 0); + SeenSet * new_tags = nullptr, + SeenSet * tag_set = nullptr); // get the tags present in this sequence. void get_tags_for_sequence(const std::string& seq, diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 8ebb034577..ff2a76e8f5 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -54,119 +54,99 @@ Contact: khmer-project@idyll.org namespace oxli { -class Junction { -protected: - - static uint64_t junction_counter; +using std::make_shared; +using std::shared_ptr; + +typedef std::pair HashIDPair; +typedef std::unordered_set HashSet; +typedef std::vector HashVector; +typedef std::unordered_multimap HashIDMap; +class CompactEdge; +class CompactNode { +protected: + static uint64_t node_counter; public: + Kmer kmer; + uint32_t count; + const uint64_t node_id; + CompactEdge* in_edges[4] = {nullptr, nullptr, nullptr, nullptr}; + CompactEdge* out_edges[4] = {nullptr, nullptr, nullptr, nullptr}; - // [u]-->[v (HDN)]-->[w] - HashIntoType u; - HashIntoType v; - HashIntoType w; - const uint64_t ID; + CompactNode(Kmer kmer): kmer(kmer), count(0), node_id(node_counter++) {} - Junction() = default; + friend bool operator== (const CompactNode* lhs, const CompactNode* rhs) { + return lhs->node_id == rhs->node_id; + } - Junction(HashIntoType u, HashIntoType v, HashIntoType w) : - u(u), v(v), w(w), ID(junction_counter++) {} + void add_in_edge(const char base, CompactEdge* edge) { + in_edges[twobit_repr(base)] = edge; + } - uint64_t index() const { return u ^ v ^ w; } + CompactEdge* get_in_edge(const char base) { + return in_edges[twobit_repr(base)]; + } - bool matches(HashIntoType u, - HashIntoType v, - HashintoType w) { return (u ^ v ^ w) == index(); } + void add_out_edge(const char base, CompactEdge* edge) { + out_edges[twobit_repr(base)] = edge; + } - friend std::ostream& operator<< (std::ostream& stream, - const Junction& j); - friend bool operator== (const Junction& lhs, - const Junction& rhs) { - return lhs.index() == rhs.index(); + CompactEdge* get_out_edge(const char base) { + return out_edges[twobit_repr(base)]; } }; -typedef std::list JunctionList; - -#define FW 1 -#define RC 0 - -class LinkTreeSegment { -protected: - static uint64_t segment_counter; +class CompactEdge { public: - const uint64_t segment_id; - std::vector - std::vector junctions; // IDs of junctions in this segment - uint64_t children[4]; // child segment IDs - uint32_t count; - - LinkTreeSegment() : segment_id(segment_counter++) {} - + HashIntoType left; // left and right HDNs + HashIntoType right; + HashSet tags; + bool is_tip; + bool dirty; + std::string sequence; // optional + + CompactEdge(HashIntoType left, HashIntoType right) : + left(left), right(right), is_tip(false), dirty(false) {} + CompactEdge(HashIntoType left, HashIntoType right, bool is_tip) : + left(left), right(right), is_tip(is_tip), dirty(false) {} }; - -// Compare the two link cursors first by their age within the -// traversal, then by their global age. We prefer links -// that were found earlier in the traversal, but were created -// most recently. -/* -inline bool CompareLinks(const LinkCursor& a, const LinkCursor& b) -{ - if (a.traversal_age < b.traversal_age) { return false; } - if (b.traversal_age < a.traversal_age) { return true; } - - if ((a.link)->time_created > (b.link)->time_created) { return false; } - if ((b.link)->time_created > (a.link)->time_created) { return true; } - - return false; -} -*/ +typedef std::vector CompactNodeVector; +typedef std::unordered_multimap TagEdgeMap; +typedef std::pair TagEdgePair; +typedef std::set TagEdgePairSet; -typedef std::unordered_multimap LinkSegmentMap; -typedef std::vector LinkSegmentVector; -typedef std::vector JunctionVector; -typedef std::unordered_map JunctionMap; -typedef std::pair LinkMapPair; - class GraphLinker { protected: - // map from starting high degree nodes to associated links - LinkSegmentMap link_segments; - // linear storage for Junctions - JunctionVector junctions; - // map from junction keys to Junction* - JunctionMap junction_map; - uint64_t n_sequences_added; - - // can cause duplicate junctions if we don't check the map first; - // this just keeps junctions on the same cache line if they're - // allocated one after the other - Junction* new_junction(HashIntoType u, HashIntoType v, HashIntoType w) - { - junctions.emplace_back(u, v, w); + // map from HDN hashes to CompactNode IDs + HashIDMap hdn_ids; + // linear storage for CompactNodes + CompactNodeVector compact_nodes; + // map from tags to CompactEdges + TagEdgeMap tags_to_edges; - Junction* j = &(junctions.back); - junctions[j->index()] = j; - return j; - } + uint64_t n_sequences_added; - LinkTreeSegment* new_link_tree_segment(uint64_t junction_id) + CompactNode* new_compact_node(Kmer hdn) { - link_segments.emplace_back(); - l = &(link_segments.back); - return l; + CompactNode * v = get_compact_node_by_kmer(hdn); + if (v == nullptr) { + compact_nodes.emplace_back(hdn); + v = &(compact_nodes.back); + hdn_ids[hdn] = v->node_id; + } + return v; } public: - std::shared_ptr graph; + shared_ptr graph; - GraphLinker(std::shared_ptr graph) : + GraphLinker(shared_ptr graph) : graph(graph), n_sequences_added(0) { @@ -186,235 +166,190 @@ public: std::cout << " * " << links.size() << " links" << std::endl; } - Junction* get_junction_by_index(HashIntoType key) const + CompactNode* get_compact_node_by_kmer(HashIntoType hdn) const { - auto search = junctions.find(key); - if (search != junctions.end()) { - return search->second; + auto search = hdn_ids.find(hdn); + if (search != hdn_ids.end()) { + uint64_t ID = search->second; + return &(compact_nodes[ID]); } return nullptr; } - Junction* get_junction_by_index(HashIntoType u, HashIntoType v, HashIntoType w) const - { - return get_junction(u ^ v ^ w); - } - - Junction* fetch_or_new_junction(HashIntoType u, HashIntoType v, - HashIntoType w, uint64_t& counter) + CompactNode* fetch_or_new_compact_node(Kmer hdn) { - Junction* j = get_junction_by_index(u, v, w); - if (j != nullptr) { - j->count = j->count + 1; + CompactNode* v = get_compact_node_by_kmer(hdn); + if (v != nullptr) { + v->count += 1; } else { - counter++; - j = new_junction(u, v, w); - j->count = 1; + v = new_compact_node(hdn); + v->count = 1; } - return j; + return v; } - std::vector get_junctions_ids(const std::string& sequence) const + std::vector get_compact_node_ids(const std::string& sequence) const { KmerIterator kmers(sequence.c_str(), graph->ksize()); - std::vector junctions; + std::vector ids; - Kmer u = kmers.next(); - if (kmers.done()) { - return junctions; - } - Kmer v = kmers.next(); - if (kmers.done()) { - return junctions; - } - Kmer w = kmers.next(); - - Junction* j; + CompactNode* node; while(!kmers.done()) { - j = get_junction_by_index(u, v, w); + Kmer kmer = kmers.next(); + + node = get_compact_node_by_kmer(kmer); if (j != nullptr) { - junctions->push_back(j->ID); + ids->push_back(node->node_id); } - u = v; - v = w; - w = kmers.next(); } - return junctions; + return ids; } - LinkSegment* get_link_segment(uint64_t start_junction_id) { - auto saerch = link_starts.find(start_junction_id); - if (search != link_starts.end()) { + CompactEdge* get_compact_edge(uint64_t tag) { + auto search = tags_to_edges.find(tag); + if (search != tags_to_edges.end()) { return search->second; } return nullptr; } - LinkSegment* fetch_or_new_link_segment(uint64_t start_junction_id) { - LinkSegment* l = get_link_segment(start_junction_id); - if (l != nullptr) { - - } else { - l = new LinkTreeSegment(); + TagEdgePair get_tag_edge_pair(uint64_t tag) { + auto search = tags_to_edges.find(tag); + if (search != tags_to_edges.end()) { + return &search; } + return nullptr; } - - void build_link(const std::string& sequence, - Link* &fw_link, Link* &rc_link) - { - std::cout << "build_link()" << std::endl; - KmerIterator kmers(sequence.c_str(), graph->ksize()); - Kmer u, v, w; - uint64_t d = 0; - - u = kmers.next(); - ++d; - if (kmers.done()) { - fw_link = rc_link = nullptr; - return; + + CompactEdge* get_edge_froms_tags(HashSet& tags) { + CompactEdge * edge = nullptr; + for (auto tag: tags) { + edge = get_compact_edge(tag); + if (edge != nullptr) { + break; + } } - v = kmers.next(); - ++d; - if (kmers.done()) { - fw_link = rc_link = nullptr; - return; + return edge; + } + + void assign_tags_to_edge(HashSet& tags, CompactEdge* edge) { + for (auto tag: tags) { + tags_to_edges[tag] = edge; } - w = kmers.next(); - ++d; - - uint64_t n_new_fw = 0; - Junction* start = nullptr; - - // find starting HDN that will index the Link - Traverser traverser(graph.get()); - while(!kmers.done() && start == nullptr) { - if (traverser.degree(v) > 2) { - std::cout << " - build_links: found lead HDN" << u << std::endl; - start = fetch_or_new_junction(u, v, w, n_new_fw); + } + + KmerFilter get_tag_finder(shared_ptr found_pairs, + shared_ptr found_unresolved_tags, + shared_ptr all_unresolved_tags) { + KmerFilter finder = [=] (const Kmer& node) { + TagEdgePair edge_pair = get_tag_edge_pair(node); + if (edge_pair != nullptr) { + found_pairs->insert(edge_pair); } + if (set_contains(all_unresolved_tags, node) { + found_unresolved_tags->insert(node); + } + return false; // never filter node, just gather while we're looking + } + } - u = v; - v = w; - w = kmers.next(); - ++d; - } + template + void update_compact_dbg(shared_ptr unresolved_tags) { + HashVector unresolved_q; + for (auto tag: unresolved_tags) { + unresolved_q.push_back(tag); + } + CompactingAssembler at(graph.get()); + shared_ptr visited = make_shared(); + while(unresolved_q.size() > 0) { + auto start_tag = unresolved_q.back(); + unresolved_q.pop_back(); + if (!set_contains(&unresolved_tags, start_tag)) { + continue; + } + + shared_ptr segment_edges = make_shared(); + shared_ptr segment_new_tags = make_shared(); + CompactingAT lcursor(graph.get(), tag - if (fw_link->size() < 1) { - std::cout << " - build_links: (fw_link) no (new) junctions found." << std::endl; - delete fw_link; - fw_link = nullptr; } - if (rc_link->size() < 1) { - std::cout << " - build_links: (rc_link) no (new) junctions found." << std::endl; - delete rc_link; - rc_link = nullptr; - } else { - rc_link->flanking_distance = d - last_rc_pos; + shared_ptr found_edges; + shared_ptr< + filters.push_back(get_tag_finder(found_edges)); + CompactingAT cursor(graph, from, visited); + at._assemble_directed(cursor); + + if (cursor.cursor_degree() == 0) { // at a tip + } + } + void update_compact_dbg(const std::string& sequence) { + std::cout << "update_compact_dbg()" << std::endl; - void add_links(const std::string& sequence) - { - std::cout << "add_links('" << sequence << "')" << std::endl; - Link * fw_link = nullptr; - Link * rc_link = nullptr; - n_sequences_added++; - build_links(sequence, fw_link, rc_link); - - if (fw_link != nullptr) { - std::cout << " - add_links: insert found fw_link" << std::endl; - Junction* start = fw_link->start_junction(); - std::cout << " * start junction: " << &start << std::endl; - links.insert(LinkMapPair(start->v, fw_link)); - } - if (rc_link != nullptr) { - std::cout << " - add_links: insert found rc_link" << std::endl; - Junction* start = rc_link->start_junction(); - std::cout << " * start junction: " << &start << std::endl; - links.insert(LinkMapPair(start->v, rc_link)); - } - } + shared_ptr new_tags = make_shared(); + shared_ptr known_tags = make_shared(); + uint64_t n_consumed = 0; + consume_sequence_and_tag(sequence, n_consumed, new_tags, known_tags); - uint64_t get_links(Kmer hdn, std::shared_ptr found_links) - { - auto range = links.equal_range(hdn); - uint64_t n_links_found = 0; - for (auto it = range.first; it != range.second; ++it) { - found_links->push_back(it->second); - n_links_found++; - } - return n_links_found; - } - uint64_t get_links(std::list high_degree_nodes, - std::shared_ptr found_links) - { - uint64_t n_links_found = 0; - for (Kmer hdn : high_degree_nodes) { - n_links_found += get_links(hdn, found_links); - } - return n_links_found; } - std::shared_ptr get_links(const std::string& sequence) - { - std::cout << "get_links('" << sequence << "')" << std::endl; - KmerIterator kmers(sequence.c_str(), graph->ksize()); - Traverser traverser(graph.get()); - std::list hdns; + // sigh at duplicated code from Hashgraph... perhaps in the future + // that function can be templated to accept multiple Set types for tags + void consume_sequence_and_tag(const std::string& seq, + unsigned long long& n_consumed, + shared_ptr new_tags, + shared_ptr known_tags) { + bool kmer_tagged; - std::cout << " - get_link: start iterating k-mers" << std::endl; - Kmer kmer = kmers.next(); - while(!kmers.done()) { - if (traverser.degree(kmer) > 2) { - hdns.push_back(kmer); - } - kmer = kmers.next(); - } - std::cout << " - get_links: found " << hdns.size() << " hdns" << std::endl; + KmerIterator kmers(seq.c_str(), _ksize); + Kmer kmer; - std::shared_ptr found_links = make_shared(); - get_links(hdns, found_links); - std::cout << " - get_links: returning " << found_links->size() << " links" << std::endl; - return found_links; - } + unsigned int since = _tag_density / 2 + 1; -}; + while(!kmers.done()) { + kmer = kmers.next(); + bool is_new_kmer; -/* + if ((is_new_kmer = graph->add(kmer))) { + ++n_consumed; + } -class LinkedAssembler -{ - std::shared_ptr linear_asm; + if (is_new_kmer) { + ++since; + } else { + kmer_tagged = set_contains(tags_to_edges, kmer); + if (kmer_tagged) { + since = 1; + known_tags->insert(kmer); + } else { + ++since; + } + } -public: + if (since >= _tag_density) { + new_tags->insert(kmer); + since = 1; + } - const std::shared_ptr graph; - const std::shared_ptr linker; - WordLength _ksize; + } // iteration over kmers - explicit LinkedAssembler(std::shared_ptr linker) : - _ksize(linker->ksize()), graph(linker->graph), linker(linker) - { - linear_asm = make_shared(graph.get()); + if (since >= _tag_density/2 - 1) { + new_tags->insert(kmer); // insert the last k-mer, too. + } } - StringVector assemble(const Kmer seed_kmer, - std::shared_ptr stop_bf=nullptr) const; - - template - void _assemble_directed(AssemblerTraverser& cursor, - StringVector& paths) const; }; -*/ + } diff --git a/include/oxli/oxli.hh b/include/oxli/oxli.hh index 13c39378ab..682d09cba9 100644 --- a/include/oxli/oxli.hh +++ b/include/oxli/oxli.hh @@ -164,7 +164,7 @@ void deallocate_ptr_set(T& s) class Kmer; typedef std::queue KmerQueue; -typedef std::set KmerSet; +typedef std::unordered_set KmerSet; // A function which takes a Kmer and returns true if it // is to be filtered / ignored diff --git a/src/oxli/assembler.cc b/src/oxli/assembler.cc index 8378343d11..692950d21e 100644 --- a/src/oxli/assembler.cc +++ b/src/oxli/assembler.cc @@ -49,10 +49,11 @@ namespace oxli * Simple Linear Assembly ********************************/ -LinearAssembler::LinearAssembler(const Hashgraph * ht) : +LinearAssembler::LinearAssembler(const Hashgraph * ht, + std::shared_ptr global_visited) : graph(ht), _ksize(ht->ksize()) { - + global_visited = global_visited; } // Starting from the given seed k-mer, assemble the maximal linear path in @@ -72,7 +73,13 @@ const node_filters.push_back(get_stop_bf_filter(stop_bf)); } - std::shared_ptr visited = std::make_shared(); + std::shared_ptr visited; + if (global_visited != nullptr) { + visited = global_visited; + } else { + visited = std::make_shared(); + } + AssemblerTraverser rcursor(graph, seed_kmer, node_filters, visited); AssemblerTraverser lcursor(graph, seed_kmer, node_filters, visited); @@ -98,7 +105,17 @@ const node_filters.push_back(get_stop_bf_filter(stop_bf)); } - AssemblerTraverser cursor(graph, seed_kmer, node_filters); + std::shared_ptr visited; + if (global_visited != nullptr) { + visited = global_visited; + } else { + visited = std::make_shared(); + } + + AssemblerTraverser cursor(graph, + seed_kmer, + node_filters, + visited); return _assemble_directed(cursor); } @@ -112,7 +129,17 @@ const node_filters.push_back(get_stop_bf_filter(stop_bf)); } - AssemblerTraverser cursor(graph, seed_kmer, node_filters); + std::shared_ptr visited; + if (global_visited != nullptr) { + visited = global_visited; + } else { + visited = std::make_shared(); + } + + AssemblerTraverser cursor(graph, + seed_kmer, + node_filters, + visited); return _assemble_directed(cursor); } @@ -190,7 +217,13 @@ const node_filters.push_back(get_stop_bf_filter(stop_bf)); } - std::shared_ptr visited = std::make_shared(); + std::shared_ptr visited; + if (global_visited != nullptr) { + visited = global_visited; + } else { + visited = std::make_shared(); + } + CompactingAT rcursor(graph, seed_kmer, node_filters, visited); CompactingAT lcursor(graph, seed_kmer, node_filters, visited); @@ -210,7 +243,14 @@ const node_filters.push_back(get_stop_bf_filter(stop_bf)); } - CompactingAT cursor(graph, seed_kmer, node_filters); + std::shared_ptr visited; + if (global_visited != nullptr) { + visited = global_visited; + } else { + visited = std::make_shared(); + } + + CompactingAT cursor(graph, seed_kmer, node_filters, visited); return LinearAssembler::_assemble_directed(cursor); } @@ -224,7 +264,14 @@ const node_filters.push_back(get_stop_bf_filter(stop_bf)); } - CompactingAT cursor(graph, seed_kmer, node_filters); + std::shared_ptr visited; + if (global_visited != nullptr) { + visited = global_visited; + } else { + visited = std::make_shared(); + } + + CompactingAT cursor(graph, seed_kmer, node_filters, visited); return LinearAssembler::_assemble_directed(cursor); } diff --git a/src/oxli/hashgraph.cc b/src/oxli/hashgraph.cc index cac8065782..e41e566112 100644 --- a/src/oxli/hashgraph.cc +++ b/src/oxli/hashgraph.cc @@ -199,9 +199,13 @@ void Hashgraph::load_tagset(std::string infilename, bool clear_tags) void Hashgraph::consume_sequence_and_tag(const std::string& seq, unsigned long long& n_consumed, - SeenSet * found_tags) + SeenSet * found_tags, + SeenSet * tag_set) { bool kmer_tagged; + if (tag_set == nullptr) { + tag_set = all_tags; + } KmerIterator kmers(seq.c_str(), _ksize); HashIntoType kmer; @@ -226,11 +230,11 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, ++since; } else { ACQUIRE_ALL_TAGS_SPIN_LOCK - kmer_tagged = set_contains(all_tags, kmer); + kmer_tagged = set_contains(tag_set, kmer); RELEASE_ALL_TAGS_SPIN_LOCK if (kmer_tagged) { since = 1; - if (found_tags) { + if (found_tags != nullptr) { found_tags->insert(kmer); } } else { @@ -238,9 +242,9 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, } } #else - if (!is_new_kmer && set_contains(all_tags, kmer)) { + if (!is_new_kmer && set_contains(tag_set, kmer)) { since = 1; - if (found_tags) { + if (found_tags != nullptr) { found_tags->insert(kmer); } } else { @@ -250,9 +254,9 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, if (since >= _tag_density) { ACQUIRE_ALL_TAGS_SPIN_LOCK - all_tags.insert(kmer); + tag_set.insert(kmer); RELEASE_ALL_TAGS_SPIN_LOCK - if (found_tags) { + if (found_tags != nullptr) { found_tags->insert(kmer); } since = 1; @@ -262,9 +266,9 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, if (since >= _tag_density/2 - 1) { ACQUIRE_ALL_TAGS_SPIN_LOCK - all_tags.insert(kmer); // insert the last k-mer, too. + tag_set.insert(kmer); // insert the last k-mer, too. RELEASE_ALL_TAGS_SPIN_LOCK - if (found_tags) { + if (found_tags != nullptr) { found_tags->insert(kmer); } } diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 6e520153c2..59e69f3409 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -4,18 +4,9 @@ using namespace oxli; -std::ostream& operator<<(std::ostream& stream, - const oxli::Junction& j) -{ - stream << "Junction<(" << j.u << ", " << j.v << << ", " << j.w << - "), count=" << j.count << ", index=" << j.index() << ">"; - return stream; -} - namespace oxli { -uint64_t Junction::junction_counter = 0; -uint64_t LinkTreeSegment::segment_counter = 0; +uint64_t CompactNode::node_counter = 0; /* template From b166e3bebbebfd68feb4940e1b9b665d3ce5446e Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 24 Oct 2017 02:23:00 -0700 Subject: [PATCH 138/185] First pass pre compile --- include/oxli/hashgraph.hh | 1 + include/oxli/kmer_hash.hh | 5 + include/oxli/links.hh | 247 ++++++++++++++++++++++++++++++-------- include/oxli/traversal.hh | 2 + src/oxli/hashgraph.cc | 5 + src/oxli/links.cc | 55 +-------- 6 files changed, 217 insertions(+), 98 deletions(-) diff --git a/include/oxli/hashgraph.hh b/include/oxli/hashgraph.hh index 60d925e2d6..6318fb52ad 100644 --- a/include/oxli/hashgraph.hh +++ b/include/oxli/hashgraph.hh @@ -245,6 +245,7 @@ public: // Calculate the graph degree of the given k-mer. unsigned int kmer_degree(HashIntoType kmer_f, HashIntoType kmer_r); unsigned int kmer_degree(const char * kmer_s); + unsigned int kmer_degree(Kmer kmer); // Find all nodes with a degree > 2. void find_high_degree_nodes(const char * sequence, diff --git a/include/oxli/kmer_hash.hh b/include/oxli/kmer_hash.hh index da1228d6df..7ba6139dc5 100644 --- a/include/oxli/kmer_hash.hh +++ b/include/oxli/kmer_hash.hh @@ -189,6 +189,11 @@ public: return kmer_u < other.kmer_u; } + bool operator== (const Kmer &other) const + { + return kmer_u == other.kmer_u; + } + std::string get_string_rep(WordLength K) const { return _revhash(kmer_u, K); diff --git a/include/oxli/links.hh b/include/oxli/links.hh index ff2a76e8f5..fed360c9ef 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -94,27 +94,69 @@ public: CompactEdge* get_out_edge(const char base) { return out_edges[twobit_repr(base)]; } + + uint8_t degree() const { + return out_degree() + in_degree(); + } + + uint8_t out_degree() const { + uint8_t acc = 0; + for (auto edge: out_edges) { + if (edge != nullptr) { + acc++; + } + } + return acc; + } + + uint8_t in_degree() const { + uint8_t acc = 0; + for (auto edge: in_edges) { + if (edge != nullptr) { + acc++; + } + } + return acc; + } + + friend std::ostream& operator<< (std::ostream& stream, + const CompactNode& node); +}; + + +enum compact_edge_meta_t { + IS_FULL_EDGE, + IS_IN_TIP, + IS_OUT_TIP, + IS_ISLAND }; class CompactEdge { public: - HashIntoType left; // left and right HDNs - HashIntoType right; + HashIntoType in_hash; // left and right HDNs + HashIntoType out_hash; HashSet tags; - bool is_tip; + compact_edge_meta_t meta; bool dirty; std::string sequence; // optional - CompactEdge(HashIntoType left, HashIntoType right) : - left(left), right(right), is_tip(false), dirty(false) {} - CompactEdge(HashIntoType left, HashIntoType right, bool is_tip) : - left(left), right(right), is_tip(is_tip), dirty(false) {} + CompactEdge(HashIntoType in_hash, HashIntoType out_hash) : + in_hash(in_hash), out_hash(out_hash), meta(IS_FULL_EDGE), dirty(false) {} + + CompactEdge(HashIntoType left, HashIntoType right, compact_edge_meta_t meta) : + in_hash(in_hash), out_hash(out_hash), meta(meta), dirty(false) {} + + void add_tags(HashSet& new_tags) { + for (auto tag: new_tags) { + tags.insert(tag); + } + } }; typedef std::vector CompactNodeVector; -typedef std::unordered_multimap TagEdgeMap; +typedef std::unordered_map TagEdgeMap; typedef std::pair TagEdgePair; typedef std::set TagEdgePairSet; @@ -130,9 +172,11 @@ protected: TagEdgeMap tags_to_edges; uint64_t n_sequences_added; + uint64_t n_compact_edges; - CompactNode* new_compact_node(Kmer hdn) - { + // protected linear creation of CompactNode + // they should never be deleted, so this is straightforward + CompactNode* new_compact_node(Kmer hdn) { CompactNode * v = get_compact_node_by_kmer(hdn); if (v == nullptr) { compact_nodes.emplace_back(hdn); @@ -142,12 +186,37 @@ protected: return v; } + CompactEdge* new_compact_edge(HashIntoType left, HashIntoType right, + compact_edge_meta_t edge_meta, std::string edge_sequence) { + CompactEdge* edge = new CompactEdge(left, right, edge_meta); + edge->sequence = edge_sequence; + n_compact_edges++; + } + + void delete_compact_edge(CompactEdge * edge) { + for (auto tag: edge->tags) { + tags_to_edges.erase(tag); + } + delete edge; + n_compact_edges--; + } + + void delete_compact_edge(HashSet& tags) { + CompactEdge* edge = get_compact_edge_from_tags(tags); + delete_compact_edge(edge): + } + + void delete_compact_edge(HashIntoType tag) { + CompactEdge* edge = get_comapct_edge(tag); + delete_compact_edge(edge); + } + public: shared_ptr graph; GraphLinker(shared_ptr graph) : - graph(graph), n_sequences_added(0) + graph(graph), n_sequences_added(0), n_compact_edges(0) { } @@ -161,9 +230,9 @@ public: { std::cout << "GraphLinker (@" << this << " with " << "Hashgraph @" << graph.get() << ")" << std::endl; - std::cout << " * " << junctions.size() << " junctions" << std::endl; + std::cout << " * " << hdn_ids.size() << " cDBG nodes (HDNs)" << std::endl; + std::cout << " * " << n_compact_edges << " cDBG edges" << std::endl; std::cout << " * " << n_sequences_added << " sequences added" << std::endl; - std::cout << " * " << links.size() << " links" << std::endl; } CompactNode* get_compact_node_by_kmer(HashIntoType hdn) const @@ -223,7 +292,7 @@ public: return nullptr; } - CompactEdge* get_edge_froms_tags(HashSet& tags) { + CompactEdge* get_edge_from_tags(HashSet& tags) { CompactEdge * edge = nullptr; for (auto tag: tags) { edge = get_compact_edge(tag); @@ -234,69 +303,149 @@ public: return edge; } - void assign_tags_to_edge(HashSet& tags, CompactEdge* edge) { - for (auto tag: tags) { - tags_to_edges[tag] = edge; - } - } + KmerFilter get_tag_finder(TagEdgePairSet& found_pairs, + KmerSet& found_tags, + KmerSet& all_unresolved_tags) { - KmerFilter get_tag_finder(shared_ptr found_pairs, - shared_ptr found_unresolved_tags, - shared_ptr all_unresolved_tags) { KmerFilter finder = [=] (const Kmer& node) { TagEdgePair edge_pair = get_tag_edge_pair(node); if (edge_pair != nullptr) { - found_pairs->insert(edge_pair); + found_pairs.insert(edge_pair); + found_tags.insert(node); } if (set_contains(all_unresolved_tags, node) { - found_unresolved_tags->insert(node); + found_tags.insert(node); } return false; // never filter node, just gather while we're looking } } - template - void update_compact_dbg(shared_ptr unresolved_tags) { + void update_compact_dbg(KmerSet& unresolved_tags) { - HashVector unresolved_q; + KmerQueue unresolved_q; for (auto tag: unresolved_tags) { unresolved_q.push_back(tag); } CompactingAssembler at(graph.get()); - shared_ptr visited = make_shared(); + Traverser traverser(graph.get()); + //shared_ptr visited = make_shared(); while(unresolved_q.size() > 0) { - auto start_tag = unresolved_q.back(); + Kmer seed_tag = unresolved_q.back(); unresolved_q.pop_back(); - if (!set_contains(&unresolved_tags, start_tag)) { + if (!set_contains(*unresolved_tags, start_tag)) { continue; } - shared_ptr segment_edges = make_shared(); - shared_ptr segment_new_tags = make_shared(); - CompactingAT lcursor(graph.get(), tag + TagEdgePairSet segment_edges; + KmerSet segment_tags; + KmerFilter tag_finder = get_tag_finder(segment_edges, + segment_tags, + *unresolved_tags); + CompactingAT lcursor(graph.get(), seed_tag); + lcursor.push_filter(tag_finder); + CompactingAT rcursor(graph.get(), seed_tag); + rcursor.push_filter(tag_finder); + + // assemble both dirs until HDN, collecting tags along the way + std::string left_seq = at._assemble_directed(lcursor); + std::string right_seq = at._assemble_directed(rcursor); + std::string edge_sequence = left_seq + right_seq.substr(graph->ksize()); + + // Generate new CompactNodes if needed + CompactNode* left_node, right_node; + left_node = right_node = nullptr; + if (traverser.degree(rcursor.cursor) > 2) { + right_node = fetch_or_new_compact_node(rcursor.cursor); + } + if (traverser.degree(lcursor.cursor) > 2) { + left_node = fetch_or_new_compact_node(lcursor.cursor); + } + // Determine if we've got a tip, full edge, or island + compact_edge_meta_t edge_meta; + if (left_node == nullptr && right_node == nullptr) { + edge_meta = IS_ISLAND; + } else if (right_node == nullptr) { + edge_meta = IS_OUT_TIP; + } else if (left_node == nullptr) { + edge_meta = IS_IN_TIP; + } else { + edge_meta = IS_FULL_EDGE; + } - } + // first check if we've got a good Edge to work with + CompactEdge* consensus_edge = nullptr; + TagEdgePair consensus_tag_edge_pair; + for (consensus_tag_edge_pair: segment_edges) { + CompactEdge* intersect_edge = consensus_tag_edge_pair->second; + if (intersect_edge->in_hash == left_node->kmer && + intersect_edge->out_hash == right_node->kmer) { + + consensus_edge = intersect_edge; + break; + } + } - shared_ptr found_edges; - shared_ptr< - filters.push_back(get_tag_finder(found_edges)); - CompactingAT cursor(graph, from, visited); - at._assemble_directed(cursor); + // ... if not, create one + if (consensus_edge == nullptr) { + consensus_edge = new_compact_edge(lcursor.cursor, + rcursor.cursor, + edge_meta, + edge_sequence); + } else { + // if so, remove it from the set of found edges + // so we can resolve them + segment_edges.remove(consensus_tag_edge_pair); + } + + // remove all of the tags for this segment from stale edges + // and map them to the consensus edge + for (auto segment_tag: segment_tags) { + + for (auto tag_edge_pair: segment_edges) { + CompactEdge* stale_edge = tag_edge_pair->second; + stale_edge->tags.erase(segment_tag); + } + + consensus_edge->tags.insert(segment_tag); + tags_to_edges[segment_tag] = consensus_edge; + // don't forget to remove from the unresolved set + unresolved_tags.erase(segment_tag); + } + + // one last run through to delete stale edges and add remaining tags + // to the unresolved set + for (auto tag_edge_pair: segment_edges) { + CompactEdge* edge = tag_edge_pair->second; + for (auto stale_tag: edge.tags) { + unresolved_tags.insert(tag_edge_pair->first); + tags_to_edges.erase(stale_tag); + } + // don't forget to clean up + delete_compact_edge(edge); + } + + if (left_node != nullptr) { + left_node->add_in_edge( + edge_sequence[edge_sequence.length()-(graph->ksize())], + edge); + } + if (right_node != nullptr) { + right_node->add_out_edge( + edge_sequence[graph->ksize()], + edge); + } - if (cursor.cursor_degree() == 0) { // at a tip - } - } void update_compact_dbg(const std::string& sequence) { std::cout << "update_compact_dbg()" << std::endl; - shared_ptr new_tags = make_shared(); - shared_ptr known_tags = make_shared(); + KmerSet new_tags; + KmerSet known_tags; uint64_t n_consumed = 0; consume_sequence_and_tag(sequence, n_consumed, new_tags, known_tags); @@ -307,8 +456,8 @@ public: // that function can be templated to accept multiple Set types for tags void consume_sequence_and_tag(const std::string& seq, unsigned long long& n_consumed, - shared_ptr new_tags, - shared_ptr known_tags) { + KmerSet& new_tags, + KmerSet& known_tags) { bool kmer_tagged; KmerIterator kmers(seq.c_str(), _ksize); @@ -330,21 +479,21 @@ public: kmer_tagged = set_contains(tags_to_edges, kmer); if (kmer_tagged) { since = 1; - known_tags->insert(kmer); + known_tags.insert(kmer); } else { ++since; } } if (since >= _tag_density) { - new_tags->insert(kmer); + new_tags.insert(kmer); since = 1; } } // iteration over kmers if (since >= _tag_density/2 - 1) { - new_tags->insert(kmer); // insert the last k-mer, too. + new_tags.insert(kmer); // insert the last k-mer, too. } } diff --git a/include/oxli/traversal.hh b/include/oxli/traversal.hh index 47b456eed7..ce0de0bde8 100644 --- a/include/oxli/traversal.hh +++ b/include/oxli/traversal.hh @@ -193,6 +193,8 @@ public: * @return Degree of the current cursor position and direction. */ unsigned int cursor_degree() const; + unsigned int in_degree() const; + unsigned int out_degree() const; }; diff --git a/src/oxli/hashgraph.cc b/src/oxli/hashgraph.cc index e41e566112..dd3a95897c 100644 --- a/src/oxli/hashgraph.cc +++ b/src/oxli/hashgraph.cc @@ -488,6 +488,11 @@ unsigned int Hashgraph::kmer_degree(const char * kmer_s) return traverser.degree(node); } +unsigned int Hashgraph::kmer_degree(Kmer kmer) +{ + return kmer_degree(kmer.kmer_r, kmer.kmer_f); +} + size_t Hashgraph::trim_on_stoptags(std::string seq) const { KmerIterator kmers(seq.c_str(), _ksize); diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 59e69f3409..55d0df3d7d 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -8,54 +8,11 @@ namespace oxli { uint64_t CompactNode::node_counter = 0; -/* -template -void LinkedAssembler:: -_assemble_directed(AssemblerTraverser cursor, - StringVector& paths) -const -{ - std::string contig = linear_asm->assemble_directed(cursor); - - LinkTraversal link_traversal; - - while(1) { - -#if DEBUG_ASSEMBLY - std::cout << "Pop: " << segments.size() << " segments on stack." << std::endl; - std::cout << "Segment: " << segment << std::endl; - std::cout << "Cursor: " << cursor.cursor.repr(_ksize) << std::endl; - std::cout << "n_filters: " << cursor.n_filters() << std::endl; -#endif - - if (cursor.cursor_degree() > 1) { - // hit a HDN, push new links - std::shared_ptr new_links; - linker->get_links(hdn, new_links); - // add them to traversal, with age being current contig size - link_traversal.add_links(new_links, contig.size()); - - if (links->size() == 0) { - paths.push_back(segment); - continue; - } - - cursor.push_filter(get_link_filter(cursor.cursor, - links, - ages); - KmerQueue linked_nodes; - cursor.neighbors(linked_nodes); - cursor.pop_filter(); - - if (branch_starts.empty()) { - paths.push_back(segment); - } - - - } - break; - - } +inline std::ostream& operator<< (std::ostream& stream, const CompactNode& node) { + stream << ""; + return stream; } -*/ + } From 6d36af45c14fae452e224f5a021cca530033a1fd Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 24 Oct 2017 17:43:11 -0700 Subject: [PATCH 139/185] Bring down more base class methods to ATs, have neighbors return directional node degree --- include/oxli/traversal.hh | 10 +++++++++- src/oxli/traversal.cc | 26 ++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/include/oxli/traversal.hh b/include/oxli/traversal.hh index ce0de0bde8..00deca679d 100644 --- a/include/oxli/traversal.hh +++ b/include/oxli/traversal.hh @@ -134,7 +134,7 @@ public: * @param node The Kmer to start at. * @param node_q To collect the results. * - * @return Number of neighbors found. + * @return Number of neighbors total (could be more than those found). */ unsigned int neighbors(const Kmer& node, KmerQueue &node_q) const; @@ -164,6 +164,7 @@ public: // The current position. Kmer cursor; using NodeGatherer::push_filter; + using NodeGatherer::neighbors; explicit NodeCursor(const Hashgraph * ht, Kmer start_kmer, @@ -253,6 +254,10 @@ protected: public: using NodeCursor::NodeCursor; + + explicit AssemblerTraverser(const Hashgraph* ht, + Kmer start_kmer, + KmerFilter filter); explicit AssemblerTraverser(const Hashgraph * ht, Kmer start_kmer, @@ -302,6 +307,9 @@ protected: Traverser traverser; public: + explicit CompactingAT(const Hashgraph * ht, + Kmer start_kmer, + KmerFilter filter); explicit CompactingAT(const Hashgraph * ht, Kmer start_kmer, diff --git a/src/oxli/traversal.cc b/src/oxli/traversal.cc index 409c82388d..6744d92a86 100644 --- a/src/oxli/traversal.cc +++ b/src/oxli/traversal.cc @@ -123,9 +123,11 @@ const // Get the putative neighboring Kmer Kmer neighbor = get_neighbor(node, base); // Now check if it's in the graph and passes the filters - if (graph->get_count(neighbor) && !(apply_kmer_filters(neighbor, filters))) { - node_q.push(neighbor); + if (graph->get_count(neighbor)) { ++found; + if (!apply_kmer_filters(neighbor, filters)) { + node_q.push(neighbor); + } } ++base; } @@ -295,6 +297,18 @@ AssemblerTraverser::AssemblerTraverser(const Hashgraph * ht, AssemblerTraverser::push_filter(get_visited_filter(visited)); } + +template +AssemblerTraverser::AssemblerTraverser(const Hashgraph * ht, + Kmer start_kmer, + KmerFilter filter) : + NodeCursor(ht, start_kmer, filter) +{ + visited = std::make_shared(); + AssemblerTraverser::push_filter(get_visited_filter(visited)); +} + + template AssemblerTraverser::AssemblerTraverser(const AssemblerTraverser& other) : AssemblerTraverser(other.graph, other.cursor, other.filters, other.visited) @@ -377,6 +391,14 @@ CompactingAT::CompactingAT(const Hashgraph * ht, { } +template +CompactingAT::CompactingAT(const Hashgraph * ht, + Kmer start_kmer, + KmerFilter filter) : + AssemblerTraverser(ht, start_kmer, filter), traverser(ht) +{ +} + template<> char CompactingAT::next_symbol() { From f394557d8b95faf37dd3e3ef3204500a873e2248 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 24 Oct 2017 17:50:03 -0700 Subject: [PATCH 140/185] First pass compiling, partially working... --- include/oxli/assembler.hh | 3 +- include/oxli/links.hh | 302 ++++++++++++++++++++++--------------- include/oxli/oxli.hh | 4 +- khmer/_oxli/graphlinks.pxd | 117 ++++++++++---- khmer/_oxli/graphlinks.pyx | 148 ++++++++++++++---- src/oxli/hashgraph.cc | 10 +- src/oxli/links.cc | 6 +- tests/test_linked_dbg.py | 39 ++--- 8 files changed, 417 insertions(+), 212 deletions(-) diff --git a/include/oxli/assembler.hh b/include/oxli/assembler.hh index d10cb85efa..8e637e9270 100644 --- a/include/oxli/assembler.hh +++ b/include/oxli/assembler.hh @@ -123,8 +123,7 @@ public: const Hashgraph * stop_bf = 0) const; template - std::string CompactingAssembler:: - _assemble_directed(CompactingAT& cursor) const + std::string _assemble_directed(CompactingAT& cursor) const { return LinearAssembler::_assemble_directed(cursor); } diff --git a/include/oxli/links.hh b/include/oxli/links.hh index fed360c9ef..c59a84c806 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -42,6 +42,8 @@ Contact: khmer-project@idyll.org #include #include #include +#include +#include #include "oxli.hh" #include "kmer_hash.hh" @@ -58,14 +60,12 @@ using std::make_shared; using std::shared_ptr; typedef std::pair HashIDPair; -typedef std::unordered_set HashSet; +typedef std::unordered_set UHashSet; typedef std::vector HashVector; -typedef std::unordered_multimap HashIDMap; +typedef std::unordered_map HashIDMap; class CompactEdge; class CompactNode { -protected: - static uint64_t node_counter; public: Kmer kmer; uint32_t count; @@ -73,10 +73,11 @@ public: CompactEdge* in_edges[4] = {nullptr, nullptr, nullptr, nullptr}; CompactEdge* out_edges[4] = {nullptr, nullptr, nullptr, nullptr}; - CompactNode(Kmer kmer): kmer(kmer), count(0), node_id(node_counter++) {} + CompactNode(Kmer kmer, uint64_t node_id) : + kmer(kmer), count(0), node_id(node_id) {} - friend bool operator== (const CompactNode* lhs, const CompactNode* rhs) { - return lhs->node_id == rhs->node_id; + friend bool operator== (const CompactNode& lhs, const CompactNode& rhs) { + return lhs.node_id == rhs.node_id; } void add_in_edge(const char base, CompactEdge* edge) { @@ -128,7 +129,8 @@ enum compact_edge_meta_t { IS_FULL_EDGE, IS_IN_TIP, IS_OUT_TIP, - IS_ISLAND + IS_ISLAND, + IS_TRIVIAL }; @@ -136,7 +138,7 @@ class CompactEdge { public: HashIntoType in_hash; // left and right HDNs HashIntoType out_hash; - HashSet tags; + UHashSet tags; compact_edge_meta_t meta; bool dirty; std::string sequence; // optional @@ -144,10 +146,10 @@ public: CompactEdge(HashIntoType in_hash, HashIntoType out_hash) : in_hash(in_hash), out_hash(out_hash), meta(IS_FULL_EDGE), dirty(false) {} - CompactEdge(HashIntoType left, HashIntoType right, compact_edge_meta_t meta) : + CompactEdge(HashIntoType in_hash, HashIntoType out_hash, compact_edge_meta_t meta) : in_hash(in_hash), out_hash(out_hash), meta(meta), dirty(false) {} - void add_tags(HashSet& new_tags) { + void add_tags(UHashSet& new_tags) { for (auto tag: new_tags) { tags.insert(tag); } @@ -161,7 +163,7 @@ typedef std::pair TagEdgePair; typedef std::set TagEdgePairSet; -class GraphLinker +class StreamingCompactor { protected: // map from HDN hashes to CompactNode IDs @@ -173,41 +175,48 @@ protected: uint64_t n_sequences_added; uint64_t n_compact_edges; + uint64_t n_compact_nodes; + uint32_t tag_density; // protected linear creation of CompactNode // they should never be deleted, so this is straightforward CompactNode* new_compact_node(Kmer hdn) { CompactNode * v = get_compact_node_by_kmer(hdn); if (v == nullptr) { - compact_nodes.emplace_back(hdn); - v = &(compact_nodes.back); + compact_nodes.emplace_back(hdn, n_compact_nodes); + n_compact_nodes++; + v = &(compact_nodes.back()); hdn_ids[hdn] = v->node_id; } return v; } CompactEdge* new_compact_edge(HashIntoType left, HashIntoType right, - compact_edge_meta_t edge_meta, std::string edge_sequence) { + compact_edge_meta_t edge_meta, + std::string edge_sequence) { CompactEdge* edge = new CompactEdge(left, right, edge_meta); edge->sequence = edge_sequence; n_compact_edges++; + return edge; } void delete_compact_edge(CompactEdge * edge) { - for (auto tag: edge->tags) { - tags_to_edges.erase(tag); + if (edge != nullptr) { + for (auto tag: edge->tags) { + tags_to_edges.erase(tag); + } + delete edge; + n_compact_edges--; } - delete edge; - n_compact_edges--; } - void delete_compact_edge(HashSet& tags) { - CompactEdge* edge = get_compact_edge_from_tags(tags); - delete_compact_edge(edge): + void delete_compact_edge(UHashSet& tags) { + CompactEdge* edge = get_compact_edge(tags); + delete_compact_edge(edge); } void delete_compact_edge(HashIntoType tag) { - CompactEdge* edge = get_comapct_edge(tag); + CompactEdge* edge = get_compact_edge(tag); delete_compact_edge(edge); } @@ -215,28 +224,34 @@ public: shared_ptr graph; - GraphLinker(shared_ptr graph) : - graph(graph), n_sequences_added(0), n_compact_edges(0) + StreamingCompactor(shared_ptr graph) : + graph(graph), n_sequences_added(0), + n_compact_edges(0), n_compact_nodes(0) { - + tag_density = DEFAULT_TAG_DENSITY; } - WordLength ksize() const - { + WordLength ksize() const { return graph->ksize(); } - void report() const - { - std::cout << "GraphLinker (@" << this << " with " + uint64_t n_nodes() const { + return compact_nodes.size(); + } + + uint64_t n_edges() const { + return n_compact_edges; + } + + void report() const { + std::cout << "StreamingCompactor(@" << this << " with " << "Hashgraph @" << graph.get() << ")" << std::endl; std::cout << " * " << hdn_ids.size() << " cDBG nodes (HDNs)" << std::endl; std::cout << " * " << n_compact_edges << " cDBG edges" << std::endl; std::cout << " * " << n_sequences_added << " sequences added" << std::endl; } - CompactNode* get_compact_node_by_kmer(HashIntoType hdn) const - { + CompactNode* get_compact_node_by_kmer(HashIntoType hdn) { auto search = hdn_ids.find(hdn); if (search != hdn_ids.end()) { uint64_t ID = search->second; @@ -245,8 +260,14 @@ public: return nullptr; } - CompactNode* fetch_or_new_compact_node(Kmer hdn) - { + CompactNode* get_compact_node_by_id(uint64_t id) { + if (id < compact_nodes.size()) { + return nullptr; + } + return &(compact_nodes[id]); + } + + CompactNode* fetch_or_new_compact_node(Kmer hdn) { CompactNode* v = get_compact_node_by_kmer(hdn); if (v != nullptr) { v->count += 1; @@ -257,8 +278,7 @@ public: return v; } - std::vector get_compact_node_ids(const std::string& sequence) const - { + std::vector get_compact_node_ids(const std::string& sequence) { KmerIterator kmers(sequence.c_str(), graph->ksize()); std::vector ids; @@ -268,8 +288,8 @@ public: Kmer kmer = kmers.next(); node = get_compact_node_by_kmer(kmer); - if (j != nullptr) { - ids->push_back(node->node_id); + if (node != nullptr) { + ids.push_back(node->node_id); } } @@ -284,15 +304,17 @@ public: return nullptr; } - TagEdgePair get_tag_edge_pair(uint64_t tag) { + bool get_tag_edge_pair(uint64_t tag, TagEdgePair& pair) { auto search = tags_to_edges.find(tag); if (search != tags_to_edges.end()) { - return &search; + pair = *search; + return true; + } else { + return false; } - return nullptr; } - CompactEdge* get_edge_from_tags(HashSet& tags) { + CompactEdge* get_compact_edge(UHashSet& tags) { CompactEdge * edge = nullptr; for (auto tag: tags) { edge = get_compact_edge(tag); @@ -304,49 +326,59 @@ public: } KmerFilter get_tag_finder(TagEdgePairSet& found_pairs, - KmerSet& found_tags, - KmerSet& all_unresolved_tags) { + UHashSet& found_tags, + UHashSet& all_unresolved_tags) { + + KmerFilter finder = [&] (const Kmer& node) { - KmerFilter finder = [=] (const Kmer& node) { - TagEdgePair edge_pair = get_tag_edge_pair(node); - if (edge_pair != nullptr) { + TagEdgePair edge_pair; + bool found_pair = get_tag_edge_pair(node, edge_pair); + if (found_pair) { found_pairs.insert(edge_pair); found_tags.insert(node); } - if (set_contains(all_unresolved_tags, node) { + if (set_contains(all_unresolved_tags, node)) { found_tags.insert(node); } return false; // never filter node, just gather while we're looking - } + }; + + return finder; } - void update_compact_dbg(KmerSet& unresolved_tags) { + void update_compact_dbg(UHashSet& unresolved_tags) { - KmerQueue unresolved_q; + HashVector unresolved_q; for (auto tag: unresolved_tags) { unresolved_q.push_back(tag); } CompactingAssembler at(graph.get()); - Traverser traverser(graph.get()); - //shared_ptr visited = make_shared(); + while(unresolved_q.size() > 0) { - Kmer seed_tag = unresolved_q.back(); + HashIntoType seed_tag = unresolved_q.back(); + Kmer seed_kmer = graph->build_kmer(seed_tag); unresolved_q.pop_back(); - if (!set_contains(*unresolved_tags, start_tag)) { + if (!set_contains(unresolved_tags, seed_tag)) { continue; } - + + // build filters to prepare for traversal... + // first, tag gatherer TagEdgePairSet segment_edges; - KmerSet segment_tags; - KmerFilter tag_finder = get_tag_finder(segment_edges, - segment_tags, - *unresolved_tags); - CompactingAT lcursor(graph.get(), seed_tag); - lcursor.push_filter(tag_finder); - CompactingAT rcursor(graph.get(), seed_tag); - rcursor.push_filter(tag_finder); + UHashSet segment_tags; + KmerFilterList filters; + filters.push_back(get_tag_finder(segment_edges, + segment_tags, + unresolved_tags)); + // ... and a shared list of known k-mers + shared_ptr visited = make_shared(); + filters.push_back(get_visited_filter(visited)); + CompactingAT lcursor(graph.get(), seed_kmer, + filters); + CompactingAT rcursor(graph.get(), seed_kmer, + filters); // assemble both dirs until HDN, collecting tags along the way std::string left_seq = at._assemble_directed(lcursor); @@ -354,13 +386,44 @@ public: std::string edge_sequence = left_seq + right_seq.substr(graph->ksize()); // Generate new CompactNodes if needed - CompactNode* left_node, right_node; - left_node = right_node = nullptr; - if (traverser.degree(rcursor.cursor) > 2) { + CompactNode *left_node = nullptr, *right_node = nullptr; + // Check degree and gather neighbors simultaneously + // can't just check if total degree >2, as node could be: + // _(neighbor_1) + // (HDN)/ + // \_(neighbor_2) + // which we'd still like in the cDBG + KmerQueue rneighbors; + if (rcursor.neighbors(rneighbors) > 1 || + lcursor.neighbors(rcursor.cursor, rneighbors) > 1) { + right_node = fetch_or_new_compact_node(rcursor.cursor); + if (right_node->count == 1) { + while(rneighbors.size() > 0) { + // if it was a new HDN, we it could be between tag and + // nearest existing HDN; so, we have to check from its + // neighbors + auto rn = rneighbors.back(); + rneighbors.pop(); + unresolved_q.push_back(rn); + unresolved_tags.insert(rn); + } + } } - if (traverser.degree(lcursor.cursor) > 2) { + // same for other side + KmerQueue lneighbors; + if (lcursor.neighbors(lneighbors) > 1 || + rcursor.neighbors(lcursor.cursor, lneighbors) > 1) { + left_node = fetch_or_new_compact_node(lcursor.cursor); + if (left_node->count == 1) { + while(lneighbors.size() > 0) { + auto ln = lneighbors.back(); + lneighbors.pop(); + unresolved_q.push_back(ln); + unresolved_tags.insert(ln); + } + } } // Determine if we've got a tip, full edge, or island @@ -378,12 +441,17 @@ public: // first check if we've got a good Edge to work with CompactEdge* consensus_edge = nullptr; TagEdgePair consensus_tag_edge_pair; - for (consensus_tag_edge_pair: segment_edges) { - CompactEdge* intersect_edge = consensus_tag_edge_pair->second; - if (intersect_edge->in_hash == left_node->kmer && - intersect_edge->out_hash == right_node->kmer) { + for (auto tag_edge_pair: segment_edges) { + CompactEdge* intersect_edge = tag_edge_pair.second; + if (intersect_edge->meta != edge_meta) { + continue; + } + + if (intersect_edge->in_hash == (HashIntoType)(left_node->kmer) && + intersect_edge->out_hash == (HashIntoType)(right_node->kmer)) { consensus_edge = intersect_edge; + consensus_tag_edge_pair = tag_edge_pair; break; } } @@ -391,13 +459,13 @@ public: // ... if not, create one if (consensus_edge == nullptr) { consensus_edge = new_compact_edge(lcursor.cursor, - rcursor.cursor, - edge_meta, - edge_sequence); + rcursor.cursor, + edge_meta, + edge_sequence); } else { // if so, remove it from the set of found edges // so we can resolve them - segment_edges.remove(consensus_tag_edge_pair); + segment_edges.erase(consensus_tag_edge_pair); } // remove all of the tags for this segment from stale edges @@ -405,12 +473,12 @@ public: for (auto segment_tag: segment_tags) { for (auto tag_edge_pair: segment_edges) { - CompactEdge* stale_edge = tag_edge_pair->second; + CompactEdge* stale_edge = tag_edge_pair.second; stale_edge->tags.erase(segment_tag); } - consensus_edge->tags.insert(segment_tag); - tags_to_edges[segment_tag] = consensus_edge; + consensus_edge->tags.insert((HashIntoType)segment_tag); + tags_to_edges[(HashIntoType)segment_tag] = consensus_edge; // don't forget to remove from the unresolved set unresolved_tags.erase(segment_tag); } @@ -418,24 +486,27 @@ public: // one last run through to delete stale edges and add remaining tags // to the unresolved set for (auto tag_edge_pair: segment_edges) { - CompactEdge* edge = tag_edge_pair->second; - for (auto stale_tag: edge.tags) { - unresolved_tags.insert(tag_edge_pair->first); - tags_to_edges.erase(stale_tag); + CompactEdge* edge = tag_edge_pair.second; + if (edge != nullptr) { + for (auto stale_tag: edge->tags) { + unresolved_tags.insert(tag_edge_pair.first); + tags_to_edges.erase(stale_tag); + } } // don't forget to clean up delete_compact_edge(edge); } - if (left_node != nullptr) { - left_node->add_in_edge( + if (right_node != nullptr) { + right_node->add_in_edge( edge_sequence[edge_sequence.length()-(graph->ksize())], - edge); + consensus_edge); } - if (right_node != nullptr) { - right_node->add_out_edge( + + if (left_node != nullptr) { + left_node->add_out_edge( edge_sequence[graph->ksize()], - edge); + consensus_edge); } } @@ -444,55 +515,48 @@ public: void update_compact_dbg(const std::string& sequence) { std::cout << "update_compact_dbg()" << std::endl; - KmerSet new_tags; - KmerSet known_tags; + UHashSet new_tags; + UHashSet known_tags; uint64_t n_consumed = 0; + graph->consume_sequence(sequence); consume_sequence_and_tag(sequence, n_consumed, new_tags, known_tags); - - + update_compact_dbg(new_tags); } - // sigh at duplicated code from Hashgraph... perhaps in the future - // that function can be templated to accept multiple Set types for tags - void consume_sequence_and_tag(const std::string& seq, - unsigned long long& n_consumed, - KmerSet& new_tags, - KmerSet& known_tags) { + void index_sequence_tags(const std::string& seq, + KmerQueue unresolved_tags) { bool kmer_tagged; - KmerIterator kmers(seq.c_str(), _ksize); + KmerIterator kmers(seq.c_str(), ksize()); + Traverser traverser(graph.get()); Kmer kmer; - unsigned int since = _tag_density / 2 + 1; + KmerFilter finder = [&] (const Kmer& node) { + + }; - while(!kmers.done()) { - kmer = kmers.next(); - bool is_new_kmer; - if ((is_new_kmer = graph->add(kmer))) { - ++n_consumed; - } + uint32_t since = tag_density / 2 + 1; - if (is_new_kmer) { - ++since; + while(!kmers.done()) { + kmer = kmers.next(); + kmer_tagged = set_contains(tags_to_edges, kmer); + if (kmer_tagged) { + since = 1; } else { - kmer_tagged = set_contains(tags_to_edges, kmer); - if (kmer_tagged) { - since = 1; - known_tags.insert(kmer); - } else { - ++since; - } + ++since; } - if (since >= _tag_density) { + if (since >= tag_density) { new_tags.insert(kmer); since = 1; } + if (traverser.degree + } // iteration over kmers - if (since >= _tag_density/2 - 1) { + if (since >= tag_density/2 - 1) { new_tags.insert(kmer); // insert the last k-mer, too. } } diff --git a/include/oxli/oxli.hh b/include/oxli/oxli.hh index 682d09cba9..f8b6bb6059 100644 --- a/include/oxli/oxli.hh +++ b/include/oxli/oxli.hh @@ -72,6 +72,7 @@ private:\ #include #include #include +#include #include #include #include @@ -164,7 +165,8 @@ void deallocate_ptr_set(T& s) class Kmer; typedef std::queue KmerQueue; -typedef std::unordered_set KmerSet; +typedef std::set KmerSet; + // A function which takes a Kmer and returns true if it // is to be filtered / ignored diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index e3dfe0b2ce..effc8b5251 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -1,49 +1,106 @@ cimport cython from libcpp.memory cimport shared_ptr from libcpp.list cimport list as stdlist +from libcpp.pair cimport pair +from libcpp.unordered_set cimport unordered_set as uset +from libcpp.unordered_map cimport unordered_map as umap +from libcpp.vector cimport vector from libc.stdint cimport uint8_t, uint32_t, uint64_t from khmer._oxli.oxli_types cimport * +from khmer._oxli.hashing cimport CpKmer, Kmer from khmer._oxli.graphs cimport CpHashgraph, Hashgraph, Nodegraph, Countgraph -cdef extern from "oxli/links.hh" namespace "oxli": - ctypedef struct Junction: - HashIntoType u - HashIntoType v - HashIntoType w - uint64_t count +cdef extern from "oxli/links.hh" namespace "oxli" nogil: - ctypedef stdlist[Junction*] JunctionList + ctypedef pair[HashIntoType, uint64_t] HashIDPair + ctypedef uset[HashIntoType] UHashSet + ctypedef vector[HashIntoType] HashVector + ctypedef umap[HashIntoType, uint64_t] HashIDMap - cdef cppclass CpLink "oxli::Link": - CpLink(uint64_t, bool) - CpLink(uint64_t) - bool is_forward() - Junction* start_junction() - Junction* end_junction() - uint64_t size() + cdef cppclass CpCompactEdge "oxli::CompactEdge" + cdef cppclass CpCompactNode "oxli::CompactNode": + CpKmer kmer + uint32_t count + const uint64_t node_id - stdlist[Junction*].iterator begin() - stdlist[Junction*].iterator end() - JunctionList& get_junctions() + CpCompactEdge* in_edges[4] + CpCompactEdge* out_edges[4] - ctypedef stdlist[CpLink*] LinkList - - cdef cppclass CpGraphLinker "oxli::GraphLinker": - CpGraphLinker(shared_ptr[CpHashgraph]) + CpCompactNode(CpKmer) + void add_in_edge(const char, CpCompactEdge*) + CpCompactEdge* get_in_edge(const char) + void add_out_edge(const char, CpCompactEdge*) + CpCompactEdge* get_out_edge(const char) - Junction* get_junction(HashIntoType) - Junction* get_junction(HashIntoType, HashIntoType) - Junction* get_junction(Junction&) - shared_ptr[JunctionList] get_junctions(const string&) + uint8_t degree() + uint8_t out_degree() + uint8_t in_degree() - void add_links(const string&) - shared_ptr[LinkList] get_links(const string&) - void report() const + ctypedef enum compact_edge_meta_t: + IS_FULL_EDGE + IS_IN_TIP + IS_OUT_TIP + IS_ISLAND + IS_TRIVIAL + cdef cppclass CpCompactEdge "oxli::CompactEdge": + HashIntoType in_hash + HashIntoType out_hash + UHashSet tags + compact_edge_meta_t meta + string sequence -cdef class GraphLinker: + CpCompactEdge(HashIntoType, HashIntoType) + CpComapctEdge(HashIntoType, HashIntoType, compact_edge_meta_t) + + void add_tags(UHashSet&) + + ctypedef vector[CpCompactNode] CompactNodeVector + ctypedef umap[HashIntoType, CpCompactEdge*] TagEdgeMap + ctypedef pair[HashIntoType, CpCompactEdge*] TagEdgePair + ctypedef set[TagEdgePair] TagEdgePairSet + + cdef cppclass CpStreamingCompactor "oxli::StreamingCompactor": + shared_ptr[CpHashgraph] graph + + CpStreamingCompactor(shared_ptr[CpHashgraph]) + WordLength ksize() + void report() + uint64_t n_nodes() + uint64_t n_edges() + + CpCompactNode* get_compact_node_by_kmer(HashIntoType) + CpCompactNode* get_compact_node_by_id(uint64_t) + CpCompactNode* fetch_or_new_compact_node(CpKmer hdn) + vector[uint64_t] get_compact_node_ids(const string&) + + CpCompactEdge* get_compact_edge(uint64_t) + CpCompactEdge* get_tag_edge_pair(uint64_t, TagEdgePair&) + CpCompactEdge* get_compact_edge(UHashSet&) + + void update_compact_dbg(UHashSet&) + void update_compact_dbg(const string&) + void consume_sequence_and_tag(const string&, uint64_t&, UHashSet&, + UHashSet&) + +cdef class CompactNode: + cdef CpCompactNode* _cn_this + + @staticmethod + cdef CompactNode _wrap(CpCompactNode*) + + +cdef class CompactEdge: + cdef CpCompactEdge* _ce_this + + @staticmethod + cdef CompactEdge _wrap(CpCompactEdge*) + +cdef class StreamingCompactor: cdef shared_ptr[CpHashgraph] _graph - cdef shared_ptr[CpGraphLinker] _gl_this + cdef shared_ptr[CpStreamingCompactor] _sc_this + cdef Kmer kmer + diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 3b1208fdd4..12c060e614 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -2,40 +2,136 @@ from cython.operator cimport dereference as deref from libcpp.memory cimport make_shared from khmer._oxli.utils cimport _bstring +from khmer._oxli.sequence cimport Alphabets -cdef class GraphLinker: + +cdef class CompactEdge: + + @staticmethod + cdef CompactEdge _wrap(CpCompactEdge* _edge): + cdef CompactEdge edge = CompactEdge() + edge._ce_this = _edge + return edge + + def tags(self): + cdef HashIntoType tag + for tag in deref(self._ce_this).tags: + yield tag + + @property + def edge_type(self): + cdef compact_edge_meta_t meta = deref(self._ce_this).meta + if meta == IS_FULL_EDGE: + return 'FULL' + elif meta == IS_IN_TIP: + return 'IN_TIP' + elif meta == IS_OUT_TIP: + return 'OUT_TIP' + elif meta == IS_ISLAND: + return 'ISLAND' + elif meta == IS_TRIVIAL: + return 'TRIVIAL' + else: + raise ValueError('Malformed edge metadata') + + @property + def sequence(self): + return deref(self._ce_this).sequence + + def in_node(self): + if deref(self._ce_this).meta == IS_IN_TIP or \ + deref(self._ce_this).meta == IS_ISLAND: + return None + return deref(self._ce_this).in_hash + + def out_node(self): + if deref(self._ce_this).meta == IS_OUT_TIP or \ + deref(self._ce_this).meta == IS_ISLAND: + return None + return deref(self._ce_this).out_hash + + + +cdef class CompactNode: + + def __cinit__(self): + self.kmer = None + + @staticmethod + cdef CompactNode _wrap(CpCompactNode* _node): + cdef CompactNode node = CompactNode() + node._cn_this = _node + return node + + @property + def count(self): + return deref(self._cn_this).count + + @property + def node_id(self): + return deref(self._cn_this).node_id + + @property + def out_degree(self): + return deref(self._cn_this).out_degree() + + @property + def in_degree(self): + return deref(self._cn_this).in_degree() + + def node_kmer(self, WordLength K): + if self.kmer is None: + self.kmer = Kmer.wrap(&deref(self._cn_this).kmer, K) + return self.kmer + + def out_edges(self): + cdef string bases = Alphabets._get('DNA_SIMPLE') + cdef char base + cdef CpCompactEdge * edge + for base in bases: + edge = deref(self._cn_this).get_out_edge(base) + if edge != NULL: + return base, CompactEdge._wrap(edge) + + def in_edges(self): + cdef string bases = Alphabets._get('DNA_SIMPLE') + cdef char base + cdef CpCompactEdge * edge + for base in bases: + edge = deref(self._cn_this).get_in_edge(base) + if edge != NULL: + return base, CompactEdge._wrap(edge) + + +cdef class StreamingCompactor: def __cinit__(self, Hashgraph graph): self._graph = graph._hg_this - if type(self) is GraphLinker: - self._gl_this = make_shared[CpGraphLinker](self._graph) + if type(self) is StreamingCompactor: + self._sc_this = make_shared[CpStreamingCompactor](self._graph) - def add_links(self, str sequence): + def update(self, str sequence): cdef string _sequence = _bstring(sequence) - deref(self._gl_this).add_links(_sequence) + deref(self._sc_this).update_compact_dbg(_sequence) - def get_junctions(self, str sequence): + def sequence_nodes(self, str sequence): cdef string _sequence = _bstring(sequence) - cdef shared_ptr[JunctionList] junctions = deref(self._gl_this).get_junctions(_sequence) - cdef Junction* j - for j in deref(junctions): - yield deref(j) - - def get_links(self, str sequence): - cdef string _sequence = _bstring(sequence) - cdef shared_ptr[LinkList] links = deref(self._gl_this).get_links(_sequence) - - cdef CpLink* link - cdef Junction* j - cdef stdlist[Junction*].iterator it - for link in deref(links): - if deref(link).size() > 0: - ret = [] - for j in deref(link).get_junctions(): - ret.append(deref(j)) - yield ret - + cdef vector[uint64_t] node_ids = deref(self._sc_this).get_compact_node_ids(_sequence) + cdef HashIntoType node_id + cdef CpCompactNode* node + for node_id in node_ids: + node = deref(self._sc_this).get_compact_node_by_id(node_id) + if node != NULL: + yield CompactNode._wrap(node) def report(self): - deref(self._gl_this).report() + deref(self._sc_this).report() + + @property + def n_nodes(self): + return deref(self._sc_this).n_nodes() + + @property + def n_edges(self): + return deref(self._sc_this).n_edges() diff --git a/src/oxli/hashgraph.cc b/src/oxli/hashgraph.cc index dd3a95897c..6eadcaa141 100644 --- a/src/oxli/hashgraph.cc +++ b/src/oxli/hashgraph.cc @@ -204,7 +204,7 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, { bool kmer_tagged; if (tag_set == nullptr) { - tag_set = all_tags; + tag_set = &all_tags; } KmerIterator kmers(seq.c_str(), _ksize); @@ -230,7 +230,7 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, ++since; } else { ACQUIRE_ALL_TAGS_SPIN_LOCK - kmer_tagged = set_contains(tag_set, kmer); + kmer_tagged = set_contains(*tag_set, kmer); RELEASE_ALL_TAGS_SPIN_LOCK if (kmer_tagged) { since = 1; @@ -242,7 +242,7 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, } } #else - if (!is_new_kmer && set_contains(tag_set, kmer)) { + if (!is_new_kmer && set_contains(*tag_set, kmer)) { since = 1; if (found_tags != nullptr) { found_tags->insert(kmer); @@ -254,7 +254,7 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, if (since >= _tag_density) { ACQUIRE_ALL_TAGS_SPIN_LOCK - tag_set.insert(kmer); + tag_set->insert(kmer); RELEASE_ALL_TAGS_SPIN_LOCK if (found_tags != nullptr) { found_tags->insert(kmer); @@ -266,7 +266,7 @@ void Hashgraph::consume_sequence_and_tag(const std::string& seq, if (since >= _tag_density/2 - 1) { ACQUIRE_ALL_TAGS_SPIN_LOCK - tag_set.insert(kmer); // insert the last k-mer, too. + tag_set->insert(kmer); // insert the last k-mer, too. RELEASE_ALL_TAGS_SPIN_LOCK if (found_tags != nullptr) { found_tags->insert(kmer); diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 55d0df3d7d..a270e112ce 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -6,12 +6,10 @@ using namespace oxli; namespace oxli { -uint64_t CompactNode::node_counter = 0; - inline std::ostream& operator<< (std::ostream& stream, const CompactNode& node) { - stream << ""; + << " out_degree=" << node.out_degree() << ">"; return stream; } diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 29f4168616..b25f8e98de 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -8,7 +8,7 @@ from . import khmer_tst_utils as utils from .graph_features import * -from khmer._oxli.graphlinks import GraphLinker +from khmer._oxli.graphlinks import StreamingCompactor from khmer import Nodegraph import pytest @@ -17,38 +17,26 @@ def teardown(): utils.cleanup() -def test_get_junctions_single(right_tip_structure): +def test_compact_fork(right_tip_structure): '''Should have no links. Need two junctions. ''' graph, contig, L, HDN, R, tip = right_tip_structure - linker = GraphLinker(graph) - - linker.add_links(contig) - linker.report() + compactor = StreamingCompactor(graph) + compactor.update(contig) + compactor.report() - links = list(linker.get_links(contig)) - assert len(links) == 1 + nodes = list(compactor.sequence_nodes(contig)) + assert len(nodes) == 0 + assert compactor.n_nodes == 0 + assert compactor.n_edges == 0 - junctions = list(linker.get_junctions(contig)) - assert len(junctions) == 1 - junction = junctions.pop() - assert junction['count'] == 1 - assert junction['u'] == forward_hash(L, K) - assert junction['v'] == forward_hash(HDN, K) - assert junction['w'] == forward_hash(R, K) - - linker.add_links(contig) - linker.report() - links = list(linker.get_links(contig)) - print(links) - assert len(links) == 1 - junctions = list(linker.get_junctions(contig)) - assert len(junctions) == 1 - junction = junctions.pop() - assert junction['count'] == 2 + #assert junction['u'] == forward_hash(L, K) + #assert junction['v'] == forward_hash(HDN, K) + #assert junction['w'] == forward_hash(R, K) +''' def test_links_bubble(snp_bubble_structure): graph, wildtype_sequence, _, HDN_L, HDN_R = snp_bubble_structure linker = GraphLinker(graph) @@ -66,3 +54,4 @@ def test_links_bubble(snp_bubble_structure): assert link_b[0]['v'] == forward_hash(HDN_R, K) elif link_a[0]['v'] == forward_hash(HDN_R, K): assert link_b[0]['v'] == forward_hash(HDN_L, K) +''' From a4d9abf4c125c8c1ee0e610a5f1f1e57fb11bf0a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 24 Oct 2017 22:16:27 -0700 Subject: [PATCH 141/185] ignore ctags file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 300b2c4b86..40c917a9c9 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,4 @@ pep257_report.txt .cache/ khmer/_oxli/*.cpp .eggs +tags From ac95a46cfcebe7515b3e9d4111aa98f21492a3e0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 24 Oct 2017 22:16:51 -0700 Subject: [PATCH 142/185] Modify seeding method to create tags around high degree nodes --- include/oxli/links.hh | 53 +++++++++++++++++++++++---------------- include/oxli/oxli.hh | 2 +- khmer/_oxli/hashing.pxd | 4 +-- khmer/_oxli/traversal.pyx | 4 +-- src/oxli/assembler.cc | 4 +-- src/oxli/hashgraph.cc | 10 ++++---- src/oxli/partitioning.cc | 10 ++++---- src/oxli/subset.cc | 12 ++++----- src/oxli/traversal.cc | 2 +- 9 files changed, 55 insertions(+), 46 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index c59a84c806..26d5e85792 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -154,6 +154,10 @@ public: tags.insert(tag); } } + + float tag_density() const { + return (float)tags.size() / (float)sequence.length(); + } }; @@ -165,6 +169,9 @@ typedef std::set TagEdgePairSet; class StreamingCompactor { + friend class CompactEdge; + friend class CompactNode; + protected: // map from HDN hashes to CompactNode IDs HashIDMap hdn_ids; @@ -346,11 +353,12 @@ public: return finder; } - void update_compact_dbg(UHashSet& unresolved_tags) { + void update_compact_dbg(KmerQueue& unresolved_q) { - HashVector unresolved_q; - for (auto tag: unresolved_tags) { - unresolved_q.push_back(tag); + UHashSet unresolved_tags; + + for (auto tag: unresolved_q) { + unresolved_tags.insert(tag); } CompactingAssembler at(graph.get()); @@ -404,7 +412,7 @@ public: // nearest existing HDN; so, we have to check from its // neighbors auto rn = rneighbors.back(); - rneighbors.pop(); + rneighbors.pop_back(); unresolved_q.push_back(rn); unresolved_tags.insert(rn); } @@ -419,7 +427,7 @@ public: if (left_node->count == 1) { while(lneighbors.size() > 0) { auto ln = lneighbors.back(); - lneighbors.pop(); + lneighbors.pop_back(); unresolved_q.push_back(ln); unresolved_tags.insert(ln); } @@ -515,15 +523,14 @@ public: void update_compact_dbg(const std::string& sequence) { std::cout << "update_compact_dbg()" << std::endl; - UHashSet new_tags; - UHashSet known_tags; + KmerQueue unresolved_tags; uint64_t n_consumed = 0; - graph->consume_sequence(sequence); - consume_sequence_and_tag(sequence, n_consumed, new_tags, known_tags); - update_compact_dbg(new_tags); + graph->consume_string(sequence); + seed_sequence_tags(sequence, unresolved_tags); + update_compact_dbg(unresolved_tags); } - void index_sequence_tags(const std::string& seq, + void seed_sequence_tags(const std::string& seq, KmerQueue unresolved_tags) { bool kmer_tagged; @@ -531,33 +538,35 @@ public: Traverser traverser(graph.get()); Kmer kmer; - KmerFilter finder = [&] (const Kmer& node) { - - }; - - uint32_t since = tag_density / 2 + 1; while(!kmers.done()) { kmer = kmers.next(); - kmer_tagged = set_contains(tags_to_edges, kmer); - if (kmer_tagged) { + if (set_contains(tags_to_edges, kmer)) { since = 1; } else { ++since; } if (since >= tag_density) { - new_tags.insert(kmer); + unresolved_tags.push_back(kmer); since = 1; } - if (traverser.degree + if (traverser.degree_left(kmer) > 1 || + traverser.degree_right(kmer) > 1) { + + CompactNode* hdn = get_compact_node_by_kmer(kmer); + if (hdn == nullptr) { // new HDN + traverser.traverse(kmer, unresolved_tags); + } + + } } // iteration over kmers if (since >= tag_density/2 - 1) { - new_tags.insert(kmer); // insert the last k-mer, too. + unresolved_tags.push_back(kmer); // insert the last k-mer, too. } } diff --git a/include/oxli/oxli.hh b/include/oxli/oxli.hh index f8b6bb6059..aafbfd1e17 100644 --- a/include/oxli/oxli.hh +++ b/include/oxli/oxli.hh @@ -164,7 +164,7 @@ void deallocate_ptr_set(T& s) } class Kmer; -typedef std::queue KmerQueue; +typedef std::deque KmerQueue; typedef std::set KmerSet; diff --git a/khmer/_oxli/hashing.pxd b/khmer/_oxli/hashing.pxd index ae052e060d..0e0e65926f 100644 --- a/khmer/_oxli/hashing.pxd +++ b/khmer/_oxli/hashing.pxd @@ -1,6 +1,6 @@ from libcpp cimport bool from libcpp.memory cimport shared_ptr -from libcpp.queue cimport queue +from libcpp.deque cimport deque from libcpp.set cimport set from libcpp.string cimport string @@ -55,7 +55,7 @@ cdef extern from "oxli/kmer_hash.hh" namespace "oxli": cdef extern from "oxli/oxli.hh" namespace "oxli": - ctypedef queue[CpKmer] KmerQueue + ctypedef deque[CpKmer] KmerQueue ctypedef set[CpKmer] KmerSet ctypedef bool (*KmerFilter) (CpKmer kmer) diff --git a/khmer/_oxli/traversal.pyx b/khmer/_oxli/traversal.pyx index 52db83a1f6..dcf7455f48 100644 --- a/khmer/_oxli/traversal.pyx +++ b/khmer/_oxli/traversal.pyx @@ -27,7 +27,7 @@ cdef class Traverser: cpkmer = deref(kmers).front() kmer = Kmer.wrap(new CpKmer(cpkmer), deref(self._graph_ptr).ksize()) result.append(kmer) - deref(kmers).pop() + deref(kmers).pop_back() return result cdef list _kmerqueue_to_hash_list(self, KmerQueue * kmers): @@ -36,7 +36,7 @@ cdef class Traverser: while(deref(kmers).empty() == 0): cpkmer = deref(kmers).front() result.append(cpkmer.kmer_u) - deref(kmers).pop() + deref(kmers).pop_back() return result cdef list _neighbors(self, CpKmer start, int direction=0): diff --git a/src/oxli/assembler.cc b/src/oxli/assembler.cc index 692950d21e..0fe7676b52 100644 --- a/src/oxli/assembler.cc +++ b/src/oxli/assembler.cc @@ -411,7 +411,7 @@ const // spin off a cursor for the new branch AssemblerTraverser branch_cursor(cursor); branch_cursor.cursor = branch_starts.front(); - branch_starts.pop(); + branch_starts.pop_front(); #if DEBUG_ASSEMBLY std::cout << "Branch cursor: " << branch_cursor.cursor.repr( @@ -614,7 +614,7 @@ const AssemblerTraverser branch_cursor(cursor); branch_cursor.cursor = branch_starts.front(); - branch_starts.pop(); + branch_starts.pop_front(); // assemble linearly as far as possible std::string branch = linear_asm._assemble_directed(branch_cursor); diff --git a/src/oxli/hashgraph.cc b/src/oxli/hashgraph.cc index 6eadcaa141..08373d8a5c 100644 --- a/src/oxli/hashgraph.cc +++ b/src/oxli/hashgraph.cc @@ -435,7 +435,7 @@ const } KmerQueue node_q; - node_q.push(start); + node_q.push_front(start); // Avoid high-circumference k-mers Traverser traverser(this); @@ -447,7 +447,7 @@ const while(!node_q.empty()) { Kmer node = node_q.front(); - node_q.pop(); + node_q.pop_front(); // have we already seen me? don't count; exit. if (set_contains(keeper, node)) { @@ -527,12 +527,12 @@ const }; Traverser traverser(this, filter); - node_q.push(start); + node_q.push_front(start); breadth_q.push(0); while(!node_q.empty()) { Kmer node = node_q.front(); - node_q.pop(); + node_q.pop_front(); unsigned int breadth = breadth_q.front(); breadth_q.pop(); @@ -878,7 +878,7 @@ const while (node_q.size()) { Kmer node = node_q.front(); - node_q.pop(); + node_q.pop_front(); if (set_contains(high_degree_nodes, node)) { // if there are any adjacent high degree nodes, record; diff --git a/src/oxli/partitioning.cc b/src/oxli/partitioning.cc index 97df7da9d1..1ebebbbda6 100644 --- a/src/oxli/partitioning.cc +++ b/src/oxli/partitioning.cc @@ -266,7 +266,7 @@ uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, }*/ intersection.clear(); - seeds.push(kmer); + seeds.push_back(kmer); in_known_territory = false; found_tag_in_territory = false; ++since; @@ -297,11 +297,11 @@ uint64_t StreamingPartitioner::seed_sequence(const std::string& seq, if (since >= _tag_density / 2) { tags.push_back(kmer); } - seeds.push(kmer); + seeds.push_back(kmer); // now go back and make sure to search from the first k-mer kmer = kmers.first(); - seeds.push(kmer); + seeds.push_back(kmer); #if(DEBUG_SP) std::cout << "Done iterating k-mers" << std::endl; @@ -326,7 +326,7 @@ ComponentPtr StreamingPartitioner::find_nearest_component(Kmer kmer) const TagVector tags; std::set seen; KmerQueue node_q; - node_q.push(kmer); + node_q.push_front(kmer); find_connected_tags(node_q, tags, seen, true); if (tags.size() > 0) { @@ -364,7 +364,7 @@ void StreamingPartitioner::find_connected_tags(KmerQueue& node_q, while(!node_q.empty()) { Kmer node = node_q.front(); - node_q.pop(); + node_q.pop_front(); unsigned int breadth = breadth_q.front(); breadth_q.pop(); diff --git a/src/oxli/subset.cc b/src/oxli/subset.cc index 280d217a74..1116c80a4f 100644 --- a/src/oxli/subset.cc +++ b/src/oxli/subset.cc @@ -234,7 +234,7 @@ void SubsetPartition::find_all_tags( }; Traverser traverser(_ht, filter); - node_q.push(start_kmer); + node_q.push_front(start_kmer); breadth_q.push(0); while(!node_q.empty()) { @@ -245,7 +245,7 @@ void SubsetPartition::find_all_tags( } Kmer node = node_q.front(); - node_q.pop(); + node_q.pop_front(); unsigned int breadth = breadth_q.front(); breadth_q.pop(); @@ -331,7 +331,7 @@ unsigned int SubsetPartition::sweep_for_tags( Kmer node = kmers.next(); traversed_nodes.insert(node); - node_q.push(node); + node_q.push_front(node); breadth_q.push(0); } @@ -347,7 +347,7 @@ unsigned int SubsetPartition::sweep_for_tags( } Kmer node = node_q.front(); - node_q.pop(); + node_q.pop_front(); unsigned int breadth = breadth_q.front(); breadth_q.pop(); @@ -423,7 +423,7 @@ void SubsetPartition::find_all_tags_truncate_on_abundance( Traverser traverser(_ht, filter); - node_q.push(start_kmer); + node_q.push_front(start_kmer); breadth_q.push(0); while(!node_q.empty()) { @@ -433,7 +433,7 @@ void SubsetPartition::find_all_tags_truncate_on_abundance( } Kmer node = node_q.front(); - node_q.pop(); + node_q.pop_front(); unsigned int breadth = breadth_q.front(); breadth_q.pop(); diff --git a/src/oxli/traversal.cc b/src/oxli/traversal.cc index 6744d92a86..db29ea1433 100644 --- a/src/oxli/traversal.cc +++ b/src/oxli/traversal.cc @@ -126,7 +126,7 @@ const if (graph->get_count(neighbor)) { ++found; if (!apply_kmer_filters(neighbor, filters)) { - node_q.push(neighbor); + node_q.push_back(neighbor); } } ++base; From e80e372d79410a2f6fdf2862aa000b84a75edc35 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 26 Oct 2017 13:38:15 -0700 Subject: [PATCH 143/185] transition toward constrained approach... --- include/oxli/kmer_filters.hh | 1 + include/oxli/kmer_hash.hh | 8 + include/oxli/links.hh | 317 +++++++++++++++++++++++++---------- include/oxli/oxli.hh | 2 + include/oxli/traversal.hh | 23 +++ khmer/_oxli/graphlinks.pxd | 1 + src/oxli/kmer_filters.cc | 12 ++ src/oxli/links.cc | 10 -- src/oxli/traversal.cc | 3 +- tests/graph_features.py | 3 + tests/test_linked_dbg.py | 11 +- 11 files changed, 293 insertions(+), 98 deletions(-) diff --git a/include/oxli/kmer_filters.hh b/include/oxli/kmer_filters.hh index 35113248bc..c3e45021b4 100644 --- a/include/oxli/kmer_filters.hh +++ b/include/oxli/kmer_filters.hh @@ -52,6 +52,7 @@ class LabelHash; bool apply_kmer_filters(const Kmer& node, const KmerFilterList& filters); +void apply_kmer_helpers(const Kmer& node, const KmerHelperList& helpers); KmerFilter get_label_filter(const Label label, const LabelHash * lh); diff --git a/include/oxli/kmer_hash.hh b/include/oxli/kmer_hash.hh index 7ba6139dc5..c18a722951 100644 --- a/include/oxli/kmer_hash.hh +++ b/include/oxli/kmer_hash.hh @@ -217,6 +217,14 @@ public: { return kmer_f == kmer_u; } + + void set_forward() + { + if (!is_forward()) { + kmer_r = kmer_f; + kmer_f = kmer_u; + } + } }; diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 26d5e85792..df5d91c835 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -54,6 +54,16 @@ Contact: khmer-project@idyll.org #include "assembler.hh" +# define DEBUG_LINKS +# ifdef DEBUG_LINKS +# define pdebug(x) do { std::cout << std::endl << "@ " << __FILE__ <<\ + ":" << __FUNCTION__ << ":" <<\ + __LINE__ << std::endl << x << std::endl;\ + } while (0) +# else +# define pdebug(x) do {} while (0) +# endif + namespace oxli { using std::make_shared; @@ -120,8 +130,14 @@ public: return acc; } - friend std::ostream& operator<< (std::ostream& stream, - const CompactNode& node); + friend std::ostream& operator<<(std::ostream& stream, + const CompactNode& node) { + stream << ""; + return stream; + } }; @@ -136,18 +152,17 @@ enum compact_edge_meta_t { class CompactEdge { public: - HashIntoType in_hash; // left and right HDNs - HashIntoType out_hash; + Kmer in_node; // left and right HDNs + Kmer out_hash; UHashSet tags; compact_edge_meta_t meta; - bool dirty; - std::string sequence; // optional + std::string sequence; - CompactEdge(HashIntoType in_hash, HashIntoType out_hash) : - in_hash(in_hash), out_hash(out_hash), meta(IS_FULL_EDGE), dirty(false) {} + CompactEdge(Kmer in_node, Kmer out_node) : + in_node(in_node), out_node(out_node), meta(IS_FULL_EDGE) {} - CompactEdge(HashIntoType in_hash, HashIntoType out_hash, compact_edge_meta_t meta) : - in_hash(in_hash), out_hash(out_hash), meta(meta), dirty(false) {} + CompactEdge(Kmer in_node, Kmer out_node, compact_edge_meta_t meta) : + CompactEdge(in_node, out_node), meta(meta), {} void add_tags(UHashSet& new_tags) { for (auto tag: new_tags) { @@ -156,8 +171,23 @@ public: } float tag_density() const { - return (float)tags.size() / (float)sequence.length(); + return (float)sequence.length() / (float)tags.size(); + } + + std::string tag_viz(WordLength K) const { + uint64_t pos; + std::string ret = "L=" + std::to_string(sequence.length()) + " "; + const char * _s = sequence.c_str(); + + for (pos = 0; pos < sequence.length() - K + 1; pos++) { + if (set_contains(tags, _hash(_s+pos, K))) { + ret += ("(" + std::to_string(pos) + ")"); + } + ret += sequence[pos]; + } + return ret; } + }; @@ -165,6 +195,7 @@ typedef std::vector CompactNodeVector; typedef std::unordered_map TagEdgeMap; typedef std::pair TagEdgePair; typedef std::set TagEdgePairSet; +typedef std::set CompactEdgeSet; class StreamingCompactor @@ -180,6 +211,8 @@ protected: // map from tags to CompactEdges TagEdgeMap tags_to_edges; + KmerFilter compact_node_filter; + uint64_t n_sequences_added; uint64_t n_compact_edges; uint64_t n_compact_nodes; @@ -202,13 +235,17 @@ protected: compact_edge_meta_t edge_meta, std::string edge_sequence) { CompactEdge* edge = new CompactEdge(left, right, edge_meta); + pdebug("new compact edge: left=" << left << " right=" << right + << " sequence=" << edge_sequence); edge->sequence = edge_sequence; n_compact_edges++; return edge; } void delete_compact_edge(CompactEdge * edge) { + pdebug("attempt edge delete @" << edge); if (edge != nullptr) { + pdebug("edge not null, proceeding"); for (auto tag: edge->tags) { tags_to_edges.erase(tag); } @@ -236,6 +273,10 @@ public: n_compact_edges(0), n_compact_nodes(0) { tag_density = DEFAULT_TAG_DENSITY; + + compact_node_filter = [&] (const Kmer& node) { + return get_compact_node_by_kmer(node) != nullptr; + }; } WordLength ksize() const { @@ -332,57 +373,148 @@ public: return edge; } - KmerFilter get_tag_finder(TagEdgePairSet& found_pairs, - UHashSet& found_tags, - UHashSet& all_unresolved_tags) { - - KmerFilter finder = [&] (const Kmer& node) { + KmerFilter get_tag_stopper(TagEdgePair& te_pair, + bool& found_tag) { + KmerFilter stopper = [&] (const Kmer& node) { + found_tag = get_tag_edge_pair(node, te_pair); + return found_tag; + }; + } - TagEdgePair edge_pair; - bool found_pair = get_tag_edge_pair(node, edge_pair); - if (found_pair) { - found_pairs.insert(edge_pair); - found_tags.insert(node); + KmerHelper get_tag_collector(KmerSet& tags) { + KmerHelper collector = [&] (const Kmer& node) { + if (set_contains(tags_to_egdes, node)) { + tags.insert(node); } - if (set_contains(all_unresolved_tags, node)) { - found_tags.insert(node); - } - return false; // never filter node, just gather while we're looking }; - - return finder; } void update_compact_dbg(KmerQueue& unresolved_q) { - UHashSet unresolved_tags; - - for (auto tag: unresolved_q) { - unresolved_tags.insert(tag); + KmerIterator kmers(sequence.c_str(), ksize()); + + KmerSet induced_hdns; + Kmer kmer = kmers.next(); + kmer.set_forward(); + CompactingAT lcursor(graph.get(), kmer); + CompactingAT rcursor(graph.get(), kmer); + + while(!kmers.done()) { + if(lcursor.degree(kmer) > 1 || rcursor.degree(kmer) > 1) { + if (fetch_or_new_compact_node(kmer)->count == 1) { + induced_hdns.insert(kmer); + } + } + kmer = kmers.next(); } + CompactNode* left_flank=nullptr, right_flank=nullptr; + if ((left_flank = get_compact_node_by_kmer(lcursor.cursor)) != nullptr) { + + } + + + while(!induced_hdns.empty()) { + Kmer root_kmer = *induced_hdns.begin(); + induced_hdns.erase(root_hdn); + root_kmer.set_forward(); + + CompactNode* root_node = get_compact_node_by_kmer(root_kmer); + + KmerQueue left_neighbors; + lcursor.neighbors(root_kmer, left_neighbors); + while(!left_neighbors.empty()) { + Kmer neighbor = left_neighbors.back(); + left_neighbors.pop_back(); + lcursor.set_cursor(neighbor); + + KmerSet tags_left; + lcursor.push_helper(get_tag_collector(tags_left)); + std::string segment_seq = at._assemble_directed(lcursor); + + CompactNode* left_node = get_compact_node_by_kmer(lcursor.cursor); + CompactEdge* segment_edge = root_node->get_in_edge(*segment_seq.rbegin()); + + if (segment_edge == nullptr) { + if (!tags_left.empty()) { + segment_edge = get_compact_edge(*tags_left.begin()); + + } + } + + + } else { + segment_edge = + } + if (segment_edge->in_node == lcursor.cursor) { + // so far so good + if (segment_edge->out_node == root_node->kmer) { + // this edge is fine + + } else { + // root must have split this edge + // this edge's old out node must be right of root + // let's fix it! + + for (auto edge_tag : segment_edge->tags) { + if (!set_contains(segment_tags, edge_tag)) { + tags_to_edges.erase(edge_tag); + } + } + } + } else { + + } + } + + } + + lcursor.set_cursor(root_hdn); + rcursor.set_cursor(root_hdn); + + KmerSet tags_left, tags_right; + + rcursor.push_helper(get_tag_collector(tags_right)); + + std::string left_seq = at._assemble_directed(lcursor); + std::string right_seq = at._assemble_directed(rcursor); + + + + } + + CompactingAssembler at(graph.get()); - + CompactingAT lcursor(graph.get(), seed_kmer, + filters); + CompactingAT rcursor(graph.get(), seed_kmer, + filters); while(unresolved_q.size() > 0) { HashIntoType seed_tag = unresolved_q.back(); Kmer seed_kmer = graph->build_kmer(seed_tag); + seed_kmer.set_forward(); unresolved_q.pop_back(); - + pdebug("iter unresolved q, seed=" << seed_kmer << + ", " << unresolved_tags.size() << " unresolved tags left"); if (!set_contains(unresolved_tags, seed_tag)) { + pdebug("skip tag not in unresolved set"); continue; } // build filters to prepare for traversal... // first, tag gatherer - TagEdgePairSet segment_edges; + CompactEdgeSet segment_edges; UHashSet segment_tags; KmerFilterList filters; filters.push_back(get_tag_finder(segment_edges, segment_tags, unresolved_tags)); + // ... and a shared list of known k-mers shared_ptr visited = make_shared(); filters.push_back(get_visited_filter(visited)); + filters.push_back(compact_node_filter); + CompactingAT lcursor(graph.get(), seed_kmer, filters); CompactingAT rcursor(graph.get(), seed_kmer, @@ -393,6 +525,10 @@ public: std::string right_seq = at._assemble_directed(rcursor); std::string edge_sequence = left_seq + right_seq.substr(graph->ksize()); + pdebug("assembled linear path: " << segment_edges.size() << + " existing edges, " << segment_tags.size() << + " tags gathered, sequence=" << edge_sequence); + // Generate new CompactNodes if needed CompactNode *left_node = nullptr, *right_node = nullptr; // Check degree and gather neighbors simultaneously @@ -406,17 +542,8 @@ public: lcursor.neighbors(rcursor.cursor, rneighbors) > 1) { right_node = fetch_or_new_compact_node(rcursor.cursor); - if (right_node->count == 1) { - while(rneighbors.size() > 0) { - // if it was a new HDN, we it could be between tag and - // nearest existing HDN; so, we have to check from its - // neighbors - auto rn = rneighbors.back(); - rneighbors.pop_back(); - unresolved_q.push_back(rn); - unresolved_tags.insert(rn); - } - } + pdebug("has right HDN=" << *right_node); + } // same for other side KmerQueue lneighbors; @@ -424,14 +551,7 @@ public: rcursor.neighbors(lcursor.cursor, lneighbors) > 1) { left_node = fetch_or_new_compact_node(lcursor.cursor); - if (left_node->count == 1) { - while(lneighbors.size() > 0) { - auto ln = lneighbors.back(); - lneighbors.pop_back(); - unresolved_q.push_back(ln); - unresolved_tags.insert(ln); - } - } + pdebug("has left HDN=" << *left_node); } // Determine if we've got a tip, full edge, or island @@ -448,24 +568,22 @@ public: // first check if we've got a good Edge to work with CompactEdge* consensus_edge = nullptr; - TagEdgePair consensus_tag_edge_pair; - for (auto tag_edge_pair: segment_edges) { - CompactEdge* intersect_edge = tag_edge_pair.second; - if (intersect_edge->meta != edge_meta) { + for (auto intersecting_edge: segment_edges) { + if (intersecting_edge->meta != edge_meta) { continue; } - if (intersect_edge->in_hash == (HashIntoType)(left_node->kmer) && - intersect_edge->out_hash == (HashIntoType)(right_node->kmer)) { - - consensus_edge = intersect_edge; - consensus_tag_edge_pair = tag_edge_pair; + if (intersecting_edge->in_hash == (HashIntoType)lcursor.cursor && + intersecting_edge->out_hash == (HashIntoType)rcursor.cursor) { + + consensus_edge = intersecting_edge; break; } } // ... if not, create one if (consensus_edge == nullptr) { + pdebug("create consensus edge"); consensus_edge = new_compact_edge(lcursor.cursor, rcursor.cursor, edge_meta, @@ -473,15 +591,15 @@ public: } else { // if so, remove it from the set of found edges // so we can resolve them - segment_edges.erase(consensus_tag_edge_pair); + pdebug("use existing consensus edge"); + segment_edges.erase(consensus_edge); } // remove all of the tags for this segment from stale edges // and map them to the consensus edge for (auto segment_tag: segment_tags) { - for (auto tag_edge_pair: segment_edges) { - CompactEdge* stale_edge = tag_edge_pair.second; + for (auto stale_edge: segment_edges) { stale_edge->tags.erase(segment_tag); } @@ -493,52 +611,71 @@ public: // one last run through to delete stale edges and add remaining tags // to the unresolved set - for (auto tag_edge_pair: segment_edges) { - CompactEdge* edge = tag_edge_pair.second; - if (edge != nullptr) { - for (auto stale_tag: edge->tags) { - unresolved_tags.insert(tag_edge_pair.first); + for (auto stale_edge: segment_edges) { + if (stale_edge != nullptr) { + for (auto stale_tag: stale_edge->tags) { + unresolved_tags.insert(stale_tag); tags_to_edges.erase(stale_tag); } } // don't forget to clean up - delete_compact_edge(edge); + delete_compact_edge(stale_edge); } if (right_node != nullptr) { right_node->add_in_edge( edge_sequence[edge_sequence.length()-(graph->ksize())], consensus_edge); + if (right_node->count == 1) { + while(rneighbors.size() > 0) { + // if it was a new HDN, we it could be between tag and + // nearest existing HDN; so, we have to check from its + // neighbors + auto rn = rneighbors.back(); + rneighbors.pop_back(); + unresolved_q.push_back(rn); + unresolved_tags.insert(rn); + } + } } if (left_node != nullptr) { left_node->add_out_edge( edge_sequence[graph->ksize()], consensus_edge); + if (left_node->count == 1) { + while(lneighbors.size() > 0) { + auto ln = lneighbors.back(); + lneighbors.pop_back(); + unresolved_q.push_back(ln); + unresolved_tags.insert(ln); + } + } } } } void update_compact_dbg(const std::string& sequence) { - std::cout << "update_compact_dbg()" << std::endl; + pdebug("sequence=" << sequence); KmerQueue unresolved_tags; - uint64_t n_consumed = 0; + uint64_t prev_n_kmers = graph->n_unique_kmers(); graph->consume_string(sequence); - seed_sequence_tags(sequence, unresolved_tags); - update_compact_dbg(unresolved_tags); + if (graph->n_unique_kmers() - prev_n_kmers) { + seed_sequence_tags(sequence, unresolved_tags); + update_compact_dbg(unresolved_tags); + } } - void seed_sequence_tags(const std::string& seq, - KmerQueue unresolved_tags) { - bool kmer_tagged; + void orient_sequence(const std::string& seq, + KmerQueue& unresolved_tags) { KmerIterator kmers(seq.c_str(), ksize()); Traverser traverser(graph.get()); - Kmer kmer; + Kmer kmer = kmer.next(); - uint32_t since = tag_density / 2 + 1; + while(!kmers.done()) { kmer = kmers.next(); @@ -549,16 +686,26 @@ public: } if (since >= tag_density) { - unresolved_tags.push_back(kmer); + + if(!compact_node_filter(kmer)) { + unresolved_tags.push_back(kmer); + } + since = 1; } if (traverser.degree_left(kmer) > 1 || traverser.degree_right(kmer) > 1) { + pdebug("found HDN " << kmer); CompactNode* hdn = get_compact_node_by_kmer(kmer); if (hdn == nullptr) { // new HDN + pdebug("HDN is new, seed neighbors. " << + unresolved_tags.size() << " current tags"); + traverser.traverse(kmer, unresolved_tags); + + pdebug(unresolved_tags.size() << " after seeding"); } } @@ -566,8 +713,12 @@ public: } // iteration over kmers if (since >= tag_density/2 - 1) { - unresolved_tags.push_back(kmer); // insert the last k-mer, too. + if (!compact_node_filter(kmer)) { + unresolved_tags.push_back(kmer); // insert the last k-mer, too. + } } + + pdebug("seeded " << unresolved_tags.size() << " tags"); } }; @@ -577,6 +728,4 @@ public: } - - #endif diff --git a/include/oxli/oxli.hh b/include/oxli/oxli.hh index aafbfd1e17..b6a6fdd20a 100644 --- a/include/oxli/oxli.hh +++ b/include/oxli/oxli.hh @@ -171,7 +171,9 @@ typedef std::set KmerSet; // A function which takes a Kmer and returns true if it // is to be filtered / ignored typedef std::function KmerFilter; +typedef std::function KmerHelper; typedef std::list KmerFilterList; +typedef std::list KmerHelperList; typedef std::vector StringVector; } diff --git a/include/oxli/traversal.hh b/include/oxli/traversal.hh index 00deca679d..2e9aa35bc2 100644 --- a/include/oxli/traversal.hh +++ b/include/oxli/traversal.hh @@ -251,6 +251,7 @@ class AssemblerTraverser: public NodeCursor { protected: std::shared_ptr visited; + KmerHelperList helpers; public: using NodeCursor::NodeCursor; @@ -296,6 +297,28 @@ public: std::string join_contigs(std::string& contig_a, std::string& contig_b, WordLength offset = 0) const; + + void push_helper(KmerHelper helper) + { + filters.push_back(helper); + } + + KmerHelper pop_helper() + { + KmerHelper back = this->helpers.back(); + this->helpers.pop_back(); + return back; + } + + unsigned int n_filters() + { + return filters.size(); + } + + unsigned int n_helpers() + { + return helpers.size(); + } }; diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index effc8b5251..46cb710ac0 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -38,6 +38,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: uint8_t out_degree() uint8_t in_degree() + ctypedef enum compact_edge_meta_t: IS_FULL_EDGE IS_IN_TIP diff --git a/src/oxli/kmer_filters.cc b/src/oxli/kmer_filters.cc index 2386325355..82e092234d 100644 --- a/src/oxli/kmer_filters.cc +++ b/src/oxli/kmer_filters.cc @@ -61,6 +61,18 @@ bool apply_kmer_filters(const Kmer& node, const std::list& filters) } +void apply_kmer_helpers(const Kmer& node, const KmerHelperList& helpers) +{ + if (!filters.size()) { + return; + } + + for (auto helper: helpers) { + helper(node); + } +} + + KmerFilter get_label_filter(const Label label, const LabelHash * lh) { KmerFilter filter = [=] (const Kmer& node) { diff --git a/src/oxli/links.cc b/src/oxli/links.cc index a270e112ce..c6906a108f 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -4,13 +4,3 @@ using namespace oxli; -namespace oxli { - -inline std::ostream& operator<< (std::ostream& stream, const CompactNode& node) { - stream << ""; - return stream; -} - -} diff --git a/src/oxli/traversal.cc b/src/oxli/traversal.cc index db29ea1433..a7d0f034af 100644 --- a/src/oxli/traversal.cc +++ b/src/oxli/traversal.cc @@ -127,7 +127,7 @@ const ++found; if (!apply_kmer_filters(neighbor, filters)) { node_q.push_back(neighbor); - } + } } ++base; } @@ -342,6 +342,7 @@ char AssemblerTraverser::next_symbol() Kmer cursor_next; visited->insert(this->cursor); + apply_kmer_helpers(this->cursor, this->helpers); for (auto base : alphabets::DNA_SIMPLE) { // Get the putative neighbor for this base at the cursor position neighbor = NodeCursor::get_neighbor(this->cursor, base); diff --git a/tests/graph_features.py b/tests/graph_features.py index c2d6912846..3900d9228a 100644 --- a/tests/graph_features.py +++ b/tests/graph_features.py @@ -64,6 +64,9 @@ def __new__(cls, value, pos=0): raise ValueError('bad k-mer length') return str.__new__(cls, value) + def __repr__(self): + return str(self) + " @" + str(self.pos) + def mutate_base(base): if base in 'AT': diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index b25f8e98de..5627fdc81b 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -21,14 +21,19 @@ def test_compact_fork(right_tip_structure): '''Should have no links. Need two junctions. ''' graph, contig, L, HDN, R, tip = right_tip_structure + print("Contig FWD:", contig) + print("Contig RC:", revcomp(contig)) + print("HDN: ", repr(HDN)) + print("Tip FW:", tip) + print("Tip RC:", revcomp(tip)) compactor = StreamingCompactor(graph) compactor.update(contig) compactor.report() nodes = list(compactor.sequence_nodes(contig)) - assert len(nodes) == 0 - assert compactor.n_nodes == 0 - assert compactor.n_edges == 0 + assert len(nodes) == 1 + assert compactor.n_nodes == 1 + assert compactor.n_edges == 3 From 6b696f0b3560111af12e7918edd4e83ab204497b Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 31 Oct 2017 17:34:31 -0700 Subject: [PATCH 144/185] First pass at bounded compact dbg updating... --- include/oxli/links.hh | 417 ++++++++++++-------------------------- include/oxli/traversal.hh | 9 +- src/oxli/traversal.cc | 5 +- 3 files changed, 133 insertions(+), 298 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index df5d91c835..fb15bd26de 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -90,6 +90,16 @@ public: return lhs.node_id == rhs.node_id; } + bool delete_in_edge(CompactEdge* edge) { + for (uint8_t i=0; i<4; i++) { + if (in_edges[i] == edge) { + in_edges[i] = nullptr; + return true; + } + } + return false; + } + void add_in_edge(const char base, CompactEdge* edge) { in_edges[twobit_repr(base)] = edge; } @@ -98,6 +108,16 @@ public: return in_edges[twobit_repr(base)]; } + bool delete_out_edge(CompactEdge* edge) { + for (uint8_t i=0; i<4; i++) { + if (out_edges[i] == edge) { + out_edges[i] = nullptr; + return true; + } + } + return false; + } + void add_out_edge(const char base, CompactEdge* edge) { out_edges[twobit_repr(base)] = edge; } @@ -231,7 +251,7 @@ protected: return v; } - CompactEdge* new_compact_edge(HashIntoType left, HashIntoType right, + CompactEdge* new_compact_edge(Kmer left, Kmer right, compact_edge_meta_t edge_meta, std::string edge_sequence) { CompactEdge* edge = new CompactEdge(left, right, edge_meta); @@ -389,30 +409,72 @@ public: }; } - void update_compact_dbg(KmerQueue& unresolved_q) { + compact_edge_meta_t deduce_edge_meta(CompactNode* in, CompactNode* out) { + compact_edge_meta_t edge_meta; + if (in == nullptr && out == nullptr) { + edge_meta = IS_ISLAND; + } else if (out == nullptr) { + edge_meta = IS_OUT_TIP; + } else if (in == nullptr) { + edge_meta = IS_IN_TIP; + } else { + edge_meta = IS_FULL_EDGE; + } + return edge_meta; + } - KmerIterator kmers(sequence.c_str(), ksize()); + /* Update a compact dbg where there are no induced + * HDNs */ + void update_compact_dbg_linear(std::string& sequence) { - KmerSet induced_hdns; + } + + uint64_t update_compact_dbg(const std::string& sequence) { + uint64_t prev_n_kmers = graph->n_unique_kmers(); + graph->consume_string(sequence); + if (graph->n_unique_kmers() - prev_n_kmers == 0) { + // only need to update if there's something new + return 0; + } + + // first gather up all k-mers that could have been disturbed -- + // k-mers in the read, and the neighbors of the flanking nodes + KmerIterator kmers(sequence.c_str(), ksize()); + KmerQueue disturbed_kmers; Kmer kmer = kmers.next(); kmer.set_forward(); CompactingAT lcursor(graph.get(), kmer); + lcursor.neighbors(disturbed_kmers); + while(!kmers.done()) { + kmer = kmers.next(); + kmer.set_forward(); + disturbed_kmers.push_back(kmer); + } CompactingAT rcursor(graph.get(), kmer); + rcursor.neighbors(disturbed_kmers); - while(!kmers.done()) { + // find the induced HDNs in the disturbed k-mers + KmerSet induced_hdns; + uint64_t n_updates = 0; + while(!disturbed_kmers.empty()) { + Kmer kmer = disturbed_kmers.back(); + disturbed_kmers.pop_back(); if(lcursor.degree(kmer) > 1 || rcursor.degree(kmer) > 1) { if (fetch_or_new_compact_node(kmer)->count == 1) { induced_hdns.insert(kmer); + n_updates++; } } - kmer = kmers.next(); - } - CompactNode* left_flank=nullptr, right_flank=nullptr; - if ((left_flank = get_compact_node_by_kmer(lcursor.cursor)) != nullptr) { - } + /* If there are no induced HDNs, we must have extended + * a tip or merged two tips into a linear segment */ + // handle_merge() + + /* Update from all induced HDNs + */ + KmerQueue neighbors; while(!induced_hdns.empty()) { Kmer root_kmer = *induced_hdns.begin(); induced_hdns.erase(root_hdn); @@ -420,305 +482,74 @@ public: CompactNode* root_node = get_compact_node_by_kmer(root_kmer); - KmerQueue left_neighbors; - lcursor.neighbors(root_kmer, left_neighbors); - while(!left_neighbors.empty()) { - Kmer neighbor = left_neighbors.back(); - left_neighbors.pop_back(); + // check left (in) edges + lcursor.neighbors(root_kmer, neighbors); + while(!neighbors.empty()) { + Kmer neighbor = neighbors.back(); + neighbors.pop_back(); lcursor.set_cursor(neighbor); - KmerSet tags_left; - lcursor.push_helper(get_tag_collector(tags_left)); - std::string segment_seq = at._assemble_directed(lcursor); - - CompactNode* left_node = get_compact_node_by_kmer(lcursor.cursor); - CompactEdge* segment_edge = root_node->get_in_edge(*segment_seq.rbegin()); - - if (segment_edge == nullptr) { - if (!tags_left.empty()) { - segment_edge = get_compact_edge(*tags_left.begin()); - - } - } + TagEdgePair left_tag_pair; + bool found_left_tag; + lcursor.push_filter(get_tag_stopper(left_tag_pair, found_left_tag)); + std::string segment_seq = at._assemble_directed(lcursor); - } else { - segment_edge = - } - if (segment_edge->in_node == lcursor.cursor) { - // so far so good - if (segment_edge->out_node == root_node->kmer) { - // this edge is fine - - } else { - // root must have split this edge - // this edge's old out node must be right of root - // let's fix it! - - for (auto edge_tag : segment_edge->tags) { - if (!set_contains(segment_tags, edge_tag)) { - tags_to_edges.erase(edge_tag); - } - } - } + CompactEdge* segment_edge = nullptr; + CompactNode* left_node = nullptr; + /* + * Should also keep a set of pair to track resolved + * segments + */ + if (found_left_tag) { + // must be an existing segment + segment_edge = get_compact_edge(tag_edge_pair.first); + left_node = get_compact_node_by_kmer(segment_edge->in_node); + // check if we need to split the edge + if (*(segment_edge->out_node) == *root_node) { + // good, we're set + continue; // continue through neighbors } else { - - } - } - - } - - lcursor.set_cursor(root_hdn); - rcursor.set_cursor(root_hdn); - - KmerSet tags_left, tags_right; - - rcursor.push_helper(get_tag_collector(tags_right)); - - std::string left_seq = at._assemble_directed(lcursor); - std::string right_seq = at._assemble_directed(rcursor); - - - - } - - - - CompactingAssembler at(graph.get()); - CompactingAT lcursor(graph.get(), seed_kmer, - filters); - CompactingAT rcursor(graph.get(), seed_kmer, - filters); - while(unresolved_q.size() > 0) { - HashIntoType seed_tag = unresolved_q.back(); - Kmer seed_kmer = graph->build_kmer(seed_tag); - seed_kmer.set_forward(); - unresolved_q.pop_back(); - pdebug("iter unresolved q, seed=" << seed_kmer << - ", " << unresolved_tags.size() << " unresolved tags left"); - if (!set_contains(unresolved_tags, seed_tag)) { - pdebug("skip tag not in unresolved set"); - continue; - } - - // build filters to prepare for traversal... - // first, tag gatherer - CompactEdgeSet segment_edges; - UHashSet segment_tags; - KmerFilterList filters; - filters.push_back(get_tag_finder(segment_edges, - segment_tags, - unresolved_tags)); - - // ... and a shared list of known k-mers - shared_ptr visited = make_shared(); - filters.push_back(get_visited_filter(visited)); - filters.push_back(compact_node_filter); - - CompactingAT lcursor(graph.get(), seed_kmer, - filters); - CompactingAT rcursor(graph.get(), seed_kmer, - filters); - - // assemble both dirs until HDN, collecting tags along the way - std::string left_seq = at._assemble_directed(lcursor); - std::string right_seq = at._assemble_directed(rcursor); - std::string edge_sequence = left_seq + right_seq.substr(graph->ksize()); - - pdebug("assembled linear path: " << segment_edges.size() << - " existing edges, " << segment_tags.size() << - " tags gathered, sequence=" << edge_sequence); - - // Generate new CompactNodes if needed - CompactNode *left_node = nullptr, *right_node = nullptr; - // Check degree and gather neighbors simultaneously - // can't just check if total degree >2, as node could be: - // _(neighbor_1) - // (HDN)/ - // \_(neighbor_2) - // which we'd still like in the cDBG - KmerQueue rneighbors; - if (rcursor.neighbors(rneighbors) > 1 || - lcursor.neighbors(rcursor.cursor, rneighbors) > 1) { - - right_node = fetch_or_new_compact_node(rcursor.cursor); - pdebug("has right HDN=" << *right_node); - - } - // same for other side - KmerQueue lneighbors; - if (lcursor.neighbors(lneighbors) > 1 || - rcursor.neighbors(lcursor.cursor, lneighbors) > 1) { - - left_node = fetch_or_new_compact_node(lcursor.cursor); - pdebug("has left HDN=" << *left_node); - } - - // Determine if we've got a tip, full edge, or island - compact_edge_meta_t edge_meta; - if (left_node == nullptr && right_node == nullptr) { - edge_meta = IS_ISLAND; - } else if (right_node == nullptr) { - edge_meta = IS_OUT_TIP; - } else if (left_node == nullptr) { - edge_meta = IS_IN_TIP; - } else { - edge_meta = IS_FULL_EDGE; - } - - // first check if we've got a good Edge to work with - CompactEdge* consensus_edge = nullptr; - for (auto intersecting_edge: segment_edges) { - if (intersecting_edge->meta != edge_meta) { - continue; - } - - if (intersecting_edge->in_hash == (HashIntoType)lcursor.cursor && - intersecting_edge->out_hash == (HashIntoType)rcursor.cursor) { - - consensus_edge = intersecting_edge; - break; - } - } - - // ... if not, create one - if (consensus_edge == nullptr) { - pdebug("create consensus edge"); - consensus_edge = new_compact_edge(lcursor.cursor, - rcursor.cursor, - edge_meta, - edge_sequence); - } else { - // if so, remove it from the set of found edges - // so we can resolve them - pdebug("use existing consensus edge"); - segment_edges.erase(consensus_edge); - } - - // remove all of the tags for this segment from stale edges - // and map them to the consensus edge - for (auto segment_tag: segment_tags) { - - for (auto stale_edge: segment_edges) { - stale_edge->tags.erase(segment_tag); - } - - consensus_edge->tags.insert((HashIntoType)segment_tag); - tags_to_edges[(HashIntoType)segment_tag] = consensus_edge; - // don't forget to remove from the unresolved set - unresolved_tags.erase(segment_tag); - } - - // one last run through to delete stale edges and add remaining tags - // to the unresolved set - for (auto stale_edge: segment_edges) { - if (stale_edge != nullptr) { - for (auto stale_tag: stale_edge->tags) { - unresolved_tags.insert(stale_tag); - tags_to_edges.erase(stale_tag); - } - } - // don't forget to clean up - delete_compact_edge(stale_edge); - } - - if (right_node != nullptr) { - right_node->add_in_edge( - edge_sequence[edge_sequence.length()-(graph->ksize())], - consensus_edge); - if (right_node->count == 1) { - while(rneighbors.size() > 0) { - // if it was a new HDN, we it could be between tag and - // nearest existing HDN; so, we have to check from its - // neighbors - auto rn = rneighbors.back(); - rneighbors.pop_back(); - unresolved_q.push_back(rn); - unresolved_tags.insert(rn); - } - } - } - - if (left_node != nullptr) { - left_node->add_out_edge( - edge_sequence[graph->ksize()], - consensus_edge); - if (left_node->count == 1) { - while(lneighbors.size() > 0) { - auto ln = lneighbors.back(); - lneighbors.pop_back(); - unresolved_q.push_back(ln); - unresolved_tags.insert(ln); + // need to be careful here, the out node for the + // edge we delete could be linked to another induced + // HDN... + n_updates++; + segment_edge->out_node->delete_in_edge(segment_edge); + delete_compact_edge(segment_edge); + // deleted tags; assemble out to the HDN + segment_seq = at._assemble_directed(lcursor) + + segment_seq.substr(ksize()); + if (left_node != nullptr) { + // not an IN_TIP + left_node->delete_out_edge(segment_edge); + } } + } else { + // then left node must have been new, or does not exist + left_node = get_compact_node_by_kmer(lcursor.cursor); } - } - - } - } - - void update_compact_dbg(const std::string& sequence) { - pdebug("sequence=" << sequence); - - KmerQueue unresolved_tags; - uint64_t prev_n_kmers = graph->n_unique_kmers(); - graph->consume_string(sequence); - if (graph->n_unique_kmers() - prev_n_kmers) { - seed_sequence_tags(sequence, unresolved_tags); - update_compact_dbg(unresolved_tags); - } - } - - void orient_sequence(const std::string& seq, - KmerQueue& unresolved_tags) { - KmerIterator kmers(seq.c_str(), ksize()); - Traverser traverser(graph.get()); - Kmer kmer = kmer.next(); + // construct the compact edge + segment_seq = segment_seq.substr(1); + compact_edge_meta_t edge_meta = (left_node == nullptr) ? + IS_IN_TIP : IS_FULL_EDGE; - + n_update++; + segment_edge = new_compact_edge(lcursor.cursor, + root_node->kmer, + edge_meta, + segment_seq); + if (IS_FULL_EDGE) { + left_node->add_out_edge(segment_seq[ksize()], segment_edge); + } - while(!kmers.done()) { - kmer = kmers.next(); - if (set_contains(tags_to_edges, kmer)) { - since = 1; - } else { - ++since; - } - - if (since >= tag_density) { - - if(!compact_node_filter(kmer)) { - unresolved_tags.push_back(kmer); - } - - since = 1; } - if (traverser.degree_left(kmer) > 1 || - traverser.degree_right(kmer) > 1) { - - pdebug("found HDN " << kmer); - CompactNode* hdn = get_compact_node_by_kmer(kmer); - if (hdn == nullptr) { // new HDN - pdebug("HDN is new, seed neighbors. " << - unresolved_tags.size() << " current tags"); + // now the right neighbors... - traverser.traverse(kmer, unresolved_tags); - pdebug(unresolved_tags.size() << " after seeding"); - } - - } - - } // iteration over kmers - - if (since >= tag_density/2 - 1) { - if (!compact_node_filter(kmer)) { - unresolved_tags.push_back(kmer); // insert the last k-mer, too. - } } - pdebug("seeded " << unresolved_tags.size() << " tags"); } }; diff --git a/include/oxli/traversal.hh b/include/oxli/traversal.hh index 2e9aa35bc2..f625d82c87 100644 --- a/include/oxli/traversal.hh +++ b/include/oxli/traversal.hh @@ -136,8 +136,9 @@ public: * * @return Number of neighbors total (could be more than those found). */ + template unsigned int neighbors(const Kmer& node, - KmerQueue &node_q) const; + Container &found) const; /** * @brief Get the degree of the given Kmer in the templated direction. @@ -185,11 +186,13 @@ public: * * @return Number of neighbors found. */ - unsigned int neighbors(KmerQueue& node_q) const + template + unsigned int neighbors(Container& found) const { - return NodeGatherer::neighbors(cursor, node_q); + return NodeGatherer::neighbors(cursor, found); } + /** * @return Degree of the current cursor position and direction. */ diff --git a/src/oxli/traversal.cc b/src/oxli/traversal.cc index a7d0f034af..2b923af4f3 100644 --- a/src/oxli/traversal.cc +++ b/src/oxli/traversal.cc @@ -113,8 +113,9 @@ const template +template unsigned int NodeGatherer::neighbors(const Kmer& node, - KmerQueue & node_q) + Container& found) const { unsigned int found = 0; @@ -126,7 +127,7 @@ const if (graph->get_count(neighbor)) { ++found; if (!apply_kmer_filters(neighbor, filters)) { - node_q.push_back(neighbor); + found.insert(found.end(), neighbor); } } ++base; From 814a4f13b3114b062b7d95ef3d51ffda399f0655 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 1 Nov 2017 16:51:12 -0700 Subject: [PATCH 145/185] compator redux with left exploration --- include/oxli/assembler.hh | 3 +- include/oxli/links.hh | 56 ++++++++++++++++++-------------------- include/oxli/traversal.hh | 24 ++++++++++------ khmer/_oxli/graphlinks.pxd | 17 ++++++++---- khmer/_oxli/graphlinks.pyx | 13 +++++++-- khmer/_oxli/hashing.pxd | 2 ++ khmer/_oxli/hashing.pyx | 7 +++++ src/oxli/kmer_filters.cc | 2 +- src/oxli/traversal.cc | 27 +++++++++++++----- 9 files changed, 96 insertions(+), 55 deletions(-) diff --git a/include/oxli/assembler.hh b/include/oxli/assembler.hh index 8e637e9270..85fbdf2bd7 100644 --- a/include/oxli/assembler.hh +++ b/include/oxli/assembler.hh @@ -111,7 +111,8 @@ class CompactingAssembler: public LinearAssembler public: explicit CompactingAssembler(const Hashgraph* ht, - std::shared_ptr global_visited=nullptr) : LinearAssembler(ht, global_visited) {} + std::shared_ptr global_visited=nullptr) + : LinearAssembler(ht, global_visited) {} virtual std::string assemble(const Kmer seed_kmer, const Hashgraph * stop_bf) const; diff --git a/include/oxli/links.hh b/include/oxli/links.hh index fb15bd26de..42aa08373c 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -173,7 +173,7 @@ enum compact_edge_meta_t { class CompactEdge { public: Kmer in_node; // left and right HDNs - Kmer out_hash; + Kmer out_node; UHashSet tags; compact_edge_meta_t meta; std::string sequence; @@ -182,7 +182,7 @@ public: in_node(in_node), out_node(out_node), meta(IS_FULL_EDGE) {} CompactEdge(Kmer in_node, Kmer out_node, compact_edge_meta_t meta) : - CompactEdge(in_node, out_node), meta(meta), {} + in_node(in_node), out_node(out_node), meta(meta) {} void add_tags(UHashSet& new_tags) { for (auto tag: new_tags) { @@ -212,6 +212,7 @@ public: typedef std::vector CompactNodeVector; +typedef std::vector CompactEdgeVector; typedef std::unordered_map TagEdgeMap; typedef std::pair TagEdgePair; typedef std::set TagEdgePairSet; @@ -231,8 +232,6 @@ protected: // map from tags to CompactEdges TagEdgeMap tags_to_edges; - KmerFilter compact_node_filter; - uint64_t n_sequences_added; uint64_t n_compact_edges; uint64_t n_compact_nodes; @@ -294,9 +293,6 @@ public: { tag_density = DEFAULT_TAG_DENSITY; - compact_node_filter = [&] (const Kmer& node) { - return get_compact_node_by_kmer(node) != nullptr; - }; } WordLength ksize() const { @@ -399,14 +395,8 @@ public: found_tag = get_tag_edge_pair(node, te_pair); return found_tag; }; - } - KmerHelper get_tag_collector(KmerSet& tags) { - KmerHelper collector = [&] (const Kmer& node) { - if (set_contains(tags_to_egdes, node)) { - tags.insert(node); - } - }; + return stopper; } compact_edge_meta_t deduce_edge_meta(CompactNode* in, CompactNode* out) { @@ -423,6 +413,19 @@ public: return edge_meta; } + uint64_t consume_sequence(const std::string& sequence) { + uint64_t prev_n_kmers = graph->n_unique_kmers(); + graph->consume_string(sequence); + return graph->n_unique_kmers() - prev_n_kmers; + } + + uint64_t consume_sequence_and_update(const std::string& sequence) { + if (consume_sequence(sequence) > 0) { + return update_compact_dbg(sequence); + } + return 0; + } + /* Update a compact dbg where there are no induced * HDNs */ void update_compact_dbg_linear(std::string& sequence) { @@ -430,13 +433,7 @@ public: } uint64_t update_compact_dbg(const std::string& sequence) { - uint64_t prev_n_kmers = graph->n_unique_kmers(); - graph->consume_string(sequence); - if (graph->n_unique_kmers() - prev_n_kmers == 0) { - // only need to update if there's something new - return 0; - } - + // first gather up all k-mers that could have been disturbed -- // k-mers in the read, and the neighbors of the flanking nodes KmerIterator kmers(sequence.c_str(), ksize()); @@ -474,10 +471,11 @@ public: /* Update from all induced HDNs */ + CompactingAssembler cassem(graph.get()); KmerQueue neighbors; while(!induced_hdns.empty()) { Kmer root_kmer = *induced_hdns.begin(); - induced_hdns.erase(root_hdn); + induced_hdns.erase(root_kmer); root_kmer.set_forward(); CompactNode* root_node = get_compact_node_by_kmer(root_kmer); @@ -487,13 +485,13 @@ public: while(!neighbors.empty()) { Kmer neighbor = neighbors.back(); neighbors.pop_back(); - lcursor.set_cursor(neighbor); + lcursor.cursor = neighbor; TagEdgePair left_tag_pair; bool found_left_tag; lcursor.push_filter(get_tag_stopper(left_tag_pair, found_left_tag)); - std::string segment_seq = at._assemble_directed(lcursor); + std::string segment_seq = cassem._assemble_directed(lcursor); CompactEdge* segment_edge = nullptr; CompactNode* left_node = nullptr; @@ -503,10 +501,10 @@ public: */ if (found_left_tag) { // must be an existing segment - segment_edge = get_compact_edge(tag_edge_pair.first); + segment_edge = get_compact_edge(left_tag_pair.first); left_node = get_compact_node_by_kmer(segment_edge->in_node); // check if we need to split the edge - if (*(segment_edge->out_node) == *root_node) { + if (*get_compact_node_by_kmer(segment_edge->out_node) == *root_node) { // good, we're set continue; // continue through neighbors } else { @@ -514,10 +512,10 @@ public: // edge we delete could be linked to another induced // HDN... n_updates++; - segment_edge->out_node->delete_in_edge(segment_edge); + get_compact_node_by_kmer(segment_edge->out_node)->delete_in_edge(segment_edge); delete_compact_edge(segment_edge); // deleted tags; assemble out to the HDN - segment_seq = at._assemble_directed(lcursor) + + segment_seq = cassem._assemble_directed(lcursor) + segment_seq.substr(ksize()); if (left_node != nullptr) { // not an IN_TIP @@ -534,7 +532,7 @@ public: compact_edge_meta_t edge_meta = (left_node == nullptr) ? IS_IN_TIP : IS_FULL_EDGE; - n_update++; + n_updates++; segment_edge = new_compact_edge(lcursor.cursor, root_node->kmer, edge_meta, diff --git a/include/oxli/traversal.hh b/include/oxli/traversal.hh index f625d82c87..0d3bb6f4d1 100644 --- a/include/oxli/traversal.hh +++ b/include/oxli/traversal.hh @@ -189,7 +189,7 @@ public: template unsigned int neighbors(Container& found) const { - return NodeGatherer::neighbors(cursor, found); + return NodeGatherer::neighbors(cursor, found); } @@ -257,7 +257,11 @@ protected: KmerHelperList helpers; public: - using NodeCursor::NodeCursor; + + using NodeCursor::push_filter; + + explicit AssemblerTraverser(const Hashgraph * ht, + Kmer start_kmer); explicit AssemblerTraverser(const Hashgraph* ht, Kmer start_kmer, @@ -274,6 +278,11 @@ public: AssemblerTraverser(const AssemblerTraverser& other); + void _init_visited() { + visited = std::make_shared(); + push_filter(get_visited_filter(visited)); + } + /** * @brief Get the next symbol. @@ -303,7 +312,7 @@ public: void push_helper(KmerHelper helper) { - filters.push_back(helper); + helpers.push_back(helper); } KmerHelper pop_helper() @@ -313,11 +322,6 @@ public: return back; } - unsigned int n_filters() - { - return filters.size(); - } - unsigned int n_helpers() { return helpers.size(); @@ -333,6 +337,10 @@ protected: Traverser traverser; public: + + explicit CompactingAT(const Hashgraph * ht, + Kmer start_kmer); + explicit CompactingAT(const Hashgraph * ht, Kmer start_kmer, KmerFilter filter); diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 46cb710ac0..4dc5e303fd 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -30,8 +30,10 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpCompactNode(CpKmer) void add_in_edge(const char, CpCompactEdge*) + bool delete_in_edge(CpCompactEdge*) CpCompactEdge* get_in_edge(const char) void add_out_edge(const char, CpCompactEdge*) + bool delete_out_edge(CpCompactEdge*) CpCompactEdge* get_out_edge(const char) uint8_t degree() @@ -47,8 +49,8 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: IS_TRIVIAL cdef cppclass CpCompactEdge "oxli::CompactEdge": - HashIntoType in_hash - HashIntoType out_hash + CpKmer in_node + CpKmer out_node UHashSet tags compact_edge_meta_t meta string sequence @@ -57,6 +59,8 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpComapctEdge(HashIntoType, HashIntoType, compact_edge_meta_t) void add_tags(UHashSet&) + string tag_viz(WordLength) + float tag_density() ctypedef vector[CpCompactNode] CompactNodeVector ctypedef umap[HashIntoType, CpCompactEdge*] TagEdgeMap @@ -81,10 +85,10 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpCompactEdge* get_tag_edge_pair(uint64_t, TagEdgePair&) CpCompactEdge* get_compact_edge(UHashSet&) - void update_compact_dbg(UHashSet&) - void update_compact_dbg(const string&) - void consume_sequence_and_tag(const string&, uint64_t&, UHashSet&, - UHashSet&) + uint64_t update_compact_dbg(const string&) + uint64_t consume_sequence(const string&) + uint64_t consume_sequence_and_update(const string&) + cdef class CompactNode: cdef CpCompactNode* _cn_this @@ -99,6 +103,7 @@ cdef class CompactEdge: @staticmethod cdef CompactEdge _wrap(CpCompactEdge*) + cdef class StreamingCompactor: cdef shared_ptr[CpHashgraph] _graph diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 12c060e614..9837673825 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -42,14 +42,13 @@ cdef class CompactEdge: if deref(self._ce_this).meta == IS_IN_TIP or \ deref(self._ce_this).meta == IS_ISLAND: return None - return deref(self._ce_this).in_hash + return deref(self._ce_this).in_node.kmer_u def out_node(self): if deref(self._ce_this).meta == IS_OUT_TIP or \ deref(self._ce_this).meta == IS_ISLAND: return None - return deref(self._ce_this).out_hash - + return deref(self._ce_this).out_node.kmer_u cdef class CompactNode: @@ -115,6 +114,14 @@ cdef class StreamingCompactor: cdef string _sequence = _bstring(sequence) deref(self._sc_this).update_compact_dbg(_sequence) + def consume(self, str sequence): + cdef string _sequence = _bstring(sequence) + return deref(self._sc_this).consume_sequence(_sequence) + + def consume_and_update(self, str sequence): + cdef string _sequence = _bstring(sequence) + return deref(self._sc_this).consume_sequence_and_update(sequence) + def sequence_nodes(self, str sequence): cdef string _sequence = _bstring(sequence) cdef vector[uint64_t] node_ids = deref(self._sc_this).get_compact_node_ids(_sequence) diff --git a/khmer/_oxli/hashing.pxd b/khmer/_oxli/hashing.pxd index 0e0e65926f..46d9fb7cf9 100644 --- a/khmer/_oxli/hashing.pxd +++ b/khmer/_oxli/hashing.pxd @@ -66,6 +66,8 @@ cdef class Kmer: @staticmethod cdef Kmer wrap(CpKmer * cpkmer, WordLength K) + @staticmethod + cdef Kmer wrap_partial(CpKmer *cpkmer) cpdef HashIntoType forward_hash(object kmer, unsigned int K) diff --git a/khmer/_oxli/hashing.pyx b/khmer/_oxli/hashing.pyx index 265b1ef789..996ab9d839 100644 --- a/khmer/_oxli/hashing.pyx +++ b/khmer/_oxli/hashing.pyx @@ -59,6 +59,13 @@ cdef class Kmer: kmer.kmer = _revhash(kmer.kmer_u, K) return kmer + @staticmethod + cdef Kmer wrap_partial(CpKmer* cpkmer): + cdef Kmer kmer = Kmer() + kmer._this.reset(cpkmer) + kmer.kmer = "" + return kmer + @staticmethod def create(HashIntoType tag, WordLength K): cdef Kmer kmer = Kmer() diff --git a/src/oxli/kmer_filters.cc b/src/oxli/kmer_filters.cc index 82e092234d..8d7e98b38b 100644 --- a/src/oxli/kmer_filters.cc +++ b/src/oxli/kmer_filters.cc @@ -63,7 +63,7 @@ bool apply_kmer_filters(const Kmer& node, const std::list& filters) void apply_kmer_helpers(const Kmer& node, const KmerHelperList& helpers) { - if (!filters.size()) { + if (!helpers.size()) { return; } diff --git a/src/oxli/traversal.cc b/src/oxli/traversal.cc index 2b923af4f3..a2024dd234 100644 --- a/src/oxli/traversal.cc +++ b/src/oxli/traversal.cc @@ -118,14 +118,14 @@ unsigned int NodeGatherer::neighbors(const Kmer& node, Container& found) const { - unsigned int found = 0; + unsigned int n_found = 0; for (auto base : alphabets::DNA_SIMPLE) { // Get the putative neighboring Kmer Kmer neighbor = get_neighbor(node, base); // Now check if it's in the graph and passes the filters if (graph->get_count(neighbor)) { - ++found; + ++n_found; if (!apply_kmer_filters(neighbor, filters)) { found.insert(found.end(), neighbor); } @@ -133,7 +133,7 @@ const ++base; } - return found; + return n_found; } @@ -277,6 +277,14 @@ unsigned int Traverser::degree_right(const Kmer& node) const * AssemblerTraverser ******************************************/ +template +AssemblerTraverser::AssemblerTraverser(const Hashgraph * ht, + Kmer start_kmer) : + NodeCursor(ht, start_kmer) +{ + _init_visited(); +} + template AssemblerTraverser::AssemblerTraverser(const Hashgraph * ht, Kmer start_kmer, @@ -284,8 +292,7 @@ AssemblerTraverser::AssemblerTraverser(const Hashgraph * ht, NodeCursor(ht, start_kmer, filters) { - visited = std::make_shared(); - AssemblerTraverser::push_filter(get_visited_filter(visited)); + _init_visited(); } template @@ -305,8 +312,7 @@ AssemblerTraverser::AssemblerTraverser(const Hashgraph * ht, KmerFilter filter) : NodeCursor(ht, start_kmer, filter) { - visited = std::make_shared(); - AssemblerTraverser::push_filter(get_visited_filter(visited)); + _init_visited(); } @@ -375,6 +381,13 @@ char AssemblerTraverser::next_symbol() * CompactingAT ******************************************/ +template +CompactingAT::CompactingAT(const Hashgraph * ht, + Kmer start_kmer) : + AssemblerTraverser(ht, start_kmer), traverser(ht) +{ +} + template CompactingAT::CompactingAT(const Hashgraph * ht, Kmer start_kmer, From c60c23932cee0e1229cde9c6f0f98b1cdb6cfe6a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 1 Nov 2017 21:37:39 -0700 Subject: [PATCH 146/185] Fix kmer indexing for compact edges, now working on forks --- include/oxli/links.hh | 224 +++++++++++++++++++++++++++---------- khmer/_oxli/graphlinks.pxd | 5 +- khmer/_oxli/graphlinks.pyx | 40 +++++-- tests/test_linked_dbg.py | 46 +++++++- 4 files changed, 240 insertions(+), 75 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 42aa08373c..3357a8f527 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -74,11 +74,85 @@ typedef std::unordered_set UHashSet; typedef std::vector HashVector; typedef std::unordered_map HashIDMap; -class CompactEdge; + +enum compact_edge_meta_t { + IS_FULL_EDGE, + IS_IN_TIP, + IS_OUT_TIP, + IS_ISLAND, + IS_TRIVIAL +}; + + +inline const char * edge_meta_repr(compact_edge_meta_t meta) { + switch(meta) { + case IS_FULL_EDGE: + return "FULL EDGE"; + case IS_IN_TIP: + return "IN TIP"; + case IS_OUT_TIP: + return "OUT TIP"; + case IS_ISLAND: + return "ISLAND"; + case IS_TRIVIAL: + return "TRIVIAL"; + } +} + + +class CompactEdge { +public: + Kmer in_node; // left and right HDNs + Kmer out_node; + UHashSet tags; + compact_edge_meta_t meta; + std::string sequence; + + CompactEdge(Kmer in_node, Kmer out_node) : + in_node(in_node), out_node(out_node), meta(IS_FULL_EDGE) {} + + CompactEdge(Kmer in_node, Kmer out_node, compact_edge_meta_t meta) : + in_node(in_node), out_node(out_node), meta(meta) {} + + void add_tags(UHashSet& new_tags) { + for (auto tag: new_tags) { + tags.insert(tag); + } + } + + float tag_density() const { + return (float)sequence.length() / (float)tags.size(); + } + + std::string tag_viz(WordLength K) const { + uint64_t pos; + std::string ret = "L=" + std::to_string(sequence.length()) + " "; + const char * _s = sequence.c_str(); + + for (pos = 0; pos < sequence.length() - K + 1; pos++) { + if (set_contains(tags, _hash(_s+pos, K))) { + ret += ("(" + std::to_string(pos) + ")"); + } + ret += sequence[pos]; + } + return ret; + } + + friend std::ostream& operator<<(std::ostream& stream, + const CompactEdge& edge) { + stream << ""; + return stream; + } + +}; + class CompactNode { public: Kmer kmer; uint32_t count; + std::string sequence; const uint64_t node_id; CompactEdge* in_edges[4] = {nullptr, nullptr, nullptr, nullptr}; CompactEdge* out_edges[4] = {nullptr, nullptr, nullptr, nullptr}; @@ -86,6 +160,9 @@ public: CompactNode(Kmer kmer, uint64_t node_id) : kmer(kmer), count(0), node_id(node_id) {} + CompactNode(Kmer kmer, std::string sequence, uint64_t node_id) : + kmer(kmer), count(0), sequence(sequence), node_id(node_id) {} + friend bool operator== (const CompactNode& lhs, const CompactNode& rhs) { return lhs.node_id == rhs.node_id; } @@ -101,6 +178,8 @@ public: } void add_in_edge(const char base, CompactEdge* edge) { + pdebug("add in edge to " << *this << ", base=" << base + << ", edge: " << *edge); in_edges[twobit_repr(base)] = edge; } @@ -119,6 +198,8 @@ public: } void add_out_edge(const char base, CompactEdge* edge) { + pdebug("add out edge to " << *this << ", base=" << base + << ", edge: " << *edge); out_edges[twobit_repr(base)] = edge; } @@ -161,56 +242,6 @@ public: }; -enum compact_edge_meta_t { - IS_FULL_EDGE, - IS_IN_TIP, - IS_OUT_TIP, - IS_ISLAND, - IS_TRIVIAL -}; - - -class CompactEdge { -public: - Kmer in_node; // left and right HDNs - Kmer out_node; - UHashSet tags; - compact_edge_meta_t meta; - std::string sequence; - - CompactEdge(Kmer in_node, Kmer out_node) : - in_node(in_node), out_node(out_node), meta(IS_FULL_EDGE) {} - - CompactEdge(Kmer in_node, Kmer out_node, compact_edge_meta_t meta) : - in_node(in_node), out_node(out_node), meta(meta) {} - - void add_tags(UHashSet& new_tags) { - for (auto tag: new_tags) { - tags.insert(tag); - } - } - - float tag_density() const { - return (float)sequence.length() / (float)tags.size(); - } - - std::string tag_viz(WordLength K) const { - uint64_t pos; - std::string ret = "L=" + std::to_string(sequence.length()) + " "; - const char * _s = sequence.c_str(); - - for (pos = 0; pos < sequence.length() - K + 1; pos++) { - if (set_contains(tags, _hash(_s+pos, K))) { - ret += ("(" + std::to_string(pos) + ")"); - } - ret += sequence[pos]; - } - return ret; - } - -}; - - typedef std::vector CompactNodeVector; typedef std::vector CompactEdgeVector; typedef std::unordered_map TagEdgeMap; @@ -240,12 +271,16 @@ protected: // protected linear creation of CompactNode // they should never be deleted, so this is straightforward CompactNode* new_compact_node(Kmer hdn) { + pdebug("new compact node from " << hdn); CompactNode * v = get_compact_node_by_kmer(hdn); if (v == nullptr) { + hdn.set_forward(); compact_nodes.emplace_back(hdn, n_compact_nodes); n_compact_nodes++; v = &(compact_nodes.back()); + v->sequence = _revhash(hdn, ksize()); // hopefully new k-mer type in future hdn_ids[hdn] = v->node_id; + pdebug("Allocate: " << *v); } return v; } @@ -342,9 +377,10 @@ public: return v; } - std::vector get_compact_node_ids(const std::string& sequence) { + std::vector get_compact_nodes(const std::string& sequence) { + pdebug("get compact node IDs"); KmerIterator kmers(sequence.c_str(), graph->ksize()); - std::vector ids; + std::vector nodes; CompactNode* node; @@ -353,14 +389,16 @@ public: node = get_compact_node_by_kmer(kmer); if (node != nullptr) { - ids.push_back(node->node_id); + pdebug("found compact node " << *node); + nodes.push_back(node); } } - return ids; + return nodes; } - CompactEdge* get_compact_edge(uint64_t tag) { + CompactEdge* get_compact_edge(HashIntoType tag) { + pdebug("get compact edge from tag " << tag); auto search = tags_to_edges.find(tag); if (search != tags_to_edges.end()) { return search->second; @@ -368,7 +406,7 @@ public: return nullptr; } - bool get_tag_edge_pair(uint64_t tag, TagEdgePair& pair) { + bool get_tag_edge_pair(HashIntoType tag, TagEdgePair& pair) { auto search = tags_to_edges.find(tag); if (search != tags_to_edges.end()) { pair = *search; @@ -433,6 +471,7 @@ public: } uint64_t update_compact_dbg(const std::string& sequence) { + pdebug("update cDBG from " << sequence); // first gather up all k-mers that could have been disturbed -- // k-mers in the read, and the neighbors of the flanking nodes @@ -463,6 +502,8 @@ public: } } } + pdebug(disturbed_kmers.size() << " k-mers disturbed" << std::endl + << induced_hdns.size() << " induced HDNs"); /* If there are no induced HDNs, we must have extended * a tip or merged two tips into a linear segment */ @@ -479,9 +520,11 @@ public: root_kmer.set_forward(); CompactNode* root_node = get_compact_node_by_kmer(root_kmer); + pdebug("searching from induced HDN: " << *root_node); // check left (in) edges lcursor.neighbors(root_kmer, neighbors); + pdebug("checking " << neighbors.size() << " left neighbors"); while(!neighbors.empty()) { Kmer neighbor = neighbors.back(); neighbors.pop_back(); @@ -528,9 +571,14 @@ public: } // construct the compact edge - segment_seq = segment_seq.substr(1); compact_edge_meta_t edge_meta = (left_node == nullptr) ? IS_IN_TIP : IS_FULL_EDGE; + if (edge_meta == IS_FULL_EDGE) { + segment_seq = segment_seq.substr(ksize()-1, + segment_seq.length()-ksize()+1); + } else { + segment_seq = segment_seq.substr(0, segment_seq.length()-ksize()+1); + } n_updates++; segment_edge = new_compact_edge(lcursor.cursor, @@ -538,13 +586,69 @@ public: edge_meta, segment_seq); if (IS_FULL_EDGE) { - left_node->add_out_edge(segment_seq[ksize()], segment_edge); - } + left_node->add_out_edge(segment_seq.front(), segment_edge); + } + root_node->add_in_edge(segment_seq.back(), segment_edge); } // now the right neighbors... + rcursor.neighbors(root_kmer, neighbors); + pdebug("checking " << neighbors.size() << " right neighbors"); + while(!neighbors.empty()) { + Kmer neighbor = neighbors.back(); + neighbors.pop_back(); + rcursor.cursor = neighbor; + + TagEdgePair right_tag_pair; + bool found_right_tag; + + rcursor.push_filter(get_tag_stopper(right_tag_pair, found_right_tag)); + std::string segment_seq = cassem._assemble_directed(rcursor); + + CompactEdge* segment_edge = nullptr; + CompactNode* right_node = nullptr; + + if (found_right_tag) { + segment_edge = get_compact_edge(right_tag_pair.first); + right_node = get_compact_node_by_kmer(segment_edge->out_node); + if (*get_compact_node_by_kmer(segment_edge->in_node) == *root_node) { + continue; + } else { + n_updates++; + get_compact_node_by_kmer(segment_edge->in_node)->delete_out_edge(segment_edge); + delete_compact_edge(segment_edge); + segment_seq = segment_seq + + cassem._assemble_directed(rcursor).substr(ksize()); + if (right_node != nullptr) { + right_node->delete_in_edge(segment_edge); + } + } + } else { + right_node = get_compact_node_by_kmer(rcursor.cursor); + } + + compact_edge_meta_t edge_meta = (right_node == nullptr) ? + IS_OUT_TIP : IS_FULL_EDGE; + if (edge_meta == IS_FULL_EDGE) { + segment_seq = segment_seq.substr(ksize()-1, + segment_seq.length()-ksize()+1); + } else { + segment_seq = segment_seq.substr(ksize()-1); + } + n_updates++; + segment_edge = new_compact_edge(root_node->kmer, + rcursor.cursor, + edge_meta, + segment_seq); + if (IS_FULL_EDGE) { + right_node->add_in_edge(segment_seq.back(), segment_edge); + } + + root_node->add_out_edge(segment_seq.front(), segment_edge); + + } } diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 4dc5e303fd..29b5948756 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -24,6 +24,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpKmer kmer uint32_t count const uint64_t node_id + string sequence CpCompactEdge* in_edges[4] CpCompactEdge* out_edges[4] @@ -79,7 +80,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpCompactNode* get_compact_node_by_kmer(HashIntoType) CpCompactNode* get_compact_node_by_id(uint64_t) CpCompactNode* fetch_or_new_compact_node(CpKmer hdn) - vector[uint64_t] get_compact_node_ids(const string&) + vector[CpCompactNode*] get_compact_nodes(const string&) CpCompactEdge* get_compact_edge(uint64_t) CpCompactEdge* get_tag_edge_pair(uint64_t, TagEdgePair&) @@ -92,6 +93,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: cdef class CompactNode: cdef CpCompactNode* _cn_this + cdef public Kmer kmer @staticmethod cdef CompactNode _wrap(CpCompactNode*) @@ -108,5 +110,4 @@ cdef class StreamingCompactor: cdef shared_ptr[CpHashgraph] _graph cdef shared_ptr[CpStreamingCompactor] _sc_this - cdef Kmer kmer diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 9837673825..a62723b13e 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -50,6 +50,15 @@ cdef class CompactEdge: return None return deref(self._ce_this).out_node.kmer_u + def __len__(self): + return deref(self._ce_this).sequence.length() + + def __str__(self): + return 'CompactEdge: L={0} sequence={1}'.format(len(self), self.sequence) + + def __repr__(self): + return str(self) + cdef class CompactNode: @@ -78,6 +87,18 @@ cdef class CompactNode: def in_degree(self): return deref(self._cn_this).in_degree() + @property + def ID(self): + return deref(self._cn_this).node_id + + @property + def kmer_hash(self): + return deref(self._cn_this).kmer.kmer_u + + @property + def sequence(self): + return deref(self._cn_this).sequence + def node_kmer(self, WordLength K): if self.kmer is None: self.kmer = Kmer.wrap(&deref(self._cn_this).kmer, K) @@ -90,7 +111,7 @@ cdef class CompactNode: for base in bases: edge = deref(self._cn_this).get_out_edge(base) if edge != NULL: - return base, CompactEdge._wrap(edge) + yield base, CompactEdge._wrap(edge) def in_edges(self): cdef string bases = Alphabets._get('DNA_SIMPLE') @@ -99,8 +120,14 @@ cdef class CompactNode: for base in bases: edge = deref(self._cn_this).get_in_edge(base) if edge != NULL: - return base, CompactEdge._wrap(edge) + yield base, CompactEdge._wrap(edge) + def __str__(self): + return 'CompactNode: ID={0} count={1} in_degree={2}'\ + ' out_degree={3} sequence={4}'.format(self.kmer, self.count, + self.in_degree, + self.out_degree, + self.sequence) cdef class StreamingCompactor: @@ -124,13 +151,10 @@ cdef class StreamingCompactor: def sequence_nodes(self, str sequence): cdef string _sequence = _bstring(sequence) - cdef vector[uint64_t] node_ids = deref(self._sc_this).get_compact_node_ids(_sequence) - cdef HashIntoType node_id + cdef vector[CpCompactNode*] nodes = deref(self._sc_this).get_compact_nodes(_sequence) cdef CpCompactNode* node - for node_id in node_ids: - node = deref(self._sc_this).get_compact_node_by_id(node_id) - if node != NULL: - yield CompactNode._wrap(node) + for node in nodes: + yield CompactNode._wrap(node) def report(self): deref(self._sc_this).report() diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 5627fdc81b..d05ed41bde 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -6,6 +6,7 @@ from khmer import reverse_hash as revhash from khmer import forward_hash from . import khmer_tst_utils as utils +from .khmer_tst_utils import _equals_rc, _contains_rc from .graph_features import * from khmer._oxli.graphlinks import StreamingCompactor @@ -21,11 +22,7 @@ def test_compact_fork(right_tip_structure): '''Should have no links. Need two junctions. ''' graph, contig, L, HDN, R, tip = right_tip_structure - print("Contig FWD:", contig) - print("Contig RC:", revcomp(contig)) - print("HDN: ", repr(HDN)) - print("Tip FW:", tip) - print("Tip RC:", revcomp(tip)) + compactor = StreamingCompactor(graph) compactor.update(contig) compactor.report() @@ -35,7 +32,46 @@ def test_compact_fork(right_tip_structure): assert compactor.n_nodes == 1 assert compactor.n_edges == 3 + node = nodes[0] + assert _equals_rc(node.sequence, HDN) + + in_edges = list(node.in_edges()) + out_edges = list(node.out_edges()) + if len(in_edges) == 1: + _, in_edge = in_edges[0] + assert len(out_edges) == 2 + (_, edge_contig), (_, edge_tip) = out_edges + if len(edge_tip) > len(edge_contig): + edge_contig, edge_tip = edge_tip, edge_contig + assert _equals_rc(contig, in_edge.sequence + node.sequence + + edge_contig.sequence) + else: + _, out_edge = out_edges[0] + assert len(in_edges) == 2 + (_, edge_contig), (_, edge_tip) = in_edges + if len(edge_tip) > len(edge_contig): + edge_contig, edge_tip = edge_tip, edge_contig + assert _equals_rc(contig, edge_contig.sequence + node.sequence + + out_edge.sequence) + + for node in nodes: + print(node) + print('in edges:') + for base, edge in node.in_edges(): + print(base, edge) + + print('out edges:') + for base, edge in node.out_edges(): + print(base, edge) + + print("Contig FWD:", contig, len(contig)) + print("Contig RC:", revcomp(contig)) + print("HDN: ", repr(HDN)) + print("Tip FW:", tip, len(tip)) + print("Tip RC:", revcomp(tip)) + print("R FW:", R) + print("R RC:", revcomp(R)) #assert junction['u'] == forward_hash(L, K) #assert junction['v'] == forward_hash(HDN, K) From e36dc8f4ab1fe7aaca744b1ed0ce4293e84ee943 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 1 Nov 2017 21:56:54 -0700 Subject: [PATCH 147/185] Add revcomp test and remove some debug output --- include/oxli/links.hh | 27 ++++++++------- khmer/_oxli/graphlinks.pyx | 2 +- tests/test_linked_dbg.py | 70 +++++++++++++++++++++----------------- 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 3357a8f527..5a6cd88a6f 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -178,8 +178,8 @@ public: } void add_in_edge(const char base, CompactEdge* edge) { - pdebug("add in edge to " << *this << ", base=" << base - << ", edge: " << *edge); + //pdebug("add in edge to " << *this << ", base=" << base + // << ", edge: " << *edge); in_edges[twobit_repr(base)] = edge; } @@ -198,8 +198,8 @@ public: } void add_out_edge(const char base, CompactEdge* edge) { - pdebug("add out edge to " << *this << ", base=" << base - << ", edge: " << *edge); + //pdebug("add out edge to " << *this << ", base=" << base + // << ", edge: " << *edge); out_edges[twobit_repr(base)] = edge; } @@ -271,7 +271,7 @@ protected: // protected linear creation of CompactNode // they should never be deleted, so this is straightforward CompactNode* new_compact_node(Kmer hdn) { - pdebug("new compact node from " << hdn); + //pdebug("new compact node from " << hdn); CompactNode * v = get_compact_node_by_kmer(hdn); if (v == nullptr) { hdn.set_forward(); @@ -280,7 +280,7 @@ protected: v = &(compact_nodes.back()); v->sequence = _revhash(hdn, ksize()); // hopefully new k-mer type in future hdn_ids[hdn] = v->node_id; - pdebug("Allocate: " << *v); + //pdebug("Allocate: " << *v); } return v; } @@ -289,15 +289,15 @@ protected: compact_edge_meta_t edge_meta, std::string edge_sequence) { CompactEdge* edge = new CompactEdge(left, right, edge_meta); - pdebug("new compact edge: left=" << left << " right=" << right - << " sequence=" << edge_sequence); + //pdebug("new compact edge: left=" << left << " right=" << right + // << " sequence=" << edge_sequence); edge->sequence = edge_sequence; n_compact_edges++; return edge; } void delete_compact_edge(CompactEdge * edge) { - pdebug("attempt edge delete @" << edge); + //pdebug("attempt edge delete @" << edge); if (edge != nullptr) { pdebug("edge not null, proceeding"); for (auto tag: edge->tags) { @@ -378,7 +378,7 @@ public: } std::vector get_compact_nodes(const std::string& sequence) { - pdebug("get compact node IDs"); + //pdebug("get compact node IDs"); KmerIterator kmers(sequence.c_str(), graph->ksize()); std::vector nodes; @@ -398,7 +398,7 @@ public: } CompactEdge* get_compact_edge(HashIntoType tag) { - pdebug("get compact edge from tag " << tag); + //pdebug("get compact edge from tag " << tag); auto search = tags_to_edges.find(tag); if (search != tags_to_edges.end()) { return search->second; @@ -472,6 +472,7 @@ public: uint64_t update_compact_dbg(const std::string& sequence) { pdebug("update cDBG from " << sequence); + n_sequences_added++; // first gather up all k-mers that could have been disturbed -- // k-mers in the read, and the neighbors of the flanking nodes @@ -652,7 +653,9 @@ public: } - } + return n_updates; + + } // update_compact_dbg }; diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index a62723b13e..e8e2917116 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -139,7 +139,7 @@ cdef class StreamingCompactor: def update(self, str sequence): cdef string _sequence = _bstring(sequence) - deref(self._sc_this).update_compact_dbg(_sequence) + return deref(self._sc_this).update_compact_dbg(_sequence) def consume(self, str sequence): cdef string _sequence = _bstring(sequence) diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index d05ed41bde..00ea17a789 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -18,14 +18,8 @@ def teardown(): utils.cleanup() -def test_compact_fork(right_tip_structure): - '''Should have no links. Need two junctions. - ''' - graph, contig, L, HDN, R, tip = right_tip_structure - - compactor = StreamingCompactor(graph) - compactor.update(contig) - compactor.report() +def compare_right_tip_with_cdbg(rts, compactor): + graph, contig, L, HDN, R, tip = rts nodes = list(compactor.sequence_nodes(contig)) assert len(nodes) == 1 @@ -55,6 +49,18 @@ def test_compact_fork(right_tip_structure): assert _equals_rc(contig, edge_contig.sequence + node.sequence + out_edge.sequence) + +def test_compact_fork(right_tip_structure): + '''Should have no links. Need two junctions. + ''' + graph, contig, L, HDN, R, tip = right_tip_structure + + compactor = StreamingCompactor(graph) + print(compactor.update(contig), 'cDBG updates...') + compactor.report() + + compare_right_tip_with_cdbg(right_tip_structure, compactor) + for node in nodes: print(node) print('in edges:') @@ -73,26 +79,28 @@ def test_compact_fork(right_tip_structure): print("R FW:", R) print("R RC:", revcomp(R)) - #assert junction['u'] == forward_hash(L, K) - #assert junction['v'] == forward_hash(HDN, K) - #assert junction['w'] == forward_hash(R, K) - -''' -def test_links_bubble(snp_bubble_structure): - graph, wildtype_sequence, _, HDN_L, HDN_R = snp_bubble_structure - linker = GraphLinker(graph) - - # thread the wildtype half of the bubble - linker.add_links(wildtype_sequence) - linker.report() - - - links = list(linker.get_links(wildtype_sequence)) - assert len(links) == 2 - - link_a, link_b = links - if link_a[0]['v'] == forward_hash(HDN_L, K): - assert link_b[0]['v'] == forward_hash(HDN_R, K) - elif link_a[0]['v'] == forward_hash(HDN_R, K): - assert link_b[0]['v'] == forward_hash(HDN_L, K) -''' + +def test_compact_fork_double_update(right_tip_structure): + graph, contig, L, HDN, R, tip = right_tip_structure + + compactor = StreamingCompactor(graph) + print(compactor.update(contig), 'cDBG updates...') + compactor.report() + print(compactor.update(contig), 'cDBG updates...') + compactor.report() + + compare_right_tip_with_cdbg(right_tip_structure, compactor) + + +def test_compact_fork_revcomp_update(right_tip_structure): + graph, contig, L, HDN, R, tip = right_tip_structure + + compactor = StreamingCompactor(graph) + print(compactor.update(contig), 'cDBG updates...') + compactor.report() + + print(compactor.update(revcomp(contig)), 'cDBG updates...') + compactor.report() + + compare_right_tip_with_cdbg(right_tip_structure, compactor) + From 975744f51a3ce603dec85c828310926bdd9aac1a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 8 Nov 2017 14:56:48 -0800 Subject: [PATCH 148/185] WIP --- include/oxli/kmer_hash.hh | 4 + include/oxli/links.hh | 645 +++++++++++++++++++++++++++----------- tests/graph_features.py | 13 +- tests/test_linked_dbg.py | 61 +++- 4 files changed, 530 insertions(+), 193 deletions(-) diff --git a/include/oxli/kmer_hash.hh b/include/oxli/kmer_hash.hh index c18a722951..ad4a60403e 100644 --- a/include/oxli/kmer_hash.hh +++ b/include/oxli/kmer_hash.hh @@ -307,6 +307,10 @@ public: kmer_u = _hash(kmer_c, _ksize, kmer_f, kmer_r); return Kmer(kmer_f, kmer_r, kmer_u); } + + WordLength K() const { + return _ksize; + } }; /** diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 5a6cd88a6f..eb7c1b7ef7 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -64,6 +64,10 @@ Contact: khmer-project@idyll.org # define pdebug(x) do {} while (0) # endif +#define complement(ch) ((ch) == 'A' ? 'T' : \ + (ch) == 'T' ? 'A' : \ + (ch) == 'C' ? 'G' : 'C') + namespace oxli { using std::make_shared; @@ -77,8 +81,7 @@ typedef std::unordered_map HashIDMap; enum compact_edge_meta_t { IS_FULL_EDGE, - IS_IN_TIP, - IS_OUT_TIP, + IS_TIP, IS_ISLAND, IS_TRIVIAL }; @@ -88,10 +91,8 @@ inline const char * edge_meta_repr(compact_edge_meta_t meta) { switch(meta) { case IS_FULL_EDGE: return "FULL EDGE"; - case IS_IN_TIP: - return "IN TIP"; - case IS_OUT_TIP: - return "OUT TIP"; + case IS_TIP: + return "TIP"; case IS_ISLAND: return "ISLAND"; case IS_TRIVIAL: @@ -100,19 +101,24 @@ inline const char * edge_meta_repr(compact_edge_meta_t meta) { } +class CompactEdgeFactory; class CompactEdge { + friend class CompactEdgeFactory; +protected: + uint64_t in_node_id; // left and right HDN IDs + uint64_t out_node_id; + public: - Kmer in_node; // left and right HDNs - Kmer out_node; + UHashSet tags; compact_edge_meta_t meta; std::string sequence; - CompactEdge(Kmer in_node, Kmer out_node) : - in_node(in_node), out_node(out_node), meta(IS_FULL_EDGE) {} + CompactEdge(uint64_t in_node_id, uint64_t out_node_id) : + in_node_id(in_node_id), out_node_id(out_node_id), meta(IS_FULL_EDGE) {} - CompactEdge(Kmer in_node, Kmer out_node, compact_edge_meta_t meta) : - in_node(in_node), out_node(out_node), meta(meta) {} + CompactEdge(uint64_t in_node_id, uint64_t out_node_id, compact_edge_meta_t meta) : + in_node_id(in_node_id), out_node_id(out_node_id), meta(meta) {} void add_tags(UHashSet& new_tags) { for (auto tag: new_tags) { @@ -120,6 +126,26 @@ public: } } + Kmer get_fw_in() const { + return in_node_id; + } + + Kmer get_rc_in() const { + return out_node_id; + } + + Kmer get_fw_out() const { + return out_node_id; + } + + Kmer get_rc_out() const { + return in_node_id; + } + + std::string rc_sequence() const { + return _revcomp(sequence); + } + float tag_density() const { return (float)sequence.length() / (float)tags.size(); } @@ -140,7 +166,7 @@ public: friend std::ostream& operator<<(std::ostream& stream, const CompactEdge& edge) { - stream << ""; return stream; @@ -148,9 +174,125 @@ public: }; +typedef std::vector CompactEdgeVector; + +class CompactEdgeFactory : public KmerFactory { + +protected: + + uint64_t n_compact_edges; + uint32_t tag_density; + + TagEdgeMap tags_to_edges; + CompactEdgeVector compact_edges; + +public: + + CompactEdgeFacory(Wordlength K) : + KmerFactory(K), n_compact_edges(0) { + + tag_density = DEFAULT_TAG_DENSITY; + } + + uint64_t n_edges() const { + return n_compact_edges; + } + + CompactEdge* build_edge(uint64_t left_id, uint64_t right_id, + compact_edge_meta_t edge_meta, + std::string edge_sequence) { + CompactEdge* edge = new CompactEdge(left_id, right_id, edge_meta); + pdebug("new compact edge: left=" << left << " right=" << right + << " sequence=" << edge_sequence); + edge->sequence = edge_sequence; + n_compact_edges++; + return edge; + } + + void delete_edge(CompactEdge * edge) { + //pdebug("attempt edge delete @" << edge); + if (edge != nullptr) { + pdebug("edge not null, proceeding"); + for (auto tag: edge->tags) { + tags_to_edges.erase(tag); + } + delete edge; + n_compact_edges--; + } + } + + void delete_edge(UHashSet& tags) { + CompactEdge* edge = get_edge(tags); + delete_compact_edge(edge); + } + + void delete_edge(HashIntoType tag) { + CompactEdge* edge = get_edge(tag); + delete_compact_edge(edge); + } + + CompactEdge* get_edge(HashIntoType tag) { + //pdebug("get compact edge from tag " << tag); + auto search = tags_to_edges.find(tag); + if (search != tags_to_edges.end()) { + return search->second; + } + return nullptr; + } + + bool get_tag_edge_pair(HashIntoType tag, TagEdgePair& pair) { + auto search = tags_to_edges.find(tag); + if (search != tags_to_edges.end()) { + pair = *search; + return true; + } else { + return false; + } + } + + CompactEdge* get_edge(UHashSet& tags) { + CompactEdge * edge = nullptr; + for (auto tag: tags) { + edge = get_edge(tag); + if (edge != nullptr) { + break; + } + } + return edge; + } + + KmerFilter get_tag_stopper(TagEdgePair& te_pair, + bool& found_tag) { + KmerFilter stopper = [&] (const Kmer& node) { + found_tag = get_tag_edge_pair(node, te_pair); + return found_tag; + }; + + return stopper; + } + + compact_edge_meta_t deduce_meta(CompactNode* in, CompactNode* out) { + compact_edge_meta_t edge_meta; + if (in == nullptr && out == nullptr) { + edge_meta = IS_ISLAND; + } else if ((out == nullptr) != (in == nullptr)) { + edge_meta = IS_TIP; + } else { + edge_meta = IS_FULL_EDGE; + } + return edge_meta; + } + + +}; + + +class CompactNodeFactory; class CompactNode { + friend class CompactNodeFactory; public: Kmer kmer; + bool direction; uint32_t count; std::string sequence; const uint64_t node_id; @@ -158,15 +300,26 @@ public: CompactEdge* out_edges[4] = {nullptr, nullptr, nullptr, nullptr}; CompactNode(Kmer kmer, uint64_t node_id) : - kmer(kmer), count(0), node_id(node_id) {} + kmer(kmer), count(0), node_id(node_id), direction(kmer.is_forward()) {} CompactNode(Kmer kmer, std::string sequence, uint64_t node_id) : - kmer(kmer), count(0), sequence(sequence), node_id(node_id) {} + kmer(kmer), count(0), sequence(sequence), node_id(node_id), + direction(kmer.is_forward()) {} friend bool operator== (const CompactNode& lhs, const CompactNode& rhs) { return lhs.node_id == rhs.node_id; } + bool delete_edge(CompactEdge* edge) { + if (delete_in_edge(edge)) { + return true; + } + if (delete_out_edge(edge)) { + return true; + } + return false; + } + bool delete_in_edge(CompactEdge* edge) { for (uint8_t i=0; i<4; i++) { if (in_edges[i] == edge) { @@ -241,145 +394,73 @@ public: } }; - typedef std::vector CompactNodeVector; -typedef std::vector CompactEdgeVector; -typedef std::unordered_map TagEdgeMap; -typedef std::pair TagEdgePair; -typedef std::set TagEdgePairSet; -typedef std::set CompactEdgeSet; - -class StreamingCompactor -{ - friend class CompactEdge; - friend class CompactNode; +class CompactNodeFactory : public KmerFactory { protected: + // map from HDN hashes to CompactNode IDs - HashIDMap hdn_ids; + HashIDMap kmer_id_map; // linear storage for CompactNodes CompactNodeVector compact_nodes; - // map from tags to CompactEdges - TagEdgeMap tags_to_edges; - - uint64_t n_sequences_added; - uint64_t n_compact_edges; uint64_t n_compact_nodes; - uint32_t tag_density; + +public: + CompactNodeFactory(WordLength K) : + KmerFactory(K), n_compact_nodes(0) {} + + uint64_t n_nodes() const { + return n_compact_nodes; + } + // protected linear creation of CompactNode // they should never be deleted, so this is straightforward - CompactNode* new_compact_node(Kmer hdn) { + CompactNode* build_node(Kmer hdn) { //pdebug("new compact node from " << hdn); CompactNode * v = get_compact_node_by_kmer(hdn); if (v == nullptr) { - hdn.set_forward(); compact_nodes.emplace_back(hdn, n_compact_nodes); n_compact_nodes++; v = &(compact_nodes.back()); - v->sequence = _revhash(hdn, ksize()); // hopefully new k-mer type in future - hdn_ids[hdn] = v->node_id; + v->sequence = _revhash(hdn, ksize()); + kmer_id_map[hdn] = v->node_id; //pdebug("Allocate: " << *v); } return v; } - CompactEdge* new_compact_edge(Kmer left, Kmer right, - compact_edge_meta_t edge_meta, - std::string edge_sequence) { - CompactEdge* edge = new CompactEdge(left, right, edge_meta); - //pdebug("new compact edge: left=" << left << " right=" << right - // << " sequence=" << edge_sequence); - edge->sequence = edge_sequence; - n_compact_edges++; - return edge; - } - - void delete_compact_edge(CompactEdge * edge) { - //pdebug("attempt edge delete @" << edge); - if (edge != nullptr) { - pdebug("edge not null, proceeding"); - for (auto tag: edge->tags) { - tags_to_edges.erase(tag); - } - delete edge; - n_compact_edges--; - } - } - - void delete_compact_edge(UHashSet& tags) { - CompactEdge* edge = get_compact_edge(tags); - delete_compact_edge(edge); - } - - void delete_compact_edge(HashIntoType tag) { - CompactEdge* edge = get_compact_edge(tag); - delete_compact_edge(edge); - } - -public: - - shared_ptr graph; - - StreamingCompactor(shared_ptr graph) : - graph(graph), n_sequences_added(0), - n_compact_edges(0), n_compact_nodes(0) - { - tag_density = DEFAULT_TAG_DENSITY; - - } - - WordLength ksize() const { - return graph->ksize(); - } - - uint64_t n_nodes() const { - return compact_nodes.size(); - } - - uint64_t n_edges() const { - return n_compact_edges; - } - - void report() const { - std::cout << "StreamingCompactor(@" << this << " with " - << "Hashgraph @" << graph.get() << ")" << std::endl; - std::cout << " * " << hdn_ids.size() << " cDBG nodes (HDNs)" << std::endl; - std::cout << " * " << n_compact_edges << " cDBG edges" << std::endl; - std::cout << " * " << n_sequences_added << " sequences added" << std::endl; - } - - CompactNode* get_compact_node_by_kmer(HashIntoType hdn) { + CompactNode* get_node_by_kmer(HashIntoType hdn) { auto search = hdn_ids.find(hdn); - if (search != hdn_ids.end()) { + if (search != kmer_id_map.end()) { uint64_t ID = search->second; return &(compact_nodes[ID]); } return nullptr; } - CompactNode* get_compact_node_by_id(uint64_t id) { - if (id < compact_nodes.size()) { + CompactNode* get_node_by_id(uint64_t id) { + if (id >= compact_nodes.size()) { return nullptr; } return &(compact_nodes[id]); } - CompactNode* fetch_or_new_compact_node(Kmer hdn) { - CompactNode* v = get_compact_node_by_kmer(hdn); + CompactNode* get_or_build_node(Kmer hdn) { + CompactNode* v = get_node_by_kmer(hdn); if (v != nullptr) { v->count += 1; } else { - v = new_compact_node(hdn); + v = build_node(hdn); v->count = 1; } return v; } - std::vector get_compact_nodes(const std::string& sequence) { + std::vector get_nodes(const std::string& sequence) { //pdebug("get compact node IDs"); - KmerIterator kmers(sequence.c_str(), graph->ksize()); + KmerIterator kmers(sequence.c_str(), _ksize); std::vector nodes; CompactNode* node; @@ -387,9 +468,8 @@ public: while(!kmers.done()) { Kmer kmer = kmers.next(); - node = get_compact_node_by_kmer(kmer); + node = get_node_by_kmer(kmer); if (node != nullptr) { - pdebug("found compact node " << *node); nodes.push_back(node); } } @@ -397,58 +477,139 @@ public: return nodes; } - CompactEdge* get_compact_edge(HashIntoType tag) { - //pdebug("get compact edge from tag " << tag); - auto search = tags_to_edges.find(tag); - if (search != tags_to_edges.end()) { - return search->second; + bool get_pivot_from_left(CompactNode* v, + std::string& segment, + char& pivot_base) const { + const char * node_kmer = v->sequence.c_str(); + const char * _segment = sequence.c_str(); + pivot_base = _segment[segment.size()-_ksize]; + if (strncmp(node_kmer, + _segment+(segment.size())-_ksize+1, + _ksize-1) == 0) { + // same canonical orientation + return false; + } else { + // must have opposite canonical orientation + pivot_base = complement(pivot_base); + return true; } - return nullptr; } - bool get_tag_edge_pair(HashIntoType tag, TagEdgePair& pair) { - auto search = tags_to_edges.find(tag); - if (search != tags_to_edges.end()) { - pair = *search; - return true; - } else { + bool add_edge_from_left(CompactNode* v, CompactEdge* e) const { + char pivot_base; + if (!get_pivot_from_left(v, e->sequence, pivot_base)) { + // same canonical orientation + v->add_in_edge(pivot_base, e); return false; + } else { + // must have opposite canonical orientation + v->add_out_edge(pivot_base, e); + return true; } } - CompactEdge* get_compact_edge(UHashSet& tags) { - CompactEdge * edge = nullptr; - for (auto tag: tags) { - edge = get_compact_edge(tag); - if (edge != nullptr) { - break; - } + + bool get_edge_from_left(CompactNode* v, + CompactEdge* &result_edge, + std::string& segment) const { + char pivot_base; + if (!get_pivot_from_left(v, segment, pivot_base)) { + result_edge = v->get_in_edge(pivot_base); + return false; + } else { + result_edge = v->get_out_edge(pivot_base); + return true; } - return edge; } - KmerFilter get_tag_stopper(TagEdgePair& te_pair, - bool& found_tag) { - KmerFilter stopper = [&] (const Kmer& node) { - found_tag = get_tag_edge_pair(node, te_pair); - return found_tag; - }; + bool get_pivot_from_right(CompactNode* v, + std::string& segment, + char& pivot_base) const { + const char * node_kmer = v->sequence.c_str(); + const char * _segment = sequence.c_str(); + pivot_base = _segment[_ksize-1]; + if (strncmp(node_kmer+1, _segment, _ksize-1)) { + // same canonical orientation + return false; + } else { + // must have opposite canonical orientation + pivot_base = complement(pivot_base); + return true; + } + } - return stopper; + bool add_edge_from_right(CompactNode* v, CompactEdge* e) const { + char pivot_base; + if (!get_pivot_from_right(v, e->sequence, pivot_base)) { + v->add_out_edge(pivot_base, e); + return false; + } else { + v->add_in_edge(pivot_base, e); + return true; + } } - compact_edge_meta_t deduce_edge_meta(CompactNode* in, CompactNode* out) { - compact_edge_meta_t edge_meta; - if (in == nullptr && out == nullptr) { - edge_meta = IS_ISLAND; - } else if (out == nullptr) { - edge_meta = IS_OUT_TIP; - } else if (in == nullptr) { - edge_meta = IS_IN_TIP; + bool get_edge_from_right(CompactNode* v, + CompactEdge* &result_edge, + std::string& segment) const { + char pivot_base; + if (!get_pivot_base_from_right(v, segment, pivot_base)) { + result_edge = get_out_edge(pivot_base); + return false; } else { - edge_meta = IS_FULL_EDGE; + result_edge = get_in_edge(pivot_base, e); + return true; } - return edge_meta; + + } +}; + + +typedef std::unordered_map TagEdgeMap; +typedef std::pair TagEdgePair; +typedef std::set TagEdgePairSet; +typedef std::set CompactEdgeSet; + + +class StreamingCompactor +{ + +protected: + + // map from tags to CompactEdges + CompactNodeFactory nodes; + CompactEdgeFactory edges; + + uint64_t n_sequences_added; + +public: + + shared_ptr graph; + + StreamingCompactor(shared_ptr graph) : + graph(graph), n_sequences_added(0), + nodes(graph->ksize()), edges(graph->ksize()) + { + } + + WordLength ksize() const { + return graph->ksize(); + } + + uint64_t n_nodes() const { + return nodes.n_nodes(); + } + + uint64_t n_edges() const { + return edges.n_edges(); + } + + void report() const { + std::cout << std::endl << "REPORT: StreamingCompactor(@" << this << " with " + << "Hashgraph @" << graph.get() << ")" << std::endl; + std::cout << " * " << n_nodes() << " cDBG nodes (HDNs)" << std::endl; + std::cout << " * " << n_edges() << " cDBG edges" << std::endl; + std::cout << " * " << n_sequences_added << " sequences added" << std::endl; } uint64_t consume_sequence(const std::string& sequence) { @@ -464,10 +625,44 @@ public: return 0; } + bool validate_segment(CompactNode* left_node, CompactNode* right_node, + CompactEdge* edge, std::string& segment) { + + } + /* Update a compact dbg where there are no induced * HDNs */ - void update_compact_dbg_linear(std::string& sequence) { + uint64_t update_compact_dbg_linear(std::string& sequence) { + Kmer root_kmer = graph->build_kmer(sequence.substr(0, ksize())); + + CompactingAT lcursor(graph.get(), root_kmer); + CompactingAT rcursor(graph.get(), root_kmer); + CompactingAssembler cassem(graph.get()); + + std::string left_seq = cassem._assemble_directed(lcursor); + std::string right_seq = cassem._assemble_directed(rcursor); + std::string segment_seq = left_seq + right_seq.substr(ksize()); + + CompactNode *left_node = nullptr, *right_node = nullptr; + left_node = nodes.get_node_by_kmer(lcursor.cursor); + right_node = nodes.get_node_by_kmer(rcursor.cursor); + compact_edge_meta_t edge_meta = deduce_edge_meta(left_node, right_node); + char in_base = segment_seq[ksize()-1]; + char out_base = segment_seq[segment_seq.length()-ksize()+1]; + + switch(edge_meta) { + case IS_FULL_EDGE: + // then we merged two tips + pdebug("merge TIPs"); + break; + case IS_TIP: + pdebug("extend TIP"); + break; + case IS_ISLAND: + pdebug("created or extended ISLAND"); + break; + } } uint64_t update_compact_dbg(const std::string& sequence) { @@ -479,12 +674,10 @@ public: KmerIterator kmers(sequence.c_str(), ksize()); KmerQueue disturbed_kmers; Kmer kmer = kmers.next(); - kmer.set_forward(); CompactingAT lcursor(graph.get(), kmer); lcursor.neighbors(disturbed_kmers); while(!kmers.done()) { kmer = kmers.next(); - kmer.set_forward(); disturbed_kmers.push_back(kmer); } CompactingAT rcursor(graph.get(), kmer); @@ -496,8 +689,15 @@ public: while(!disturbed_kmers.empty()) { Kmer kmer = disturbed_kmers.back(); disturbed_kmers.pop_back(); - if(lcursor.degree(kmer) > 1 || rcursor.degree(kmer) > 1) { - if (fetch_or_new_compact_node(kmer)->count == 1) { + uint8_t l_degree, r_degree; + l_degree = lcursor.degree(kmer); + r_degree = rcursor.degree(kmer); + if(l_degree > 1 || r_degree > 1) { + CompactNode* hdn = nodes.get_or_build_node(kmer); + if (hdn->count == 1) { + induced_hdns.insert(kmer); + n_updates++; + } else if (hdn->degree() != (l_degree + r_degree)) { induced_hdns.insert(kmer); n_updates++; } @@ -518,7 +718,6 @@ public: while(!induced_hdns.empty()) { Kmer root_kmer = *induced_hdns.begin(); induced_hdns.erase(root_kmer); - root_kmer.set_forward(); CompactNode* root_node = get_compact_node_by_kmer(root_kmer); pdebug("searching from induced HDN: " << *root_node); @@ -531,53 +730,123 @@ public: neighbors.pop_back(); lcursor.cursor = neighbor; - TagEdgePair left_tag_pair; - bool found_left_tag; + TagEdgePair tag_pair; + bool found_tag = false; - lcursor.push_filter(get_tag_stopper(left_tag_pair, found_left_tag)); + lcursor.push_filter(get_tag_stopper(tag_pair, found_tag)); std::string segment_seq = cassem._assemble_directed(lcursor); + // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; - CompactNode* left_node = nullptr; + bool edge_invalid = false; + bool root_flipped = nodes.get_edge_from_left(root_node, segment_edge, segment_seq); + + CompactNode* left_node = nodes.get_node_by_kmer(lcursor.cursor); + + // validate edge leaving root if it exists + if (segment_edge != nullptr) { + if (segment_edge->meta == IS_TIP) { + if (left_node != nullptr) { + edge_invalid = true; + } + if (!((segment_edge->in_node_id == root_node->node_id || + segment_edge->out_node_id == root_node->node_id) && + segment_edge->sequence.length() == segment_seq.length())) { + edge_invalid = true; + } + } else if (segment_edge->meta == IS_FULL_EDGE) { + if (left_node == nullptr) { + edge_invalid = true; + } else { + bool nodes_match; + nodes_match = (segment_edge->in_node_id == root_node->node_id && + segment_edge->out_node_id == left_node->node_id) || + (segment_edge->out_node_id == root_node->node_id && + segment_edge->in_node_id == left_node->node_id); + edge_invalid = !nodes_match; + } + } + } + if (!edge_invalid) { + continue; + } + + + + /* * Should also keep a set of pair to track resolved * segments */ - if (found_left_tag) { - // must be an existing segment - segment_edge = get_compact_edge(left_tag_pair.first); - left_node = get_compact_node_by_kmer(segment_edge->in_node); - // check if we need to split the edge - if (*get_compact_node_by_kmer(segment_edge->out_node) == *root_node) { - // good, we're set + + /* + if (found_tag) { + if (segment_edge != nullptr) { + if (segment_edge + } + if (segment_edge == get_compact_edge(tag_pair.first)) { + pdebug("found tag to existing segment on left"); + left_node = nodes.get_node_by_id(segment_edge->in_node_id); + } else { + pdebug("tags to left don't match existing segment....?"); + } + } else { + */ + //pdebug("no tag to existing segment found..."); + // then left node must have been new or does not exist, + // or the segment was too short to be tagged + + //} + + // if there's an existing edge, check if we need to split it + if (segment_edge != nullptr) { + pdebug("checking existing edge for comformity"); + CompactNode* existing_out_node; + if (left_flipped) { + existing_out_node = nodes.get_node_by_id(segment_edge->in_node_id); + } else { + existing_out_node = nodes.get_node_by_id(segment_edge->out_node_id); + } + if (found_out_node != nullptr && *found_out_node == *root_node) { + // this edge is fine + pdebug("edges conforms, moving to next neighbor"); continue; // continue through neighbors } else { // need to be careful here, the out node for the // edge we delete could be linked to another induced // HDN... n_updates++; - get_compact_node_by_kmer(segment_edge->out_node)->delete_in_edge(segment_edge); + // check that it isn't a TIP from an existing node + // with ROOT being induced; if not, delete its edge + // from the out node + pdebug("edge does not conform, delete it"); + if (found_out_node != nullptr) { + pdebug("existing edge out node not root, deleting"); + // be lazy for now and use bidirectional delete + existing_out_node->delete_edge(segment_edge); + } delete_compact_edge(segment_edge); // deleted tags; assemble out to the HDN segment_seq = cassem._assemble_directed(lcursor) + segment_seq.substr(ksize()); if (left_node != nullptr) { // not an IN_TIP - left_node->delete_out_edge(segment_edge); + left_node->delete_edge(segment_edge); } } - } else { - // then left node must have been new, or does not exist - left_node = get_compact_node_by_kmer(lcursor.cursor); } // construct the compact edge compact_edge_meta_t edge_meta = (left_node == nullptr) ? - IS_IN_TIP : IS_FULL_EDGE; + IS_TIP : IS_FULL_EDGE; + if (edge_meta == IS_FULL_EDGE) { - segment_seq = segment_seq.substr(ksize()-1, + // left side includes HDN, right side does not + segment_seq = segment_seq.substr(ksize(), segment_seq.length()-ksize()+1); } else { + // unless there is none, in which case + // we take the whole contig segment_seq = segment_seq.substr(0, segment_seq.length()-ksize()+1); } @@ -600,9 +869,12 @@ public: Kmer neighbor = neighbors.back(); neighbors.pop_back(); rcursor.cursor = neighbor; + //if (rcursor.cursor.is_forward() != root_node->direction) { + // rcursor.cursor.set_forward(); + //} TagEdgePair right_tag_pair; - bool found_right_tag; + bool found_right_tag = false; rcursor.push_filter(get_tag_stopper(right_tag_pair, found_right_tag)); std::string segment_seq = cassem._assemble_directed(rcursor); @@ -612,12 +884,25 @@ public: if (found_right_tag) { segment_edge = get_compact_edge(right_tag_pair.first); - right_node = get_compact_node_by_kmer(segment_edge->out_node); - if (*get_compact_node_by_kmer(segment_edge->in_node) == *root_node) { + right_node = get_compact_node_by_kmer(segment_edge->out_node_id); + } else { + right_node = get_compact_node_by_kmer(rcursor.cursor); + if (right_node != nullptr) { + // base for hypothetical in-edge + auto base = segment_seq[segment_seq.length()-ksize()]; + segment_edge = right_node->get_in_edge(base); + } + } + + if (segment_edge != nullptr) { + CompactNode* found_in_node = get_compact_node_by_kmer(segment_edge->in_node_id); + if (found_in_node != nullptr && *found_in_node == *root_node) { continue; } else { n_updates++; - get_compact_node_by_kmer(segment_edge->in_node)->delete_out_edge(segment_edge); + if (found_in_node != nullptr) { + found_in_node->delete_out_edge(segment_edge); + } delete_compact_edge(segment_edge); segment_seq = segment_seq + cassem._assemble_directed(rcursor).substr(ksize()); @@ -625,15 +910,13 @@ public: right_node->delete_in_edge(segment_edge); } } - } else { - right_node = get_compact_node_by_kmer(rcursor.cursor); } compact_edge_meta_t edge_meta = (right_node == nullptr) ? IS_OUT_TIP : IS_FULL_EDGE; if (edge_meta == IS_FULL_EDGE) { segment_seq = segment_seq.substr(ksize()-1, - segment_seq.length()-ksize()+1); + segment_seq.length()-ksize()); } else { segment_seq = segment_seq.substr(ksize()-1); } diff --git a/tests/graph_features.py b/tests/graph_features.py index 3900d9228a..a616003ed0 100644 --- a/tests/graph_features.py +++ b/tests/graph_features.py @@ -93,7 +93,7 @@ def mutate_position(sequence, pos): return ''.join(sequence) -def get_random_sequence(length, exclude=None): +def get_random_sequence(length, exclude=None, seen=None): '''Generate a random (non-looping) nucleotide sequence. To be non-overlapping, the sequence should not include any repeated @@ -107,7 +107,7 @@ def get_random_sequence(length, exclude=None): str: A random non-looping sequence. ''' - seen = set() + seen = set() if seen is None else seen.copy() def add_seen(kmer): seen.add(kmer) @@ -207,9 +207,16 @@ def known_sequence(request): @pytest.fixture(params=list(range(500, 1600, 500)), ids=lambda val: '(L={0})'.format(val)) def random_sequence(request): + global_seen = set() def get(exclude=None): - return get_random_sequence(request.param, exclude=exclude) + sequence = get_random_sequence(request.param, + exclude=exclude, + seen=global_seen) + for i in range(len(sequence)-K): + global_seen.add(sequence[i:i+K-1]) + global_seen.add(revcomp(sequence[i:i+K-1])) + return sequence return get diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 00ea17a789..9414236e87 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -18,13 +18,11 @@ def teardown(): utils.cleanup() -def compare_right_tip_with_cdbg(rts, compactor): +def compare_tip_with_cdbg(rts, compactor): graph, contig, L, HDN, R, tip = rts nodes = list(compactor.sequence_nodes(contig)) assert len(nodes) == 1 - assert compactor.n_nodes == 1 - assert compactor.n_edges == 3 node = nodes[0] assert _equals_rc(node.sequence, HDN) @@ -50,7 +48,7 @@ def compare_right_tip_with_cdbg(rts, compactor): out_edge.sequence) -def test_compact_fork(right_tip_structure): +def test_compact_tip(right_tip_structure): '''Should have no links. Need two junctions. ''' graph, contig, L, HDN, R, tip = right_tip_structure @@ -59,7 +57,10 @@ def test_compact_fork(right_tip_structure): print(compactor.update(contig), 'cDBG updates...') compactor.report() - compare_right_tip_with_cdbg(right_tip_structure, compactor) + compare_tip_with_cdbg(right_tip_structure, compactor) + + assert compactor.n_nodes == 1 + assert compactor.n_edges == 3 for node in nodes: print(node) @@ -80,7 +81,7 @@ def test_compact_fork(right_tip_structure): print("R RC:", revcomp(R)) -def test_compact_fork_double_update(right_tip_structure): +def test_compact_tip_double_update(right_tip_structure): graph, contig, L, HDN, R, tip = right_tip_structure compactor = StreamingCompactor(graph) @@ -89,10 +90,12 @@ def test_compact_fork_double_update(right_tip_structure): print(compactor.update(contig), 'cDBG updates...') compactor.report() - compare_right_tip_with_cdbg(right_tip_structure, compactor) + compare_tip_with_cdbg(right_tip_structure, compactor) + assert compactor.n_nodes == 1 + assert compactor.n_edges == 3 -def test_compact_fork_revcomp_update(right_tip_structure): +def test_compact_tip_revcomp_update(right_tip_structure): graph, contig, L, HDN, R, tip = right_tip_structure compactor = StreamingCompactor(graph) @@ -102,5 +105,45 @@ def test_compact_fork_revcomp_update(right_tip_structure): print(compactor.update(revcomp(contig)), 'cDBG updates...') compactor.report() - compare_right_tip_with_cdbg(right_tip_structure, compactor) + compare_tip_with_cdbg(right_tip_structure, compactor) + assert compactor.n_nodes == 1 + assert compactor.n_edges == 3 + +def test_compact_two_tip_islands(left_tip_structure, right_tip_structure): + graph, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure + _, contig_l, L_l, HDN_l, R_l, tip_l = left_tip_structure + + compactor = StreamingCompactor(graph) + print(compactor.update(contig_l), 'cDBG updates from left') + compactor.report() + compare_tip_with_cdbg(left_tip_structure, compactor) + assert compactor.n_nodes == 1 + assert compactor.n_edges == 3 + + print(compactor.update(contig_r), 'cDBG updates from right') + compactor.report() + compare_tip_with_cdbg(right_tip_structure, compactor) + assert compactor.n_nodes == 2 + assert compactor.n_edges == 6 + + +def test_compact_tip_x_merge(left_tip_structure, right_tip_structure): + graph, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure + _, contig_l, L_l, HDN_l, R_l, tip_l = left_tip_structure + + contig_merge = contig_l + contig_r + + compactor = StreamingCompactor(graph) + print(compactor.update(contig_l), 'cDBG updates from left') + compactor.report() + compare_tip_with_cdbg(left_tip_structure, compactor) + assert compactor.n_nodes == 1 + assert compactor.n_edges == 3 + + compactor.consume(contig_merge) + print(compactor.update(contig_r), 'cDBG updates from right merge') + compactor.report() + #compare_tip_with_cdbg(right_tip_structure, compactor) + assert compactor.n_nodes == 2 + assert compactor.n_edges == 5 From 0eb855118de8cbbe6b1b508074ef57a5fc8dca2f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 9 Nov 2017 11:58:53 -0800 Subject: [PATCH 149/185] wip 2, compiling directionally aware --- include/oxli/links.hh | 425 +++++++++++++++++-------------------- khmer/_oxli/graphlinks.pxd | 98 ++++++--- khmer/_oxli/graphlinks.pyx | 24 +-- 3 files changed, 275 insertions(+), 272 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index eb7c1b7ef7..241e4c0e0e 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -38,8 +38,10 @@ Contact: khmer-project@idyll.org #define LINKS_HH #include +#include #include #include +#include #include #include #include @@ -70,6 +72,8 @@ Contact: khmer-project@idyll.org namespace oxli { +#define NULL_ID ULLONG_MAX + using std::make_shared; using std::shared_ptr; @@ -104,15 +108,14 @@ inline const char * edge_meta_repr(compact_edge_meta_t meta) { class CompactEdgeFactory; class CompactEdge { friend class CompactEdgeFactory; -protected: - uint64_t in_node_id; // left and right HDN IDs - uint64_t out_node_id; public: - UHashSet tags; + const uint64_t in_node_id; // left and right HDN IDs + const uint64_t out_node_id; compact_edge_meta_t meta; std::string sequence; + UHashSet tags; CompactEdge(uint64_t in_node_id, uint64_t out_node_id) : in_node_id(in_node_id), out_node_id(out_node_id), meta(IS_FULL_EDGE) {} @@ -126,22 +129,6 @@ public: } } - Kmer get_fw_in() const { - return in_node_id; - } - - Kmer get_rc_in() const { - return out_node_id; - } - - Kmer get_fw_out() const { - return out_node_id; - } - - Kmer get_rc_out() const { - return in_node_id; - } - std::string rc_sequence() const { return _revcomp(sequence); } @@ -166,7 +153,10 @@ public: friend std::ostream& operator<<(std::ostream& stream, const CompactEdge& edge) { - stream << ""; return stream; @@ -175,6 +165,10 @@ public: }; typedef std::vector CompactEdgeVector; +typedef std::unordered_map TagEdgeMap; +typedef std::pair TagEdgePair; +typedef std::set TagEdgePairSet; +typedef std::set CompactEdgeSet; class CompactEdgeFactory : public KmerFactory { @@ -188,7 +182,7 @@ protected: public: - CompactEdgeFacory(Wordlength K) : + CompactEdgeFactory(WordLength K) : KmerFactory(K), n_compact_edges(0) { tag_density = DEFAULT_TAG_DENSITY; @@ -202,7 +196,9 @@ public: compact_edge_meta_t edge_meta, std::string edge_sequence) { CompactEdge* edge = new CompactEdge(left_id, right_id, edge_meta); - pdebug("new compact edge: left=" << left << " right=" << right + pdebug("new compact edge: left=" << + std::to_string(left_id) << " right=" << + std::to_string(right_id) << " sequence=" << edge_sequence); edge->sequence = edge_sequence; n_compact_edges++; @@ -223,15 +219,15 @@ public: void delete_edge(UHashSet& tags) { CompactEdge* edge = get_edge(tags); - delete_compact_edge(edge); + delete_edge(edge); } void delete_edge(HashIntoType tag) { CompactEdge* edge = get_edge(tag); - delete_compact_edge(edge); + delete_edge(edge); } - CompactEdge* get_edge(HashIntoType tag) { + CompactEdge* get_edge(HashIntoType tag) const { //pdebug("get compact edge from tag " << tag); auto search = tags_to_edges.find(tag); if (search != tags_to_edges.end()) { @@ -240,7 +236,7 @@ public: return nullptr; } - bool get_tag_edge_pair(HashIntoType tag, TagEdgePair& pair) { + bool get_tag_edge_pair(HashIntoType tag, TagEdgePair& pair) const { auto search = tags_to_edges.find(tag); if (search != tags_to_edges.end()) { pair = *search; @@ -250,7 +246,7 @@ public: } } - CompactEdge* get_edge(UHashSet& tags) { + CompactEdge* get_edge(UHashSet& tags) const { CompactEdge * edge = nullptr; for (auto tag: tags) { edge = get_edge(tag); @@ -262,7 +258,7 @@ public: } KmerFilter get_tag_stopper(TagEdgePair& te_pair, - bool& found_tag) { + bool& found_tag) { const KmerFilter stopper = [&] (const Kmer& node) { found_tag = get_tag_edge_pair(node, te_pair); return found_tag; @@ -270,20 +266,6 @@ public: return stopper; } - - compact_edge_meta_t deduce_meta(CompactNode* in, CompactNode* out) { - compact_edge_meta_t edge_meta; - if (in == nullptr && out == nullptr) { - edge_meta = IS_ISLAND; - } else if ((out == nullptr) != (in == nullptr)) { - edge_meta = IS_TIP; - } else { - edge_meta = IS_FULL_EDGE; - } - return edge_meta; - } - - }; @@ -292,10 +274,11 @@ class CompactNode { friend class CompactNodeFactory; public: Kmer kmer; - bool direction; uint32_t count; - std::string sequence; const uint64_t node_id; + std::string sequence; + bool direction; + CompactEdge* in_edges[4] = {nullptr, nullptr, nullptr, nullptr}; CompactEdge* out_edges[4] = {nullptr, nullptr, nullptr, nullptr}; @@ -419,20 +402,20 @@ public: // they should never be deleted, so this is straightforward CompactNode* build_node(Kmer hdn) { //pdebug("new compact node from " << hdn); - CompactNode * v = get_compact_node_by_kmer(hdn); + CompactNode * v = get_node_by_kmer(hdn); if (v == nullptr) { compact_nodes.emplace_back(hdn, n_compact_nodes); n_compact_nodes++; v = &(compact_nodes.back()); - v->sequence = _revhash(hdn, ksize()); + v->sequence = _revhash(hdn, _ksize); kmer_id_map[hdn] = v->node_id; //pdebug("Allocate: " << *v); } return v; } - CompactNode* get_node_by_kmer(HashIntoType hdn) { - auto search = hdn_ids.find(hdn); + CompactNode* get_node_by_kmer(HashIntoType hdn) { + auto search = kmer_id_map.find(hdn); if (search != kmer_id_map.end()) { uint64_t ID = search->second; return &(compact_nodes[ID]); @@ -458,7 +441,7 @@ public: return v; } - std::vector get_nodes(const std::string& sequence) { + std::vector get_nodes(const std::string& sequence) { //pdebug("get compact node IDs"); KmerIterator kmers(sequence.c_str(), _ksize); std::vector nodes; @@ -477,14 +460,32 @@ public: return nodes; } + uint8_t unlink_edge(CompactEdge* edge) { + pdebug("unlink edge " << *edge); + CompactNode *left, *right; + left = get_node_by_id(edge->in_node_id); + right = get_node_by_id(edge->out_node_id); + uint8_t n_deletes = 0; + if (left != nullptr) { + // be lazy for now and use bidirectional delete + left->delete_edge(edge); + n_deletes++; + } + if (right != nullptr) { + right->delete_edge(edge); + n_deletes++; + } + return n_deletes; + } + bool get_pivot_from_left(CompactNode* v, - std::string& segment, + std::string& sequence, char& pivot_base) const { const char * node_kmer = v->sequence.c_str(); const char * _segment = sequence.c_str(); - pivot_base = _segment[segment.size()-_ksize]; + pivot_base = _segment[sequence.size()-_ksize]; if (strncmp(node_kmer, - _segment+(segment.size())-_ksize+1, + _segment+(sequence.size())-_ksize+1, _ksize-1) == 0) { // same canonical orientation return false; @@ -511,9 +512,9 @@ public: bool get_edge_from_left(CompactNode* v, CompactEdge* &result_edge, - std::string& segment) const { + std::string& sequence) const { char pivot_base; - if (!get_pivot_from_left(v, segment, pivot_base)) { + if (!get_pivot_from_left(v, sequence, pivot_base)) { result_edge = v->get_in_edge(pivot_base); return false; } else { @@ -523,7 +524,7 @@ public: } bool get_pivot_from_right(CompactNode* v, - std::string& segment, + std::string& sequence, char& pivot_base) const { const char * node_kmer = v->sequence.c_str(); const char * _segment = sequence.c_str(); @@ -551,13 +552,13 @@ public: bool get_edge_from_right(CompactNode* v, CompactEdge* &result_edge, - std::string& segment) const { + std::string& sequence) const { char pivot_base; - if (!get_pivot_base_from_right(v, segment, pivot_base)) { - result_edge = get_out_edge(pivot_base); + if (!get_pivot_from_right(v, sequence, pivot_base)) { + result_edge = v->get_out_edge(pivot_base); return false; } else { - result_edge = get_in_edge(pivot_base, e); + result_edge = v->get_in_edge(pivot_base); return true; } @@ -565,13 +566,7 @@ public: }; -typedef std::unordered_map TagEdgeMap; -typedef std::pair TagEdgePair; -typedef std::set TagEdgePairSet; -typedef std::set CompactEdgeSet; - - -class StreamingCompactor +class StreamingCompactor : public KmerFactory { protected: @@ -587,13 +582,22 @@ public: shared_ptr graph; StreamingCompactor(shared_ptr graph) : - graph(graph), n_sequences_added(0), - nodes(graph->ksize()), edges(graph->ksize()) + KmerFactory(graph->ksize()), + nodes(graph->ksize()), edges(graph->ksize()), + n_sequences_added(0), graph(graph) { } - WordLength ksize() const { - return graph->ksize(); + compact_edge_meta_t deduce_meta(CompactNode* in, CompactNode* out) { + compact_edge_meta_t edge_meta; + if (in == nullptr && out == nullptr) { + edge_meta = IS_ISLAND; + } else if ((out == nullptr) != (in == nullptr)) { + edge_meta = IS_TIP; + } else { + edge_meta = IS_FULL_EDGE; + } + return edge_meta; } uint64_t n_nodes() const { @@ -612,6 +616,31 @@ public: std::cout << " * " << n_sequences_added << " sequences added" << std::endl; } + + CompactNode* get_node_by_kmer(Kmer hdn) { + return nodes.get_node_by_kmer(hdn); + } + + CompactNode* get_node_by_id(uint64_t id) { + return nodes.get_node_by_id(id); + } + + std::vector get_nodes(const std::string& sequence) { + return nodes.get_nodes(sequence); + } + + CompactEdge* get_edge(HashIntoType tag) const { + return edges.get_edge(tag); + } + + bool get_tag_edge_pair(HashIntoType tag, TagEdgePair& pair) const { + return edges.get_tag_edge_pair(tag, pair); + } + + CompactEdge* get_edge(UHashSet& tags) const { + return edges.get_edge(tags); + } + uint64_t consume_sequence(const std::string& sequence) { uint64_t prev_n_kmers = graph->n_unique_kmers(); graph->consume_string(sequence); @@ -625,15 +654,39 @@ public: return 0; } - bool validate_segment(CompactNode* left_node, CompactNode* right_node, - CompactEdge* edge, std::string& segment) { - + bool validate_segment(CompactNode* root_node, CompactNode* other_node, + CompactEdge* edge, std::string& sequence) { + bool edge_invalid; + if (edge->meta == IS_TIP) { + if (other_node != nullptr) { + return false; + } + if (!((edge->in_node_id == root_node->node_id || + edge->out_node_id == root_node->node_id) && + edge->sequence.length() == sequence.length())) { + return false; + } + } else if (edge->meta == IS_FULL_EDGE) { + if (other_node == nullptr) { + return false; + } else { + bool nodes_match; + nodes_match = (edge->in_node_id == root_node->node_id && + edge->out_node_id == other_node->node_id) || + (edge->out_node_id == root_node->node_id && + edge->in_node_id == other_node->node_id); + if (!nodes_match) { + return false; + } + } + } + return true; } /* Update a compact dbg where there are no induced - * HDNs */ + * HDNs uint64_t update_compact_dbg_linear(std::string& sequence) { - Kmer root_kmer = graph->build_kmer(sequence.substr(0, ksize())); + Kmer root_kmer = graph->build_kmer(sequence.substr(0, _ksize)); CompactingAT lcursor(graph.get(), root_kmer); CompactingAT rcursor(graph.get(), root_kmer); @@ -641,15 +694,15 @@ public: std::string left_seq = cassem._assemble_directed(lcursor); std::string right_seq = cassem._assemble_directed(rcursor); - std::string segment_seq = left_seq + right_seq.substr(ksize()); + std::string segment_seq = left_seq + right_seq.substr(_ksize); CompactNode *left_node = nullptr, *right_node = nullptr; left_node = nodes.get_node_by_kmer(lcursor.cursor); right_node = nodes.get_node_by_kmer(rcursor.cursor); compact_edge_meta_t edge_meta = deduce_edge_meta(left_node, right_node); - char in_base = segment_seq[ksize()-1]; - char out_base = segment_seq[segment_seq.length()-ksize()+1]; + char in_base = segment_seq[_ksize-1]; + char out_base = segment_seq[segment_seq.length()-_ksize+1]; switch(edge_meta) { case IS_FULL_EDGE: @@ -664,6 +717,7 @@ public: break; } } + */ uint64_t update_compact_dbg(const std::string& sequence) { pdebug("update cDBG from " << sequence); @@ -671,7 +725,7 @@ public: // first gather up all k-mers that could have been disturbed -- // k-mers in the read, and the neighbors of the flanking nodes - KmerIterator kmers(sequence.c_str(), ksize()); + KmerIterator kmers(sequence.c_str(), _ksize); KmerQueue disturbed_kmers; Kmer kmer = kmers.next(); CompactingAT lcursor(graph.get(), kmer); @@ -682,8 +736,10 @@ public: } CompactingAT rcursor(graph.get(), kmer); rcursor.neighbors(disturbed_kmers); - - // find the induced HDNs in the disturbed k-mers + + pdebug(disturbed_kmers.size() << " k-mers disturbed" << std::endl); + + // find the induced HDNs in the disturbed k-mers KmerSet induced_hdns; uint64_t n_updates = 0; while(!disturbed_kmers.empty()) { @@ -693,6 +749,7 @@ public: l_degree = lcursor.degree(kmer); r_degree = rcursor.degree(kmer); if(l_degree > 1 || r_degree > 1) { + pdebug("found HDN... " << kmer); CompactNode* hdn = nodes.get_or_build_node(kmer); if (hdn->count == 1) { induced_hdns.insert(kmer); @@ -703,8 +760,7 @@ public: } } } - pdebug(disturbed_kmers.size() << " k-mers disturbed" << std::endl - << induced_hdns.size() << " induced HDNs"); + pdebug(induced_hdns.size() << " induced HDNs"); /* If there are no induced HDNs, we must have extended * a tip or merged two tips into a linear segment */ @@ -719,7 +775,7 @@ public: Kmer root_kmer = *induced_hdns.begin(); induced_hdns.erase(root_kmer); - CompactNode* root_node = get_compact_node_by_kmer(root_kmer); + CompactNode* root_node = nodes.get_node_by_kmer(root_kmer); pdebug("searching from induced HDN: " << *root_node); // check left (in) edges @@ -733,7 +789,7 @@ public: TagEdgePair tag_pair; bool found_tag = false; - lcursor.push_filter(get_tag_stopper(tag_pair, found_tag)); + lcursor.push_filter(edges.get_tag_stopper(tag_pair, found_tag)); std::string segment_seq = cassem._assemble_directed(lcursor); // first check for a segment going this direction from root @@ -744,96 +800,29 @@ public: CompactNode* left_node = nodes.get_node_by_kmer(lcursor.cursor); // validate edge leaving root if it exists - if (segment_edge != nullptr) { - if (segment_edge->meta == IS_TIP) { - if (left_node != nullptr) { - edge_invalid = true; - } - if (!((segment_edge->in_node_id == root_node->node_id || - segment_edge->out_node_id == root_node->node_id) && - segment_edge->sequence.length() == segment_seq.length())) { - edge_invalid = true; - } - } else if (segment_edge->meta == IS_FULL_EDGE) { - if (left_node == nullptr) { - edge_invalid = true; - } else { - bool nodes_match; - nodes_match = (segment_edge->in_node_id == root_node->node_id && - segment_edge->out_node_id == left_node->node_id) || - (segment_edge->out_node_id == root_node->node_id && - segment_edge->in_node_id == left_node->node_id); - edge_invalid = !nodes_match; - } - } - } - if (!edge_invalid) { + if (segment_edge != nullptr && + validate_segment(root_node, left_node, segment_edge, segment_seq)) { + continue; } - - - /* * Should also keep a set of pair to track resolved * segments */ - /* - if (found_tag) { - if (segment_edge != nullptr) { - if (segment_edge - } - if (segment_edge == get_compact_edge(tag_pair.first)) { - pdebug("found tag to existing segment on left"); - left_node = nodes.get_node_by_id(segment_edge->in_node_id); - } else { - pdebug("tags to left don't match existing segment....?"); - } - } else { - */ - //pdebug("no tag to existing segment found..."); - // then left node must have been new or does not exist, - // or the segment was too short to be tagged - - //} - // if there's an existing edge, check if we need to split it if (segment_edge != nullptr) { - pdebug("checking existing edge for comformity"); - CompactNode* existing_out_node; - if (left_flipped) { - existing_out_node = nodes.get_node_by_id(segment_edge->in_node_id); - } else { - existing_out_node = nodes.get_node_by_id(segment_edge->out_node_id); - } - if (found_out_node != nullptr && *found_out_node == *root_node) { - // this edge is fine - pdebug("edges conforms, moving to next neighbor"); - continue; // continue through neighbors - } else { - // need to be careful here, the out node for the - // edge we delete could be linked to another induced - // HDN... - n_updates++; - // check that it isn't a TIP from an existing node - // with ROOT being induced; if not, delete its edge - // from the out node - pdebug("edge does not conform, delete it"); - if (found_out_node != nullptr) { - pdebug("existing edge out node not root, deleting"); - // be lazy for now and use bidirectional delete - existing_out_node->delete_edge(segment_edge); - } - delete_compact_edge(segment_edge); - // deleted tags; assemble out to the HDN - segment_seq = cassem._assemble_directed(lcursor) + - segment_seq.substr(ksize()); - if (left_node != nullptr) { - // not an IN_TIP - left_node->delete_edge(segment_edge); - } - } + + // check that it isn't a TIP from an existing node + // with ROOT being induced; if not, delete its edge + // from the out node + pdebug("edge does not conform, delete it"); + n_updates += nodes.unlink_edge(segment_edge); + edges.delete_edge(segment_edge); + // deleted tags; assemble out to the HDN + segment_seq = cassem._assemble_directed(lcursor) + + segment_seq.substr(_ksize); } // construct the compact edge @@ -842,24 +831,22 @@ public: if (edge_meta == IS_FULL_EDGE) { // left side includes HDN, right side does not - segment_seq = segment_seq.substr(ksize(), - segment_seq.length()-ksize()+1); + + segment_edge = edges.build_edge(left_node->node_id, + root_node->node_id, + edge_meta, + segment_seq.substr(1)); + nodes.add_edge_from_right(left_node, segment_edge); } else { - // unless there is none, in which case - // we take the whole contig - segment_seq = segment_seq.substr(0, segment_seq.length()-ksize()+1); + segment_edge = edges.build_edge(NULL_ID, + root_node->node_id, + edge_meta, + segment_seq); } - n_updates++; - segment_edge = new_compact_edge(lcursor.cursor, - root_node->kmer, - edge_meta, - segment_seq); - if (IS_FULL_EDGE) { - left_node->add_out_edge(segment_seq.front(), segment_edge); - } - root_node->add_in_edge(segment_seq.back(), segment_edge); + n_updates++; + nodes.add_edge_from_left(root_node, segment_edge); } // now the right neighbors... @@ -869,69 +856,55 @@ public: Kmer neighbor = neighbors.back(); neighbors.pop_back(); rcursor.cursor = neighbor; - //if (rcursor.cursor.is_forward() != root_node->direction) { - // rcursor.cursor.set_forward(); - //} - TagEdgePair right_tag_pair; - bool found_right_tag = false; + TagEdgePair tag_pair; + bool found_tag = false; - rcursor.push_filter(get_tag_stopper(right_tag_pair, found_right_tag)); + rcursor.push_filter(edges.get_tag_stopper(tag_pair, found_tag)); std::string segment_seq = cassem._assemble_directed(rcursor); + // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; - CompactNode* right_node = nullptr; + bool edge_invalid = false; + bool root_flipped = nodes.get_edge_from_right(root_node, segment_edge, segment_seq); - if (found_right_tag) { - segment_edge = get_compact_edge(right_tag_pair.first); - right_node = get_compact_node_by_kmer(segment_edge->out_node_id); - } else { - right_node = get_compact_node_by_kmer(rcursor.cursor); - if (right_node != nullptr) { - // base for hypothetical in-edge - auto base = segment_seq[segment_seq.length()-ksize()]; - segment_edge = right_node->get_in_edge(base); - } + CompactNode* right_node = nodes.get_node_by_kmer(rcursor.cursor); + + // validate edge leaving root if it exists + if (segment_edge != nullptr && + validate_segment(root_node, right_node, segment_edge, segment_seq)) { + + continue; } if (segment_edge != nullptr) { - CompactNode* found_in_node = get_compact_node_by_kmer(segment_edge->in_node_id); - if (found_in_node != nullptr && *found_in_node == *root_node) { - continue; - } else { - n_updates++; - if (found_in_node != nullptr) { - found_in_node->delete_out_edge(segment_edge); - } - delete_compact_edge(segment_edge); - segment_seq = segment_seq + - cassem._assemble_directed(rcursor).substr(ksize()); - if (right_node != nullptr) { - right_node->delete_in_edge(segment_edge); - } - } + pdebug("edge does not conform, delete it"); + n_updates += nodes.unlink_edge(segment_edge); + edges.delete_edge(segment_edge); + // deleted tags; assemble out to the HDN + segment_seq = segment_seq + + cassem._assemble_directed(rcursor).substr(_ksize); } compact_edge_meta_t edge_meta = (right_node == nullptr) ? - IS_OUT_TIP : IS_FULL_EDGE; + IS_TIP : IS_FULL_EDGE; + if (edge_meta == IS_FULL_EDGE) { - segment_seq = segment_seq.substr(ksize()-1, - segment_seq.length()-ksize()); + segment_edge = edges.build_edge(root_node->node_id, + right_node->node_id, + edge_meta, + segment_seq.substr(0, segment_seq.length()-1)); + nodes.add_edge_from_left(right_node, segment_edge); } else { - segment_seq = segment_seq.substr(ksize()-1); - } - - n_updates++; - segment_edge = new_compact_edge(root_node->kmer, - rcursor.cursor, - edge_meta, - segment_seq); - if (IS_FULL_EDGE) { - right_node->add_in_edge(segment_seq.back(), segment_edge); + segment_edge = edges.build_edge(root_node->node_id, + NULL_ID, + edge_meta, + segment_seq); } - root_node->add_out_edge(segment_seq.front(), segment_edge); + n_updates++; + nodes.add_edge_from_right(root_node, segment_edge); } } diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 29b5948756..5a993aa54d 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -8,10 +8,13 @@ from libcpp.vector cimport vector from libc.stdint cimport uint8_t, uint32_t, uint64_t from khmer._oxli.oxli_types cimport * -from khmer._oxli.hashing cimport CpKmer, Kmer +from khmer._oxli.hashing cimport CpKmer, Kmer, CpKmerFactory from khmer._oxli.graphs cimport CpHashgraph, Hashgraph, Nodegraph, Countgraph +cdef extern from "oxli/links.hh": + cdef uint64_t NULL_ID + cdef extern from "oxli/links.hh" namespace "oxli" nogil: ctypedef pair[HashIntoType, uint64_t] HashIDPair @@ -19,7 +22,46 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: ctypedef vector[HashIntoType] HashVector ctypedef umap[HashIntoType, uint64_t] HashIDMap - cdef cppclass CpCompactEdge "oxli::CompactEdge" + ctypedef enum compact_edge_meta_t: + IS_FULL_EDGE + IS_TIP + IS_ISLAND + IS_TRIVIAL + + cdef const char * edge_meta_repr(compact_edge_meta_t) + + cdef cppclass CpCompactEdge "oxli::CompactEdge": + uint64_t in_node_id + uint64_t out_node_id + UHashSet tags + compact_edge_meta_t meta + string sequence + + CpCompactEdge(uint64_t, uint64_t) + CpComapctEdge(uint64_t, uint64_t, compact_edge_meta_t) + + string rc_sequence() + void add_tags(UHashSet&) + string tag_viz(WordLength) + float tag_density() + + ctypedef pair[HashIntoType, CpCompactEdge*] TagEdgePair + ctypedef set[TagEdgePair] TagEdgePairSet + + cdef cppclass CpCompactEdgeFactory "oxli::CompactEdgeFactory" (CpKmerFactory): + CpCompactEdgeFactory(WordLength) + + uint64_t n_edges() + CpCompactEdge* build_edge(uint64_t, uint64_t, compact_edge_meta_t, + string) + void delete_edge(CpCompactEdge*) + void delete_edge(UHashSet&) + void delete_edge(HashIntoType) + CpCompactEdge* get_edge(HashIntoType) + bool get_tag_edge_pair(HashIntoType, TagEdgePair&) + CpCompactEdge* get_edge(UHashSet&) + + cdef cppclass CpCompactNode "oxli::CompactNode": CpKmer kmer uint32_t count @@ -36,55 +78,49 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: void add_out_edge(const char, CpCompactEdge*) bool delete_out_edge(CpCompactEdge*) CpCompactEdge* get_out_edge(const char) + bool delete_edge(const char) uint8_t degree() uint8_t out_degree() uint8_t in_degree() + ctypedef vector[CpCompactNode] CompactNodeVector - ctypedef enum compact_edge_meta_t: - IS_FULL_EDGE - IS_IN_TIP - IS_OUT_TIP - IS_ISLAND - IS_TRIVIAL + cdef cppclass CpCompactNodeFactory "oxli::CompactNodeFactory" (CpKmerFactory): + CpCompactNodeFactory(WordLength) + uint64_t n_nodes() - cdef cppclass CpCompactEdge "oxli::CompactEdge": - CpKmer in_node - CpKmer out_node - UHashSet tags - compact_edge_meta_t meta - string sequence + CpCompactNode* build_node(CpKmer) + CpCompactNode* get_node_by_kmer(HashIntoType) + CpCompactNode* get_node_by_id(uint64_t) + CpCompactNode* get_or_build_node(CpKmer) + vector[CpCompactNode*] get_nodes(const string&) - CpCompactEdge(HashIntoType, HashIntoType) - CpComapctEdge(HashIntoType, HashIntoType, compact_edge_meta_t) + uint8_t unlink_edge(CpCompactEdge*) - void add_tags(UHashSet&) - string tag_viz(WordLength) - float tag_density() + bool get_pivot_from_left(CpCompactNode*, string&, char&) + bool add_edge_from_left(CpCompactNode*, CpCompactEdge*) + bool get_edge_from_left(CpCompactNode*, CpCompactEdge* &, string&) - ctypedef vector[CpCompactNode] CompactNodeVector - ctypedef umap[HashIntoType, CpCompactEdge*] TagEdgeMap - ctypedef pair[HashIntoType, CpCompactEdge*] TagEdgePair - ctypedef set[TagEdgePair] TagEdgePairSet + bool get_pivot_from_right(CpCompactNode*, string&, char&) + bool add_edge_from_right(CpCompactNode*, CpCompactEdge*) + bool get_edge_from_right(CpCompactNode*, CpCompactEdge* &, string&) cdef cppclass CpStreamingCompactor "oxli::StreamingCompactor": shared_ptr[CpHashgraph] graph CpStreamingCompactor(shared_ptr[CpHashgraph]) - WordLength ksize() void report() uint64_t n_nodes() uint64_t n_edges() - CpCompactNode* get_compact_node_by_kmer(HashIntoType) - CpCompactNode* get_compact_node_by_id(uint64_t) - CpCompactNode* fetch_or_new_compact_node(CpKmer hdn) - vector[CpCompactNode*] get_compact_nodes(const string&) + CpCompactNode* get_node_by_kmer(HashIntoType) + CpCompactNode* get_node_by_id(uint64_t) + vector[CpCompactNode*] get_nodes(const string&) - CpCompactEdge* get_compact_edge(uint64_t) - CpCompactEdge* get_tag_edge_pair(uint64_t, TagEdgePair&) - CpCompactEdge* get_compact_edge(UHashSet&) + CpCompactEdge* get_edge(HashIntoType) + bool get_tag_edge_pair(uint64_t, TagEdgePair&) + CpCompactEdge* get_edge(UHashSet&) uint64_t update_compact_dbg(const string&) uint64_t consume_sequence(const string&) diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index e8e2917116..02d37c7372 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -23,10 +23,8 @@ cdef class CompactEdge: cdef compact_edge_meta_t meta = deref(self._ce_this).meta if meta == IS_FULL_EDGE: return 'FULL' - elif meta == IS_IN_TIP: - return 'IN_TIP' - elif meta == IS_OUT_TIP: - return 'OUT_TIP' + elif meta == IS_TIP: + return 'TIP' elif meta == IS_ISLAND: return 'ISLAND' elif meta == IS_TRIVIAL: @@ -38,17 +36,13 @@ cdef class CompactEdge: def sequence(self): return deref(self._ce_this).sequence - def in_node(self): - if deref(self._ce_this).meta == IS_IN_TIP or \ - deref(self._ce_this).meta == IS_ISLAND: - return None - return deref(self._ce_this).in_node.kmer_u + def in_node_id(self): + cdef uint64_t nid = deref(self._ce_this).in_node_id + return None if nid == NULL_ID else nid def out_node(self): - if deref(self._ce_this).meta == IS_OUT_TIP or \ - deref(self._ce_this).meta == IS_ISLAND: - return None - return deref(self._ce_this).out_node.kmer_u + cdef uint64_t nid = deref(self._ce_this).out_node_id + return None if nid == NULL_ID else nid def __len__(self): return deref(self._ce_this).sequence.length() @@ -147,11 +141,11 @@ cdef class StreamingCompactor: def consume_and_update(self, str sequence): cdef string _sequence = _bstring(sequence) - return deref(self._sc_this).consume_sequence_and_update(sequence) + return deref(self._sc_this).consume_sequence_and_update(_sequence) def sequence_nodes(self, str sequence): cdef string _sequence = _bstring(sequence) - cdef vector[CpCompactNode*] nodes = deref(self._sc_this).get_compact_nodes(_sequence) + cdef vector[CpCompactNode*] nodes = deref(self._sc_this).get_nodes(_sequence) cdef CpCompactNode* node for node in nodes: yield CompactNode._wrap(node) From e74fa05cc05e565eae1562e87423b25ba7e199ba Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 9 Nov 2017 15:51:03 -0800 Subject: [PATCH 150/185] fix bad strncmp --- include/oxli/links.hh | 49 ++++++++++++++++++++++++++++++++++------ tests/test_linked_dbg.py | 12 +++++----- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 241e4c0e0e..ca8e504e2d 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -44,6 +44,7 @@ Contact: khmer-project@idyll.org #include #include #include +#include #include #include @@ -158,7 +159,8 @@ public: " out_node_id=" << std::to_string(edge.out_node_id) << " length=" << edge.sequence.length() - << " meta=" << edge_meta_repr(edge.meta) << " n_tags=" << edge.tags.size() << ">"; + << " meta=" << edge_meta_repr(edge.meta) + << " n_tags=" << edge.tags.size() << ">"; return stream; } @@ -196,10 +198,17 @@ public: compact_edge_meta_t edge_meta, std::string edge_sequence) { CompactEdge* edge = new CompactEdge(left_id, right_id, edge_meta); - pdebug("new compact edge: left=" << - std::to_string(left_id) << " right=" << - std::to_string(right_id) - << " sequence=" << edge_sequence); + pdebug("new compact edge: \n left=" << std::to_string(left_id) + << std::endl << " right=" << std::to_string(right_id) + << std::endl << " sequence =" << edge_sequence + << std::endl << " rc_sequence=" << _revcomp(edge_sequence) + << std::endl << " start =" << edge_sequence.substr(0, _ksize) + << std::endl << " rc_start=" << _revcomp(edge_sequence.substr(0, _ksize)) + << std::endl << " end =" + << edge_sequence.substr(edge_sequence.length()-_ksize, _ksize) + << std::endl << " rc_end =" + << _revcomp(edge_sequence.substr(edge_sequence.length()-_ksize, _ksize))); + edge->sequence = edge_sequence; n_compact_edges++; return edge; @@ -370,11 +379,31 @@ public: friend std::ostream& operator<<(std::ostream& stream, const CompactNode& node) { stream << ""; return stream; } + + std::string edges_repr() { + std::ostringstream os; + os << *this << std::endl << "\tin_edges:" << std::endl; + for (const char b : "ACGT") { + CompactEdge* e = get_in_edge(b); + if (e != nullptr) { + os << "\t " << b << "=" << *e << std::endl; + } + } + os << "\tout_edges:" << std::endl; + for (const char b : "ACGT") { + CompactEdge* e = get_out_edge(b); + if (e != nullptr) { + os << "\t -" << b << "=" << *e << std::endl; + } + } + return os.str(); + } }; typedef std::vector CompactNodeVector; @@ -500,10 +529,12 @@ public: char pivot_base; if (!get_pivot_from_left(v, e->sequence, pivot_base)) { // same canonical orientation + pdebug("add in edge " << *e << " to node " << *v << " from " << pivot_base); v->add_in_edge(pivot_base, e); return false; } else { // must have opposite canonical orientation + pdebug("add out edge " << *e << " to node " << *v << " from " << pivot_base); v->add_out_edge(pivot_base, e); return true; } @@ -529,7 +560,7 @@ public: const char * node_kmer = v->sequence.c_str(); const char * _segment = sequence.c_str(); pivot_base = _segment[_ksize-1]; - if (strncmp(node_kmer+1, _segment, _ksize-1)) { + if (strncmp(node_kmer+1, _segment, _ksize-1) == 0) { // same canonical orientation return false; } else { @@ -542,9 +573,11 @@ public: bool add_edge_from_right(CompactNode* v, CompactEdge* e) const { char pivot_base; if (!get_pivot_from_right(v, e->sequence, pivot_base)) { + pdebug("add out edge " << *e << " to node " << *v << " from " << pivot_base); v->add_out_edge(pivot_base, e); return false; } else { + pdebug("add in edge " << *e << " to node " << *v << " from " << pivot_base); v->add_in_edge(pivot_base, e); return true; } @@ -776,7 +809,7 @@ public: induced_hdns.erase(root_kmer); CompactNode* root_node = nodes.get_node_by_kmer(root_kmer); - pdebug("searching from induced HDN: " << *root_node); + pdebug("searching from induced HDN: " << root_node->edges_repr()); // check left (in) edges lcursor.neighbors(root_kmer, neighbors); @@ -803,6 +836,8 @@ public: if (segment_edge != nullptr && validate_segment(root_node, left_node, segment_edge, segment_seq)) { + pdebug("validated " << *root_node << ", " << *left_node + << ", " << *segment_edge << ", " << segment_seq); continue; } diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 9414236e87..703bc58205 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -36,16 +36,16 @@ def compare_tip_with_cdbg(rts, compactor): (_, edge_contig), (_, edge_tip) = out_edges if len(edge_tip) > len(edge_contig): edge_contig, edge_tip = edge_tip, edge_contig - assert _equals_rc(contig, in_edge.sequence + node.sequence + - edge_contig.sequence) + #assert _equals_rc(contig, in_edge.sequence[:-K+1] + node.sequence + + # edge_contig.sequence[K-1:]) else: _, out_edge = out_edges[0] assert len(in_edges) == 2 (_, edge_contig), (_, edge_tip) = in_edges if len(edge_tip) > len(edge_contig): edge_contig, edge_tip = edge_tip, edge_contig - assert _equals_rc(contig, edge_contig.sequence + node.sequence + - out_edge.sequence) + #assert _equals_rc(contig, edge_contig.sequence[:-K+1] + node.sequence + + # out_edge.sequence[K-1:]) def test_compact_tip(right_tip_structure): @@ -62,7 +62,7 @@ def test_compact_tip(right_tip_structure): assert compactor.n_nodes == 1 assert compactor.n_edges == 3 - for node in nodes: + for node in compactor.sequence_nodes(contig): print(node) print('in edges:') for base, edge in node.in_edges(): @@ -129,7 +129,7 @@ def test_compact_two_tip_islands(left_tip_structure, right_tip_structure): def test_compact_tip_x_merge(left_tip_structure, right_tip_structure): - graph, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure + _, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure _, contig_l, L_l, HDN_l, R_l, tip_l = left_tip_structure contig_merge = contig_l + contig_r From 09efacb615e3999975274d2f654e355894fa75d8 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 10 Nov 2017 12:32:08 -0800 Subject: [PATCH 151/185] Fix edge case where out or in edge from discovered node which skips induced node won't be delete --- include/oxli/hashtable.hh | 4 ++ include/oxli/links.hh | 120 +++++++++++++++++++++++++------------- include/oxli/storage.hh | 22 ++++++- khmer/_oxli/graphs.pxd | 2 + khmer/_oxli/graphs.pyx | 6 ++ src/oxli/storage.cc | 10 ++++ tests/test_linked_dbg.py | 12 ++-- 7 files changed, 128 insertions(+), 48 deletions(-) diff --git a/include/oxli/hashtable.hh b/include/oxli/hashtable.hh index 1f95605130..60ac98ed2f 100644 --- a/include/oxli/hashtable.hh +++ b/include/oxli/hashtable.hh @@ -397,6 +397,10 @@ public: return store->get_raw_tables(); } + void reset() { + store->reset(); + } + // find the minimum k-mer count in the given sequence BoundedCounterType get_min_count(const std::string &s); diff --git a/include/oxli/links.hh b/include/oxli/links.hh index ca8e504e2d..52c6f56c2e 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -689,19 +689,22 @@ public: bool validate_segment(CompactNode* root_node, CompactNode* other_node, CompactEdge* edge, std::string& sequence) { - bool edge_invalid; + pdebug("validating " << *root_node << " with " << *edge << ", " + << sequence << " and other node ID=" << + ((other_node != nullptr) ? other_node->node_id : NULL_ID)); + bool edge_valid = true; if (edge->meta == IS_TIP) { if (other_node != nullptr) { - return false; + edge_valid = false; } if (!((edge->in_node_id == root_node->node_id || edge->out_node_id == root_node->node_id) && edge->sequence.length() == sequence.length())) { - return false; + edge_valid = false; } } else if (edge->meta == IS_FULL_EDGE) { if (other_node == nullptr) { - return false; + edge_valid = false; } else { bool nodes_match; nodes_match = (edge->in_node_id == root_node->node_id && @@ -709,11 +712,12 @@ public: (edge->out_node_id == root_node->node_id && edge->in_node_id == other_node->node_id); if (!nodes_match) { - return false; + edge_valid = false; } } } - return true; + pdebug("valid? = " << edge_valid); + return edge_valid; } /* Update a compact dbg where there are no induced @@ -827,18 +831,42 @@ public: // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; - bool edge_invalid = false; - bool root_flipped = nodes.get_edge_from_left(root_node, segment_edge, segment_seq); + nodes.get_edge_from_left(root_node, segment_edge, segment_seq); CompactNode* left_node = nodes.get_node_by_kmer(lcursor.cursor); + CompactEdge* left_out_edge = nullptr; + if (left_node != nullptr) { + nodes.get_edge_from_right(left_node, left_out_edge, segment_seq); + } // validate edge leaving root if it exists - if (segment_edge != nullptr && - validate_segment(root_node, left_node, segment_edge, segment_seq)) { - - pdebug("validated " << *root_node << ", " << *left_node - << ", " << *segment_edge << ", " << segment_seq); - continue; + if (segment_edge != nullptr && left_out_edge != nullptr) { + + + if (segment_edge == left_out_edge && + validate_segment(root_node, left_node, + segment_edge, segment_seq)) { + continue; + } else { + nodes.unlink_edge(segment_edge); + nodes.unlink_edge(left_out_edge); + edges.delete_edge(segment_edge); + edges.delete_edge(left_out_edge); + } + } else if (left_out_edge != nullptr) { + // there was no edge from root, must be bad + pdebug("edge from left invalid, delete"); + n_updates += nodes.unlink_edge(left_out_edge); + edges.delete_edge(left_out_edge); + } else if (segment_edge != nullptr) { + if (validate_segment(root_node, left_node, + segment_edge, segment_seq)) { + continue; + } else { + pdebug("edge from root invalid, delete"); + n_updates += nodes.unlink_edge(segment_edge); + edges.delete_edge(segment_edge); + } } /* @@ -846,19 +874,9 @@ public: * segments */ - // if there's an existing edge, check if we need to split it - if (segment_edge != nullptr) { - - // check that it isn't a TIP from an existing node - // with ROOT being induced; if not, delete its edge - // from the out node - pdebug("edge does not conform, delete it"); - n_updates += nodes.unlink_edge(segment_edge); - edges.delete_edge(segment_edge); - // deleted tags; assemble out to the HDN - segment_seq = cassem._assemble_directed(lcursor) + - segment_seq.substr(_ksize); - } + // not needed until tags used again + //segment_seq = cassem._assemble_directed(lcursor) + + // segment_seq.substr(_ksize); // construct the compact edge compact_edge_meta_t edge_meta = (left_node == nullptr) ? @@ -900,27 +918,45 @@ public: // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; - bool edge_invalid = false; - bool root_flipped = nodes.get_edge_from_right(root_node, segment_edge, segment_seq); + nodes.get_edge_from_right(root_node, segment_edge, segment_seq); CompactNode* right_node = nodes.get_node_by_kmer(rcursor.cursor); - - // validate edge leaving root if it exists - if (segment_edge != nullptr && - validate_segment(root_node, right_node, segment_edge, segment_seq)) { - - continue; + CompactEdge* right_in_edge = nullptr; + if (right_node != nullptr) { + nodes.get_edge_from_left(right_node, right_in_edge, segment_seq); } - if (segment_edge != nullptr) { - pdebug("edge does not conform, delete it"); - n_updates += nodes.unlink_edge(segment_edge); - edges.delete_edge(segment_edge); - // deleted tags; assemble out to the HDN - segment_seq = segment_seq + - cassem._assemble_directed(rcursor).substr(_ksize); + // validate edge leaving root if it exists + if (segment_edge != nullptr && right_in_edge != nullptr) { + + + if (segment_edge == right_in_edge && + validate_segment(root_node, right_node, + segment_edge, segment_seq)) { + continue; + } else { + nodes.unlink_edge(segment_edge); + nodes.unlink_edge(right_in_edge); + edges.delete_edge(segment_edge); + edges.delete_edge(right_in_edge); + } + } else if (right_in_edge != nullptr) { + // there was no edge from root, must be bad + pdebug("edge from left invalid, delete"); + n_updates += nodes.unlink_edge(right_in_edge); + edges.delete_edge(right_in_edge); + } else if (segment_edge != nullptr) { + if (validate_segment(root_node, right_node, + segment_edge, segment_seq)) { + continue; + } else { + pdebug("edge from root invalid, delete"); + n_updates += nodes.unlink_edge(segment_edge); + edges.delete_edge(segment_edge); + } } + pdebug("segment sequence length=" << segment_seq.length()); compact_edge_meta_t edge_meta = (right_node == nullptr) ? IS_TIP : IS_FULL_EDGE; diff --git a/include/oxli/storage.hh b/include/oxli/storage.hh index 156c503630..ca380d9e65 100644 --- a/include/oxli/storage.hh +++ b/include/oxli/storage.hh @@ -73,6 +73,7 @@ public: virtual bool add(HashIntoType khash) = 0; virtual const BoundedCounterType get_count(HashIntoType khash) const = 0; virtual Byte ** get_raw_tables() = 0; + virtual void reset() = 0; void set_use_bigcount(bool b); bool get_use_bigcount(); @@ -226,6 +227,8 @@ public: return _counts; } + void reset(); + void update_from(const BitStorage&); }; @@ -309,7 +312,15 @@ public: memset(_counts[i], 0, tablebytes); } } - + + void reset() + { + for (unsigned int table_num = 0; table_num < _n_tables; table_num++) { + uint64_t tablesize = _tablesizes[table_num]; + uint64_t tablebytes = tablesize / 2 + 1; + memset(_counts[table_num], 0, tablebytes); + } + } BoundedCounterType test_and_set_bits(HashIntoType khash) { @@ -443,6 +454,7 @@ public: void load(std::string infilename, WordLength &ksize); Byte **get_raw_tables() { return nullptr; } + void reset() {}; //nop }; @@ -528,6 +540,14 @@ public: } } + void reset() + { + for (unsigned int table_num = 0; table_num < _n_tables; table_num++) { + uint64_t tablesize = _tablesizes[table_num]; + memset(_counts[table_num], 0, tablesize); + } + } + std::vector get_tablesizes() const { return _tablesizes; diff --git a/khmer/_oxli/graphs.pxd b/khmer/_oxli/graphs.pxd index 9c0ceefaca..4b5219e5fb 100644 --- a/khmer/_oxli/graphs.pxd +++ b/khmer/_oxli/graphs.pxd @@ -36,6 +36,7 @@ cdef extern from "oxli/storage.hh": void set_use_bigcount(bool) bool get_use_bigcount() + void reset() cdef extern from "oxli/hashtable.hh" namespace "oxli" nogil: @@ -103,6 +104,7 @@ cdef extern from "oxli/hashtable.hh" namespace "oxli" nogil: uint64_t trim_below_abundance(string, BoundedCounterType) const vector[uint32_t] find_spectral_error_positions(string, BoundedCounterType) + void reset() cdef cppclass CpMurmurHashtable "oxli::MurmurHashtable" (CpHashtable): CpMurmurHashtable(WordLength, CpStorage *) diff --git a/khmer/_oxli/graphs.pyx b/khmer/_oxli/graphs.pyx index 032325ddd2..0a5b517b9a 100644 --- a/khmer/_oxli/graphs.pyx +++ b/khmer/_oxli/graphs.pyx @@ -362,6 +362,9 @@ cdef class Hashtable: cdef vector[uint64_t] sizes = deref(self._ht_this).get_tablesizes() return self._get_raw_tables(table_ptrs, sizes) + def reset(self): + deref(self._ht_this).reset() + cdef class QFCounttable(Hashtable): """Count kmers using a counting quotient filter. @@ -405,6 +408,9 @@ cdef class QFCounttable(Hashtable): deref(table._qf_this).load(_bstring(file_name)) return table + def reset(self): + raise NotImplementedError() + cdef class Counttable(Hashtable): def __cinit__(self, int k, uint64_t starting_size, int n_tables): diff --git a/src/oxli/storage.cc b/src/oxli/storage.cc index 174bb5f822..7097c2229a 100644 --- a/src/oxli/storage.cc +++ b/src/oxli/storage.cc @@ -97,6 +97,16 @@ void BitStorage::update_from(const BitStorage& other) } +void BitStorage::reset() +{ + for (unsigned int table_num = 0; table_num < _n_tables; table_num++) { + uint64_t tablesize = _tablesizes[table_num]; + uint64_t tablebytes = tablesize / 8 + 1; + memset(_counts[table_num], 0, tablebytes); + } +} + + void BitStorage::save(std::string outfilename, WordLength ksize) { if (!_counts[0]) { diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 703bc58205..4685b7a09e 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -129,20 +129,22 @@ def test_compact_two_tip_islands(left_tip_structure, right_tip_structure): def test_compact_tip_x_merge(left_tip_structure, right_tip_structure): - _, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure + graph, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure _, contig_l, L_l, HDN_l, R_l, tip_l = left_tip_structure - + contig_merge = contig_l + contig_r + graph.reset() compactor = StreamingCompactor(graph) - print(compactor.update(contig_l), 'cDBG updates from left') + compactor.consume(str(tip_l)) + print(compactor.consume_and_update(contig_l), 'cDBG updates from left') compactor.report() compare_tip_with_cdbg(left_tip_structure, compactor) assert compactor.n_nodes == 1 assert compactor.n_edges == 3 - compactor.consume(contig_merge) - print(compactor.update(contig_r), 'cDBG updates from right merge') + compactor.consume(str(tip_r)) + print(compactor.consume_and_update(contig_merge), 'cDBG updates from right merge') compactor.report() #compare_tip_with_cdbg(right_tip_structure, compactor) assert compactor.n_nodes == 2 From c3dcff0cc92e401800d2cdb058b3464c1cf6304f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 13 Nov 2017 14:38:31 -0800 Subject: [PATCH 152/185] add trivial edges --- include/oxli/links.hh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 52c6f56c2e..e1694a8d72 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -198,6 +198,14 @@ public: compact_edge_meta_t edge_meta, std::string edge_sequence) { CompactEdge* edge = new CompactEdge(left_id, right_id, edge_meta); + if (edge_meta == IS_TIP) { + if (left_id == NULL_ID) { + edge_sequence = edge_sequence.substr(0, edge_sequence.length() - 1); + } + if (right_id == NULL_ID) { + edge_sequence = edge_sequence.substr(1); + } + } pdebug("new compact edge: \n left=" << std::to_string(left_id) << std::endl << " right=" << std::to_string(right_id) << std::endl << " sequence =" << edge_sequence @@ -879,16 +887,18 @@ public: // segment_seq.substr(_ksize); // construct the compact edge - compact_edge_meta_t edge_meta = (left_node == nullptr) ? - IS_TIP : IS_FULL_EDGE; + compact_edge_meta_t edge_meta = (left_node == nullptr) + ? IS_TIP : IS_FULL_EDGE; + edge_meta = (segment_seq.length() == ksize()) + ? IS_TRIVIAL : edge_meta; - if (edge_meta == IS_FULL_EDGE) { + if (edge_meta == IS_FULL_EDGE || edge_meta == IS_TRIVIAL) { // left side includes HDN, right side does not segment_edge = edges.build_edge(left_node->node_id, root_node->node_id, edge_meta, - segment_seq.substr(1)); + segment_seq); nodes.add_edge_from_right(left_node, segment_edge); } else { segment_edge = edges.build_edge(NULL_ID, @@ -959,12 +969,14 @@ public: pdebug("segment sequence length=" << segment_seq.length()); compact_edge_meta_t edge_meta = (right_node == nullptr) ? IS_TIP : IS_FULL_EDGE; + edge_meta = (segment_seq.length() == ksize()) + ? IS_TRIVIAL : edge_meta; if (edge_meta == IS_FULL_EDGE) { segment_edge = edges.build_edge(root_node->node_id, right_node->node_id, edge_meta, - segment_seq.substr(0, segment_seq.length()-1)); + segment_seq); nodes.add_edge_from_left(right_node, segment_edge); } else { segment_edge = edges.build_edge(root_node->node_id, From 67d5d5547fa7d76085a3297a5ca014b419b80530 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 14 Nov 2017 12:26:38 -0800 Subject: [PATCH 153/185] Include HDN sequence in edge sequence --- include/oxli/links.hh | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index e1694a8d72..ee3ce3153a 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -197,15 +197,9 @@ public: CompactEdge* build_edge(uint64_t left_id, uint64_t right_id, compact_edge_meta_t edge_meta, std::string edge_sequence) { + CompactEdge* edge = new CompactEdge(left_id, right_id, edge_meta); - if (edge_meta == IS_TIP) { - if (left_id == NULL_ID) { - edge_sequence = edge_sequence.substr(0, edge_sequence.length() - 1); - } - if (right_id == NULL_ID) { - edge_sequence = edge_sequence.substr(1); - } - } + pdebug("new compact edge: \n left=" << std::to_string(left_id) << std::endl << " right=" << std::to_string(right_id) << std::endl << " sequence =" << edge_sequence @@ -516,13 +510,13 @@ public: } bool get_pivot_from_left(CompactNode* v, - std::string& sequence, - char& pivot_base) const { + std::string& sequence, + char& pivot_base) const { const char * node_kmer = v->sequence.c_str(); const char * _segment = sequence.c_str(); - pivot_base = _segment[sequence.size()-_ksize]; + pivot_base = _segment[sequence.size()-_ksize-1]; if (strncmp(node_kmer, - _segment+(sequence.size())-_ksize+1, + _segment+(sequence.size())-_ksize, _ksize-1) == 0) { // same canonical orientation return false; @@ -567,8 +561,8 @@ public: char& pivot_base) const { const char * node_kmer = v->sequence.c_str(); const char * _segment = sequence.c_str(); - pivot_base = _segment[_ksize-1]; - if (strncmp(node_kmer+1, _segment, _ksize-1) == 0) { + pivot_base = _segment[_ksize]; + if (strncmp(node_kmer+1, _segment+1, _ksize-1) == 0) { // same canonical orientation return false; } else { @@ -818,6 +812,10 @@ public: KmerQueue neighbors; while(!induced_hdns.empty()) { Kmer root_kmer = *induced_hdns.begin(); + std::string root_kmer_seq = root_kmer.get_string_rep(_ksize); + char root_front = root_kmer_seq.front(); + char root_back = root_kmer_seq.back(); + induced_hdns.erase(root_kmer); CompactNode* root_node = nodes.get_node_by_kmer(root_kmer); @@ -835,7 +833,8 @@ public: bool found_tag = false; lcursor.push_filter(edges.get_tag_stopper(tag_pair, found_tag)); - std::string segment_seq = cassem._assemble_directed(lcursor); + std::string segment_seq = cassem._assemble_directed(lcursor) + + root_back; // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; @@ -889,7 +888,7 @@ public: // construct the compact edge compact_edge_meta_t edge_meta = (left_node == nullptr) ? IS_TIP : IS_FULL_EDGE; - edge_meta = (segment_seq.length() == ksize()) + edge_meta = (segment_seq.length() == _ksize) ? IS_TRIVIAL : edge_meta; if (edge_meta == IS_FULL_EDGE || edge_meta == IS_TRIVIAL) { @@ -924,7 +923,7 @@ public: bool found_tag = false; rcursor.push_filter(edges.get_tag_stopper(tag_pair, found_tag)); - std::string segment_seq = cassem._assemble_directed(rcursor); + std::string segment_seq = root_front + cassem._assemble_directed(rcursor); // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; @@ -969,7 +968,7 @@ public: pdebug("segment sequence length=" << segment_seq.length()); compact_edge_meta_t edge_meta = (right_node == nullptr) ? IS_TIP : IS_FULL_EDGE; - edge_meta = (segment_seq.length() == ksize()) + edge_meta = (segment_seq.length() == _ksize) ? IS_TRIVIAL : edge_meta; if (edge_meta == IS_FULL_EDGE) { From c673b360fe15b0609e9519c1d0311a8fa1d08d52 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 14 Nov 2017 12:29:57 -0800 Subject: [PATCH 154/185] Fix trivial edge length --- include/oxli/links.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index ee3ce3153a..6ad48b3596 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -888,7 +888,7 @@ public: // construct the compact edge compact_edge_meta_t edge_meta = (left_node == nullptr) ? IS_TIP : IS_FULL_EDGE; - edge_meta = (segment_seq.length() == _ksize) + edge_meta = (segment_seq.length() == _ksize + 1) ? IS_TRIVIAL : edge_meta; if (edge_meta == IS_FULL_EDGE || edge_meta == IS_TRIVIAL) { @@ -968,7 +968,7 @@ public: pdebug("segment sequence length=" << segment_seq.length()); compact_edge_meta_t edge_meta = (right_node == nullptr) ? IS_TIP : IS_FULL_EDGE; - edge_meta = (segment_seq.length() == _ksize) + edge_meta = (segment_seq.length() == _ksize + 1) ? IS_TRIVIAL : edge_meta; if (edge_meta == IS_FULL_EDGE) { From 65bfb5f1d5ec030c627fee1f3bae3ad4d6279fac Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 14 Nov 2017 12:30:11 -0800 Subject: [PATCH 155/185] Test right half of x structure --- tests/test_linked_dbg.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 4685b7a09e..2436720d31 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -137,15 +137,25 @@ def test_compact_tip_x_merge(left_tip_structure, right_tip_structure): compactor = StreamingCompactor(graph) compactor.consume(str(tip_l)) - print(compactor.consume_and_update(contig_l), 'cDBG updates from left') + print(compactor.consume_and_update(contig_l), + 'cDBG updates from left') compactor.report() compare_tip_with_cdbg(left_tip_structure, compactor) assert compactor.n_nodes == 1 assert compactor.n_edges == 3 compactor.consume(str(tip_r)) - print(compactor.consume_and_update(contig_merge), 'cDBG updates from right merge') + print(compactor.consume_and_update(contig_merge), + 'cDBG updates from right merge') compactor.report() - #compare_tip_with_cdbg(right_tip_structure, compactor) + compare_tip_with_cdbg(right_tip_structure, compactor) assert compactor.n_nodes == 2 assert compactor.n_edges == 5 + + +def test_flanking_hdns(): + pass + + +def test_compact_tip_split_merge(): + pass From 873232ad41ad5b8f58da26dc64329065dc5b0574 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 15 Nov 2017 00:27:35 -0800 Subject: [PATCH 156/185] Convert K size to fixture --- tests/conftest.py | 14 ++ ...eatures.py => graph_structure_fixtures.py} | 167 +++++++++++------- tests/test_assembly.py | 143 ++++++++------- tests/test_cython_partitioning.py | 2 +- 4 files changed, 185 insertions(+), 141 deletions(-) create mode 100644 tests/conftest.py rename tests/{graph_features.py => graph_structure_fixtures.py} (79%) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..eb859cb21c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ + + + +def pytest_generate_tests(metafunc): + if 'ksize' in metafunc.fixturenames: + ksize = getattr(metafunc.function, 'ksize', None) + if ksize is None: + ksize = [21] + if isinstance(ksize, int): + ksize = [ksize] + metafunc.parametrize('ksize', ksize, indirect=True, + ids=lambda k: 'K={0}'.format(k)) + + diff --git a/tests/graph_features.py b/tests/graph_structure_fixtures.py similarity index 79% rename from tests/graph_features.py rename to tests/graph_structure_fixtures.py index a616003ed0..d2a8b9261b 100644 --- a/tests/graph_features.py +++ b/tests/graph_structure_fixtures.py @@ -51,7 +51,36 @@ # We just define this globally rather than in a module-level fixture, # as we need it during parameterization and whatnot. -K = 21 + +def set_ksize(K=21): + def wrap(func): + setattr(func, 'ksize', K) + return func + return wrap + + +@pytest.fixture +def ksize(request): + print('ksize:', request.param) + return request.param + +def test_ksize(ksize): + assert ksize == 21 + + +@set_ksize(31) +def test_ksize_override(ksize): + assert ksize == 31 + + +@set_ksize([21, 25, 29]) +def test_ksize_override_param(ksize): + assert ksize in [21, 25, 29] + + +@pytest.fixture(params=[2, -2], ids=['Start', 'End']) +def flank_coords(request, ksize): + return (request.param * ksize) + request.param class Kmer(str): @@ -60,8 +89,6 @@ def __init__(self, value, pos=0): self.pos = pos def __new__(cls, value, pos=0): - if not len(value) == K: - raise ValueError('bad k-mer length') return str.__new__(cls, value) def __repr__(self): @@ -93,7 +120,7 @@ def mutate_position(sequence, pos): return ''.join(sequence) -def get_random_sequence(length, exclude=None, seen=None): +def get_random_sequence(length, ksize, exclude=None, seen=None): '''Generate a random (non-looping) nucleotide sequence. To be non-overlapping, the sequence should not include any repeated @@ -114,16 +141,16 @@ def add_seen(kmer): seen.add(revcomp(kmer)) if exclude is not None: - for pos in range(0, len(exclude) - K): - add_seen(exclude[pos:pos + K - 1]) + for pos in range(0, len(exclude) - ksize): + add_seen(exclude[pos:pos + ksize - 1]) - seq = [random.choice('ACGT') for _ in range(K - 1)] # do first K-1 bases + seq = [random.choice('ACGT') for _ in range(ksize - 1)] # do first K-1 bases add_seen(''.join(seq)) while(len(seq) < length): next_base = random.choice('ACGT') - next_kmer = ''.join(seq[-K + 2:] + [next_base]) - assert len(next_kmer) == K - 1 + next_kmer = ''.join(seq[-ksize + 2:] + [next_base]) + assert len(next_kmer) == ksize - 1 if (next_kmer) not in seen: seq.append(next_base) add_seen(next_kmer) @@ -132,12 +159,12 @@ def add_seen(kmer): return ''.join(seq) -def reads(sequence, L=100, N=100, dbg_cover=False): +def reads(sequence, ksize, L=100, N=100, dbg_cover=False): positions = list(range(len(sequence) - L)) if dbg_cover is True: - for start in range(0, len(sequence), K): + for start in range(0, len(sequence), ksize): read = sequence[start:start + L] - if len(read) < K: + if len(read) < ksize: read = sequence[-L:] yield read N -= 1 @@ -148,11 +175,18 @@ def reads(sequence, L=100, N=100, dbg_cover=False): yield sequence[start:start + L] -def kmers(sequence): +def kmers(sequence, K=21): for i in range(len(sequence) - K + 1): yield sequence[i:i + K] +@set_ksize([5, 7]) +def test_kmers(ksize): + S = 'A' * ksize + 'T' + res = list(kmers(S, ksize)) + assert res[0] == 'A' * ksize + assert res[-1] == ('A' * (ksize - 1)) + 'T' + def test_mutate_sequence(): for _ in range(100): assert 'A' not in mutate_sequence('A' * 10, 10) @@ -168,14 +202,14 @@ def test_mutate_position(): assert mutate_position('GGGG', 2) in ['GGAG', 'GGTG'] -def test_reads(): +def test_reads(ksize): contigfile = utils.get_test_data('simple-genome.fa') contig = list(screed.open(contigfile))[0].sequence - for read in reads(contig): + for read in reads(contig, ksize): assert read in contig - for read in reads(contig): + for read in reads(contig, ksize): assert mutate_sequence(read) not in contig @@ -206,16 +240,17 @@ def known_sequence(request): @pytest.fixture(params=list(range(500, 1600, 500)), ids=lambda val: '(L={0})'.format(val)) -def random_sequence(request): +def random_sequence(request, ksize): global_seen = set() def get(exclude=None): sequence = get_random_sequence(request.param, + ksize, exclude=exclude, seen=global_seen) - for i in range(len(sequence)-K): - global_seen.add(sequence[i:i+K-1]) - global_seen.add(revcomp(sequence[i:i+K-1])) + for i in range(len(sequence)-ksize): + global_seen.add(sequence[i:i+ksize-1]) + global_seen.add(revcomp(sequence[i:i+ksize-1])) return sequence return get @@ -223,14 +258,14 @@ def get(exclude=None): @pytest.fixture(params=[khmer.Nodegraph, khmer.Countgraph], ids=['(Type=Nodegraph)', '(Type=Countgraph)']) -def graph(request): +def graph(request, ksize): num_kmers = 50000 des_fp = 0.00001 args = optimal_fp(num_kmers, des_fp) print('Graph Params:', args) - return request.param(K, args.htable_size, args.num_htables) + return request.param(ksize, args.htable_size, args.num_htables) def hdn_counts(sequence, graph): @@ -247,7 +282,7 @@ def hdn_counts(sequence, graph): @pytest.fixture -def linear_structure(request, graph, random_sequence): +def linear_structure(request, graph, ksize, random_sequence): '''Sets up a simple linear path graph structure. sequence @@ -264,9 +299,8 @@ def linear_structure(request, graph, random_sequence): return graph, sequence -@pytest.fixture(params=[K * 2, -K * 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def right_tip_structure(request, graph, random_sequence): +@pytest.fixture +def right_tip_structure(request, graph, ksize, flank_coords, random_sequence): ''' Sets up a graph structure like so: ([S+1:S+K]+B tip) @@ -277,15 +311,15 @@ def right_tip_structure(request, graph, random_sequence): That is, it has a single branch at the Sth K-mer. ''' sequence = random_sequence() - S = request.param + S = flank_coords if S < 0: S = len(sequence) + S # the HDN - HDN = Kmer(sequence[S:S + K], pos=S) + HDN = Kmer(sequence[S:S + ksize], pos=S) # left of the HDN - L = Kmer(sequence[S - 1:S - 1 + K], pos=S - 1) + L = Kmer(sequence[S - 1:S - 1 + ksize], pos=S - 1) # right of the HDN - R = Kmer(sequence[S + 1:S + 1 + K], pos=S + 1) + R = Kmer(sequence[S + 1:S + 1 + ksize], pos=S + 1) # the branch kmer tip = Kmer(mutate_position(R, -1), pos=R.pos) @@ -300,9 +334,9 @@ def right_tip_structure(request, graph, random_sequence): return graph, sequence, L, HDN, R, tip -@pytest.fixture(params=[K * 2, -K * 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def right_double_fork_structure(request, linear_structure, random_sequence): +@pytest.fixture +def right_double_fork_structure(request, ksize, flank_coords, + linear_structure, random_sequence): ''' Sets up a graph structure like so: branch @@ -320,15 +354,15 @@ def right_double_fork_structure(request, linear_structure, random_sequence): print('Branch len:', len(branch_sequence)) # start position of the HDN - S = request.param + S = flank_coords if S < 0: S = len(core_sequence) + S # the HDN - HDN = Kmer(core_sequence[S:S + K], pos=S) + HDN = Kmer(core_sequence[S:S + ksize], pos=S) # left of the HDN - L = Kmer(core_sequence[S - 1:S - 1 + K], pos=S - 1) + L = Kmer(core_sequence[S - 1:S - 1 + ksize], pos=S - 1) # right of the HDN - R = Kmer(core_sequence[S + 1:S + 1 + K], pos=S + 1) + R = Kmer(core_sequence[S + 1:S + 1 + ksize], pos=S + 1) # the branch sequence, mutated at position S+1 branch_start = core_sequence[:R.pos] + mutate_position(R, -1) branch_sequence = branch_start + branch_sequence @@ -351,7 +385,7 @@ def right_double_fork_structure(request, linear_structure, random_sequence): @pytest.fixture def right_triple_fork_structure(request, right_double_fork_structure, - random_sequence): + random_sequence, ksize): ''' Sets up a graph structure like so: @@ -373,9 +407,9 @@ def right_triple_fork_structure(request, right_double_fork_structure, # the branch sequence, mutated at position S+1 # choose a base not already represented at that position bases = {'A', 'C', 'G', 'T'} - mutated = random.choice(list(bases - {R[-1], top_sequence[R.pos + K - 1]})) + mutated = random.choice(list(bases - {R[-1], top_sequence[R.pos + ksize - 1]})) - bottom_sequence = core_sequence[:HDN.pos + K] + mutated + bottom_branch + bottom_sequence = core_sequence[:HDN.pos + ksize] + mutated + bottom_branch graph.consume(bottom_sequence) @@ -393,9 +427,8 @@ def right_triple_fork_structure(request, right_double_fork_structure, return graph, core_sequence, L, HDN, R, top_sequence, bottom_sequence -@pytest.fixture(params=[K * 2, -K * 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def left_tip_structure(request, graph, random_sequence): +@pytest.fixture +def left_tip_structure(request, graph, ksize, flank_coords, random_sequence): ''' Sets up a graph structure like so: @@ -407,14 +440,14 @@ def left_tip_structure(request, graph, random_sequence): Where S is the start position of the HDN. ''' sequence = random_sequence() - S = request.param + S = flank_coords if S < 0: S = len(sequence) + S - tip = Kmer(mutate_position(sequence[S - 1:S - 1 + K], 0), - pos=S - 1 + K) - HDN = Kmer(sequence[S:S + K], pos=S) - L = Kmer(sequence[S - 1:S - 1 + K], pos=S - 1) - R = Kmer(sequence[S + 1:S + 1 + K], pos=S + 1) + tip = Kmer(mutate_position(sequence[S - 1:S - 1 + ksize], 0), + pos=S - 1 + ksize) + HDN = Kmer(sequence[S:S + ksize], pos=S) + L = Kmer(sequence[S - 1:S - 1 + ksize], pos=S - 1) + R = Kmer(sequence[S + 1:S + 1 + ksize], pos=S + 1) graph.consume(sequence) graph.count(tip) @@ -426,9 +459,9 @@ def left_tip_structure(request, graph, random_sequence): return graph, sequence, L, HDN, R, tip -@pytest.fixture(params=[K * 2, -K * 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def left_double_fork_structure(request, linear_structure, random_sequence): +@pytest.fixture +def left_double_fork_structure(request, linear_structure, ksize, + flank_coords, random_sequence): ''' Sets up a graph structure like so: @@ -443,20 +476,20 @@ def left_double_fork_structure(request, linear_structure, random_sequence): branch_sequence = random_sequence(exclude=core_sequence) # start position of the HDN - S = request.param + S = flank_coords if S < 0: S = len(core_sequence) + S # the HDN - HDN = Kmer(core_sequence[S:S + K], pos=S) + HDN = Kmer(core_sequence[S:S + ksize], pos=S) # left of the HDN - L = Kmer(core_sequence[S - 1:S - 1 + K], pos=S - 1) + L = Kmer(core_sequence[S - 1:S - 1 + ksize], pos=S - 1) # right of the HDN - R = Kmer(core_sequence[S + 1:S + 1 + K], pos=S + 1) + R = Kmer(core_sequence[S + 1:S + 1 + ksize], pos=S + 1) # the branch sequence, mutated at position 0 in L, # whih is equivalent to the K-1 prefix of HDN prepended with a new base branch_start = mutate_position(L, 0) branch_sequence = branch_sequence + \ - branch_start + core_sequence[L.pos + K:] + branch_start + core_sequence[L.pos + ksize:] graph.consume(core_sequence) graph.consume(branch_sequence) @@ -473,9 +506,8 @@ def left_double_fork_structure(request, linear_structure, random_sequence): return graph, core_sequence, L, HDN, R, branch_sequence -@pytest.fixture(params=[K * 2, (-K * 2) - 2], - ids=['(Where={0})'.format(i) for i in ['Start', 'End']]) -def snp_bubble_structure(request, linear_structure): +@pytest.fixture +def snp_bubble_structure(request, linear_structure, ksize): ''' Sets up a graph structure resulting from a SNP (Single Nucleotide Polymorphism). @@ -496,12 +528,11 @@ def snp_bubble_structure(request, linear_structure): ''' graph, wildtype_sequence = linear_structure - S = request.param - if S < 0: - S = len(wildtype_sequence) + S - snp_sequence = mutate_position(wildtype_sequence, S + K) - HDN_L = Kmer(wildtype_sequence[S:S + K], pos=S) - HDN_R = Kmer(wildtype_sequence[S + K + 1:S + 2 * K + 1], pos=S + K + 1) + S = int(len(wildtype_sequence) / 2) + snp_sequence = mutate_position(wildtype_sequence, S + ksize) + HDN_L = Kmer(wildtype_sequence[S:S + ksize], pos=S) + HDN_R = Kmer(wildtype_sequence[S + ksize + 1:S + 2 * ksize + 1], pos=S + + ksize + 1) graph.consume(wildtype_sequence) graph.consume(snp_sequence) @@ -512,8 +543,8 @@ def snp_bubble_structure(request, linear_structure): if not (w_hdns == snp_hdns == {3: 2}): print(w_hdns, snp_hdns) print(HDN_L, HDN_R) - print(wildtype_sequence[HDN_L.pos + K + 1]) - print(snp_sequence[HDN_L.pos + K + 1]) + print(wildtype_sequence[HDN_L.pos + ksize + 1]) + print(snp_sequence[HDN_L.pos + ksize + 1]) request.applymarker(pytest.mark.xfail) return graph, wildtype_sequence, snp_sequence, HDN_L, HDN_R diff --git a/tests/test_assembly.py b/tests/test_assembly.py index 86bfbf639f..ae58aeadaf 100644 --- a/tests/test_assembly.py +++ b/tests/test_assembly.py @@ -50,8 +50,7 @@ import pytest import screed -from .graph_features import * -from .graph_features import K +from .graph_structure_fixtures import * def teardown(): @@ -61,64 +60,64 @@ def teardown(): @pytest.mark.parametrize("assembler", [LinearAssembler, CompactingAssembler]) class TestNonBranching: - def test_all_start_positions(self, linear_structure, assembler): + def test_all_start_positions(self, ksize, linear_structure, assembler): # assemble entire contig, starting from wherever graph, contig = linear_structure asm = assembler(graph) for start in range(0, len(contig), 150): - path = asm.assemble(contig[start:start + K]) + path = asm.assemble(contig[start:start + ksize]) assert utils._equals_rc(path, contig), start - def test_all_left_to_beginning(self, linear_structure, assembler): + def test_all_left_to_beginning(self, ksize, linear_structure, assembler): # assemble directed left graph, contig = linear_structure asm = assembler(graph) for start in range(0, len(contig), 150): - path = asm.assemble_left(contig[start:start + K]) + path = asm.assemble_left(contig[start:start + ksize]) print(path, ', ', contig[:start]) - assert utils._equals_rc(path, contig[:start + K]), start + assert utils._equals_rc(path, contig[:start + ksize]), start - def test_all_right_to_end(self, linear_structure, assembler): + def test_all_right_to_end(self, ksize, linear_structure, assembler): # assemble directed right graph, contig = linear_structure asm = assembler(graph) for start in range(0, len(contig), 150): - path = asm.assemble_right(contig[start:start + K]) + path = asm.assemble_right(contig[start:start + ksize]) print(path, ', ', contig[:start]) assert utils._equals_rc(path, contig[start:]), start - def test_circular(self, circular_linear_structure, assembler): + def test_circular(self, ksize, circular_linear_structure, assembler): graph, contig = circular_linear_structure asm = assembler(graph) - path = asm.assemble_right(contig[:K]) + path = asm.assemble_right(contig[:ksize]) print(path, ',', contig) assert utils._equals_rc(path, contig[:len(path)]) - def test_hash_as_seed(self, linear_structure, assembler): + def test_hash_as_seed(self, ksize, linear_structure, assembler): graph, contig = linear_structure asm = assembler(graph) - left = graph.hash(contig[:K]) + left = graph.hash(contig[:ksize]) assert utils._equals_rc(asm.assemble(left), contig) class TestCompactingAssembler: - def test_beginning_to_branch_right(self, right_tip_structure): + def test_beginning_to_branch_right(self, ksize, right_tip_structure): # assemble from beginning of contig, up until branch point graph, contig, L, HDN, R, tip = right_tip_structure asm = CompactingAssembler(graph) - path = asm.assemble(contig[0:K]) + path = asm.assemble(contig[0:ksize]) - assert len(path) == HDN.pos + K + assert len(path) == HDN.pos + ksize assert utils._equals_rc(path, contig[:len(path)]) - def test_end_to_branch_right(self, right_tip_structure): + def test_end_to_branch_right(self, ksize, right_tip_structure): # in the LinearAsembler, this would continue all the way # to the beginning. The CompactingAssembler does an extra # check of the node degree in the reverse direction. @@ -128,45 +127,45 @@ def test_end_to_branch_right(self, right_tip_structure): class TestLinearAssembler_RightBranching: - def test_branch_point(self, right_tip_structure): + def test_branch_point(self, ksize, right_tip_structure): graph, contig, L, HDN, R, tip = right_tip_structure assert graph.kmer_degree(HDN) == 3 - def test_beginning_to_branch(self, right_tip_structure): + def test_beginning_to_branch(self, ksize, right_tip_structure): # assemble from beginning of contig, up until branch point graph, contig, L, HDN, R, tip = right_tip_structure asm = khmer.LinearAssembler(graph) - def test_beginning_to_branch_revcomp(self, right_tip_structure): + def test_beginning_to_branch_revcomp(self, ksize, right_tip_structure): # assemble from beginning of contig, up until branch point # starting from rev comp graph, contig, L, HDN, R, tip = right_tip_structure asm = khmer.LinearAssembler(graph) - path = asm.assemble(revcomp(contig[0:K])) + path = asm.assemble(revcomp(contig[0:ksize])) - assert len(path) == HDN.pos + K + assert len(path) == HDN.pos + ksize assert utils._equals_rc(path, contig[:len(path)]) - def test_left_of_branch_to_beginning(self, right_tip_structure): + def test_left_of_branch_to_beginning(self, ksize, right_tip_structure): # start from HDN (left of branch) graph, contig, L, HDN, R, tip = right_tip_structure asm = khmer.LinearAssembler(graph) path = asm.assemble(L) - assert len(path) == HDN.pos + K + assert len(path) == HDN.pos + ksize assert utils._equals_rc(path, contig[:len(path)]) - def test_left_of_branch_to_beginning_revcomp(self, right_tip_structure): + def test_left_of_branch_to_beginning_revcomp(self, ksize, right_tip_structure): # start from revcomp of HDN (left of branch) graph, contig, L, HDN, R, tip = right_tip_structure asm = khmer.LinearAssembler(graph) path = asm.assemble(revcomp(L)) - assert len(path) == HDN.pos + K + assert len(path) == HDN.pos + ksize assert utils._equals_rc(path, contig[:len(path)]) - def test_right_of_branch_outwards_to_ends(self, right_tip_structure): + def test_right_of_branch_outwards_to_ends(self, ksize, right_tip_structure): # assemble from right of branch point (at R) # Should get the *entire* original contig, as the assembler # will move left relative to the branch, and not consider it @@ -178,11 +177,11 @@ def test_right_of_branch_outwards_to_ends(self, right_tip_structure): assert len(path) == len(contig) assert utils._equals_rc(path, contig) - def test_end_to_beginning(self, right_tip_structure): + def test_end_to_beginning(self, ksize, right_tip_structure): # should have exact same behavior as right_of_branch_outwards graph, contig, L, HDN, R, tip = right_tip_structure asm = khmer.LinearAssembler(graph) - path = asm.assemble(contig[-K:]) + path = asm.assemble(contig[-ksize:]) assert len(path) == len(contig) assert utils._equals_rc(path, contig) @@ -190,22 +189,22 @@ def test_end_to_beginning(self, right_tip_structure): class TestLinearAssembler_LeftBranching: - def test_branch_point(self, left_tip_structure): + def test_branch_point(self, ksize, left_tip_structure): graph, contig, L, HDN, R, tip = left_tip_structure assert graph.kmer_degree(HDN) == 3 - def test_end_to_branch(self, left_tip_structure): + def test_end_to_branch(self, ksize, left_tip_structure): # assemble from end until branch point # should include HDN graph, contig, L, HDN, R, tip = left_tip_structure asm = khmer.LinearAssembler(graph) - path = asm.assemble(contig[-K:]) + path = asm.assemble(contig[-ksize:]) assert len(path) == len(contig) - HDN.pos assert utils._equals_rc(path, contig[HDN.pos:]) - def test_branch_to_end(self, left_tip_structure): + def test_branch_to_end(self, ksize, left_tip_structure): # assemble from branch point until end graph, contig, L, HDN, R, tip = left_tip_structure asm = khmer.LinearAssembler(graph) @@ -214,12 +213,12 @@ def test_branch_to_end(self, left_tip_structure): assert len(path) == len(contig) - HDN.pos assert utils._equals_rc(path, contig[HDN.pos:]) - def test_from_branch_to_ends_with_stopbf(self, left_tip_structure): + def test_from_branch_to_ends_with_stopbf(self, ksize, left_tip_structure): # block the tip with the stop_filter. should return a full length # contig. graph, contig, L, HDN, R, tip = left_tip_structure - stop_filter = khmer.Nodegraph(K, 1e5, 4) + stop_filter = khmer.Nodegraph(ksize, 1e5, 4) stop_filter.count(tip) asm = khmer.LinearAssembler(graph, stop_filter=stop_filter) @@ -229,12 +228,12 @@ def test_from_branch_to_ends_with_stopbf(self, left_tip_structure): assert len(path) == len(contig) assert utils._equals_rc(path, contig) - def test_from_branch_to_ends_with_stopbf_revcomp(self, left_tip_structure): + def test_from_branch_to_ends_with_stopbf_revcomp(self, ksize, left_tip_structure): # block the tip with the stop_filter. should return a full length # contig. graph, contig, L, HDN, R, tip = left_tip_structure - stop_filter = khmer.Nodegraph(K, 1e5, 4) + stop_filter = khmer.Nodegraph(ksize, 1e5, 4) stop_filter.count(tip) asm = khmer.LinearAssembler(graph, stop_filter=stop_filter) @@ -243,47 +242,47 @@ def test_from_branch_to_ends_with_stopbf_revcomp(self, left_tip_structure): assert len(path) == len(contig) assert utils._equals_rc(path, contig) - def test_end_thru_tip_with_stopbf(self, left_tip_structure): + def test_end_thru_tip_with_stopbf(self, ksize, left_tip_structure): # assemble up to branch point, and include introduced branch b/c # of stop bf graph, contig, L, HDN, R, tip = left_tip_structure - stop_filter = khmer.Nodegraph(K, 1e5, 4) + stop_filter = khmer.Nodegraph(ksize, 1e5, 4) stop_filter.count(L) # ...and block original path asm = khmer.LinearAssembler(graph, stop_filter=stop_filter) - path = asm.assemble(contig[-K:]) + path = asm.assemble(contig[-ksize:]) assert len(path) == len(contig) - HDN.pos + 1 # should be the tip k-kmer, plus the last base of the HDN thru # the end of the contig - assert utils._equals_rc(path, tip + contig[HDN.pos + K - 1:]) + assert utils._equals_rc(path, tip + contig[HDN.pos + ksize - 1:]) - def test_single_node_flanked_by_hdns(self, left_tip_structure): + def test_single_node_flanked_by_hdns(self, ksize, left_tip_structure): # assemble single node flanked by high-degree nodes # we'll copy the main nodegraph before mutating it graph, contig, L, HDN, R, tip = left_tip_structure asm = khmer.LinearAssembler(graph) - graph.consume(mutate_position(contig, HDN.pos + K)) + graph.consume(mutate_position(contig, HDN.pos + ksize)) path = asm.assemble(HDN) - assert len(path) == K + assert len(path) == ksize assert utils._equals_rc(path, HDN) class TestLabeledAssembler: - def test_hash_as_seed(self, linear_structure): + def test_hash_as_seed(self, ksize, linear_structure): graph, contig = linear_structure lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) - left = graph.hash(contig[:K]) + left = graph.hash(contig[:ksize]) assert utils._equals_rc(asm.assemble(left).pop(), contig) - def test_beginning_to_end_across_tip(self, right_tip_structure): + def test_beginning_to_end_across_tip(self, ksize, right_tip_structure): # assemble entire contig, ignoring branch point b/c of labels graph, contig, L, HDN, R, tip = right_tip_structure lh = khmer.GraphLabels(graph) @@ -292,7 +291,7 @@ def test_beginning_to_end_across_tip(self, right_tip_structure): # L, HDN, and R will be labeled with 1 lh.label_across_high_degree_nodes(contig, hdn, 1) - path = asm.assemble(contig[:K]) + path = asm.assemble(contig[:ksize]) assert len(path) == 1, "there should only be one path" path = path[0] # @CTB @@ -300,7 +299,7 @@ def test_beginning_to_end_across_tip(self, right_tip_structure): assert len(path) == len(contig) assert utils._equals_rc(path, contig) - def test_assemble_right_double_fork(self, right_double_fork_structure): + def test_assemble_right_double_fork(self, ksize, right_double_fork_structure): # assemble two contigs from a double forked structure graph, contig, L, HDN, R, branch = right_double_fork_structure lh = khmer.GraphLabels(graph) @@ -313,7 +312,7 @@ def test_assemble_right_double_fork(self, right_double_fork_structure): lh.label_across_high_degree_nodes(branch, hdn, 2) print(lh.get_tag_labels(list(hdn)[0])) - paths = asm.assemble(contig[:K]) + paths = asm.assemble(contig[:ksize]) print('Path lengths', [len(x) for x in paths]) assert len(paths) == 2 @@ -321,7 +320,7 @@ def test_assemble_right_double_fork(self, right_double_fork_structure): assert any(utils._equals_rc(path, contig) for path in paths) assert any(utils._equals_rc(path, branch) for path in paths) - def test_assemble_right_triple_fork(self, right_triple_fork_structure): + def test_assemble_right_triple_fork(self, ksize, right_triple_fork_structure): # assemble three contigs from a trip fork (graph, contig, L, HDN, R, top_sequence, bottom_sequence) = right_triple_fork_structure @@ -337,7 +336,7 @@ def test_assemble_right_triple_fork(self, right_triple_fork_structure): lh.label_across_high_degree_nodes(bottom_sequence, hdn, 3) print(lh.get_tag_labels(list(hdn)[0])) - paths = asm.assemble(contig[:K]) + paths = asm.assemble(contig[:ksize]) print([len(x) for x in paths]) assert len(paths) == 3 @@ -346,14 +345,14 @@ def test_assemble_right_triple_fork(self, right_triple_fork_structure): assert any(utils._equals_rc(path, top_sequence) for path in paths) assert any(utils._equals_rc(path, bottom_sequence) for path in paths) - def test_assemble_left_double_fork(self, left_double_fork_structure): + def test_assemble_left_double_fork(self, ksize, left_double_fork_structure): # assemble entire contig + branch points b/c of labels; start from end graph, contig, L, HDN, R, branch = left_double_fork_structure lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) # first try without the labels - paths = asm.assemble(contig[-K:]) + paths = asm.assemble(contig[-ksize:]) assert len(paths) == 1 # without labels, should get the beginning of the HDN thru the end @@ -367,14 +366,14 @@ def test_assemble_left_double_fork(self, left_double_fork_structure): lh.label_across_high_degree_nodes(branch, hdn, 2) print(lh.get_tag_labels(list(hdn)[0])) - paths = asm.assemble(contig[-K:]) + paths = asm.assemble(contig[-ksize:]) assert len(paths) == 2 assert any(utils._equals_rc(path, contig) for path in paths) assert any(utils._equals_rc(path, branch) for path in paths) - def test_assemble_snp_bubble_single(self, snp_bubble_structure): + def test_assemble_snp_bubble_single(self, ksize, snp_bubble_structure): # assemble entire contig + one of two paths through a bubble graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure lh = khmer.GraphLabels(graph) @@ -384,12 +383,12 @@ def test_assemble_snp_bubble_single(self, snp_bubble_structure): assert len(hdn) == 2 lh.label_across_high_degree_nodes(wildtype, hdn, 1) - paths = asm.assemble(wildtype[:K]) + paths = asm.assemble(wildtype[:ksize]) assert len(paths) == 1 assert utils._equals_rc(paths[0], wildtype) - def test_assemble_snp_bubble_both(self, snp_bubble_structure): + def test_assemble_snp_bubble_both(self, ksize, snp_bubble_structure): # assemble entire contig + both paths graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure lh = khmer.GraphLabels(graph) @@ -401,23 +400,23 @@ def test_assemble_snp_bubble_both(self, snp_bubble_structure): lh.label_across_high_degree_nodes(wildtype, hdn, 1) lh.label_across_high_degree_nodes(mutant, hdn, 2) - paths = asm.assemble(wildtype[:K]) + paths = asm.assemble(wildtype[:ksize]) assert len(paths) == 2 assert any(utils._contains_rc(wildtype, path) for path in paths) assert any(utils._contains_rc(mutant, path) for path in paths) - # assert all(path[:HDN_L.pos+K][-K:] == HDN_L for path in paths) - # assert all(path[HDN_R.pos:][:K] == HDN_R for path in paths) - # assert paths[0][:HDN_L.pos+K] == paths[1][:HDN_L.pos+K] + # assert all(path[:HDN_L.pos+ksize][-ksize:] == HDN_L for path in paths) + # assert all(path[HDN_R.pos:][:ksize] == HDN_R for path in paths) + # assert paths[0][:HDN_L.pos+ksize] == paths[1][:HDN_L.pos+ksize] # assert paths[0][HDN_R.pos:] == paths[1][HDN_R.pos:] - def test_assemble_snp_bubble_stopbf(self, snp_bubble_structure): + def test_assemble_snp_bubble_stopbf(self, ksize, snp_bubble_structure): # assemble one side of bubble, blocked with stop_filter, # when labels on both branches # stop_filter should trip a filter failure, negating the label spanning graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure - stop_filter = khmer.Nodegraph(K, 1e5, 4) + stop_filter = khmer.Nodegraph(ksize, 1e5, 4) lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh, stop_filter=stop_filter) @@ -428,29 +427,29 @@ def test_assemble_snp_bubble_stopbf(self, snp_bubble_structure): lh.label_across_high_degree_nodes(mutant, hdn, 2) # do the labeling, but block the mutant with stop_filter - stop_filter.count(mutant[HDN_L.pos + 1:HDN_L.pos + K + 1]) - paths = asm.assemble(wildtype[:K]) + stop_filter.count(mutant[HDN_L.pos + 1:HDN_L.pos + ksize + 1]) + paths = asm.assemble(wildtype[:ksize]) assert len(paths) == 1 assert any(utils._equals_rc(path, wildtype) for path in paths) # @pytest.mark.skip(reason='destroys your computer and then the world') - def test_assemble_tandem_repeats(self, tandem_repeat_structure): + def test_assemble_tandem_repeats(self, ksize, tandem_repeat_structure): # assemble one copy of a tandem repeat graph, repeat, tandem_repeats = tandem_repeat_structure lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) - paths = asm.assemble(repeat[:K]) + paths = asm.assemble(repeat[:ksize]) assert len(paths) == 1 - # There are K-1 k-mers spanning the junction between + # There are ksize-1 k-mers spanning the junction between # the beginning and end of the repeat - assert len(paths[0]) == len(repeat) + K - 1 + assert len(paths[0]) == len(repeat) + ksize - 1 class TestJunctionCountAssembler: - def test_beginning_to_end_across_tip(self, right_tip_structure): + def test_beginning_to_end_across_tip(self, ksize, right_tip_structure): # assemble entire contig, ignoring branch point b/c of labels graph, contig, L, HDN, R, tip = right_tip_structure asm = khmer.JunctionCountAssembler(graph) @@ -458,7 +457,7 @@ def test_beginning_to_end_across_tip(self, right_tip_structure): asm.consume(contig) asm.consume(contig) - path = asm.assemble(contig[:K]) + path = asm.assemble(contig[:ksize]) print('P:', path[0]) print('T:', tip) print('C:', contig) diff --git a/tests/test_cython_partitioning.py b/tests/test_cython_partitioning.py index 15c4d0b23e..aea4d975af 100644 --- a/tests/test_cython_partitioning.py +++ b/tests/test_cython_partitioning.py @@ -11,7 +11,7 @@ from khmer import reverse_complement as revcomp from khmer import reverse_hash as revhash from . import khmer_tst_utils as utils -from .graph_features import * +from .graph_stucture_fixtures import * import pytest import screed From 2b7487c4724e7148cdb930fa495e13105fcd3f20 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 15 Nov 2017 01:38:17 -0800 Subject: [PATCH 157/185] fix K value in hdn_counts --- tests/conftest.py | 4 ++-- tests/graph_structure_fixtures.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index eb859cb21c..1e7bf3f2a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,12 +3,12 @@ def pytest_generate_tests(metafunc): if 'ksize' in metafunc.fixturenames: - ksize = getattr(metafunc.function, 'ksize', None) + ksize = getattr(metafunc.function, '_ksize', None) if ksize is None: ksize = [21] if isinstance(ksize, int): ksize = [ksize] - metafunc.parametrize('ksize', ksize, indirect=True, + metafunc.parametrize('ksize', ksize, ids=lambda k: 'K={0}'.format(k)) diff --git a/tests/graph_structure_fixtures.py b/tests/graph_structure_fixtures.py index d2a8b9261b..26ce716c73 100644 --- a/tests/graph_structure_fixtures.py +++ b/tests/graph_structure_fixtures.py @@ -54,15 +54,17 @@ def set_ksize(K=21): def wrap(func): - setattr(func, 'ksize', K) + setattr(func, '_ksize', K) return func return wrap - +''' @pytest.fixture def ksize(request): print('ksize:', request.param) return request.param +''' + def test_ksize(ksize): assert ksize == 21 @@ -73,9 +75,10 @@ def test_ksize_override(ksize): assert ksize == 31 -@set_ksize([21, 25, 29]) +@set_ksize([25, 29]) def test_ksize_override_param(ksize): - assert ksize in [21, 25, 29] + print('ksize is', ksize) + assert ksize in [25, 29] @pytest.fixture(params=[2, -2], ids=['Start', 'End']) @@ -175,7 +178,7 @@ def reads(sequence, ksize, L=100, N=100, dbg_cover=False): yield sequence[start:start + L] -def kmers(sequence, K=21): +def kmers(sequence, K): for i in range(len(sequence) - K + 1): yield sequence[i:i + K] @@ -187,6 +190,7 @@ def test_kmers(ksize): assert res[0] == 'A' * ksize assert res[-1] == ('A' * (ksize - 1)) + 'T' + def test_mutate_sequence(): for _ in range(100): assert 'A' not in mutate_sequence('A' * 10, 10) @@ -263,7 +267,7 @@ def graph(request, ksize): num_kmers = 50000 des_fp = 0.00001 args = optimal_fp(num_kmers, des_fp) - print('Graph Params:', args) + print('Graph Params:', args,'K =', ksize) return request.param(ksize, args.htable_size, args.num_htables) @@ -273,7 +277,7 @@ def hdn_counts(sequence, graph): ''' hdns = {} - for kmer in kmers(sequence): + for kmer in kmers(sequence, graph.ksize()): d = graph.kmer_degree(kmer) if d > 2: hdns[d] = hdns.get(d, 0) + 1 From 8486a406580a0953643113523f21fe0e30f11d96 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 15 Nov 2017 01:40:29 -0800 Subject: [PATCH 158/185] Change set_ksize to using_ksize --- tests/graph_structure_fixtures.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/graph_structure_fixtures.py b/tests/graph_structure_fixtures.py index 26ce716c73..dc558b579d 100644 --- a/tests/graph_structure_fixtures.py +++ b/tests/graph_structure_fixtures.py @@ -52,7 +52,7 @@ # We just define this globally rather than in a module-level fixture, # as we need it during parameterization and whatnot. -def set_ksize(K=21): +def using_ksize(K=21): def wrap(func): setattr(func, '_ksize', K) return func @@ -70,12 +70,12 @@ def test_ksize(ksize): assert ksize == 21 -@set_ksize(31) +@using_ksize(31) def test_ksize_override(ksize): assert ksize == 31 -@set_ksize([25, 29]) +@using_ksize([25, 29]) def test_ksize_override_param(ksize): print('ksize is', ksize) assert ksize in [25, 29] @@ -183,7 +183,7 @@ def kmers(sequence, K): yield sequence[i:i + K] -@set_ksize([5, 7]) +@using_ksize([5, 7]) def test_kmers(ksize): S = 'A' * ksize + 'T' res = list(kmers(S, ksize)) From 9a02638e9e98ecd4d8d34b78f45792baec2f7a6f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 15 Nov 2017 01:52:29 -0800 Subject: [PATCH 159/185] Update partitioning tests with ksize fixture, reduce test case sizes --- tests/test_cython_partitioning.py | 125 ++++++++++++++++-------------- 1 file changed, 66 insertions(+), 59 deletions(-) diff --git a/tests/test_cython_partitioning.py b/tests/test_cython_partitioning.py index aea4d975af..fda2983f72 100644 --- a/tests/test_cython_partitioning.py +++ b/tests/test_cython_partitioning.py @@ -11,7 +11,7 @@ from khmer import reverse_complement as revcomp from khmer import reverse_hash as revhash from . import khmer_tst_utils as utils -from .graph_stucture_fixtures import * +from .graph_structure_fixtures import * import pytest import screed @@ -46,20 +46,20 @@ def teardown_method(self, method): # real memory leaks. gc.collect() - def test_one_component(self, known_sequence): + def test_one_component(self, ksize, known_sequence): inpath = utils.get_test_data('random-20-a.fa') - cg = khmer.Countgraph(K, 1e5, 4) + cg = khmer.Countgraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(known_sequence) assert sp.n_components == 1 - def test_two_components(self, random_sequence): + def test_two_components(self, ksize, random_sequence): comp1 = random_sequence() comp2 = random_sequence(exclude=comp1) - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(comp1) @@ -68,11 +68,11 @@ def test_two_components(self, random_sequence): sp.consume(comp2) assert sp.n_components == 2 - def test_components_iter(self, random_sequence): + def test_components_iter(self, ksize, random_sequence): comp1 = random_sequence() comp2 = random_sequence(exclude=comp1) - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(comp1) @@ -82,22 +82,22 @@ def test_components_iter(self, random_sequence): comps = list(sp.components()) assert len(comps) == 2 - def test_component_n_tags(self, random_sequence): + def test_component_n_tags(self, ksize, random_sequence): seq = random_sequence() - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(seq) tags = [t for t,c in sp.tag_components()] - comp = sp.find_nearest_component(seq[:K]) + comp = sp.find_nearest_component(seq[:ksize]) assert len(tags) == len(comp) - def test_tag_components_iter(self, random_sequence): + def test_tag_components_iter(self, ksize, random_sequence): comp1 = random_sequence() comp2 = random_sequence(exclude=comp1) - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(comp1) @@ -114,33 +114,33 @@ def test_tag_components_iter(self, random_sequence): assert len(comps) == 2 assert len(tags) == sum([len(c) for c in comps]) - def test_find_nearest_component(self, random_sequence): + def test_find_nearest_component(self, ksize, random_sequence): seq1 = random_sequence() seq2 = random_sequence(exclude=seq1) - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(seq1) sp.consume(seq2) - c1 = sp.find_nearest_component(seq1[:K]) - c2 = sp.find_nearest_component(seq2[:K]) + c1 = sp.find_nearest_component(seq1[:ksize]) + c2 = sp.find_nearest_component(seq2[:ksize]) assert c1.component_id != c2.component_id for tag in c1: - assert utils._contains_rc(seq1, revhash(tag, K)) - assert not utils._contains_rc(seq2, revhash(tag, K)) + assert utils._contains_rc(seq1, revhash(tag, ksize)) + assert not utils._contains_rc(seq2, revhash(tag, ksize)) for tag in c2: - assert utils._contains_rc(seq2, revhash(tag, K)) - assert not utils._contains_rc(seq1, revhash(tag, K)) + assert utils._contains_rc(seq2, revhash(tag, ksize)) + assert not utils._contains_rc(seq1, revhash(tag, ksize)) - def test_merge_components(self, random_sequence): + def test_merge_components(self, ksize, random_sequence): seq1 = random_sequence() seq2 = random_sequence(exclude=seq1) - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(seq1) @@ -154,12 +154,12 @@ def test_merge_components(self, random_sequence): assert len(comps) == 1 - def test_multi_merge_components(self, random_sequence): + def test_multi_merge_components(self, ksize, random_sequence): seq1 = random_sequence() seq2 = random_sequence(exclude=seq1) seq3 = random_sequence(exclude=seq1+seq2) - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(seq1) @@ -170,69 +170,72 @@ def test_multi_merge_components(self, random_sequence): sp.consume(seq1 + seq2 + seq3) assert sp.n_components == 1 - def test_nomerge_k_minus_2_overlap(self, single_component, random_sequence): + def test_nomerge_k_minus_2_overlap(self, ksize, single_component, + random_sequence): '''Test that components are not merged when they have a length K-2 overlap. ''' graph, partitioner, seq = single_component asm = khmer.LinearAssembler(graph) - first = seq[:K-2] + first = seq[:ksize-2] neighbor = random_sequence(exclude=seq) + first assert partitioner.n_components == 1 partitioner.consume(neighbor) - print(seq, neighbor, asm.assemble(seq[:K]), sep='\n') + print(seq, neighbor, asm.assemble(seq[:ksize]), sep='\n') assert partitioner.n_components == 2 @pytest.mark.parametrize("where", ["beginning", "end"]) - def test_merge_k_minus_1_overlap(self, single_component, random_sequence, - where): + def test_merge_k_minus_1_overlap(self, single_component, ksize, + random_sequence, where): '''Test that components are merged when they have a length K-1 overlap. ''' graph, partitioner, seq = single_component asm = khmer.LinearAssembler(graph) if where == "beginning": - overlap = seq[:K-1] + overlap = seq[:ksize-1] neighbor = random_sequence(exclude=seq) + overlap else: - overlap = seq[-K+1:] + overlap = seq[-ksize+1:] neighbor = overlap + random_sequence(exclude=seq) assert partitioner.n_components == 1 partitioner.consume(neighbor) - path = asm.assemble(seq[:K]) + path = asm.assemble(seq[:ksize]) assert partitioner.n_components == 1 - def test_merge_k_overlap(self, single_component, random_sequence): + def test_merge_k_overlap(self, single_component, + random_sequence, ksize): '''Test that components are merged when they have a length K overlap. ''' graph, partitioner, seq = single_component asm = khmer.LinearAssembler(graph) - first = seq[:K] + first = seq[:ksize] neighbor = random_sequence(exclude=seq) + first assert partitioner.n_components == 1 partitioner.consume(neighbor) - print(seq, neighbor, asm.assemble(seq[:K]), sep='\n') + print(seq, neighbor, asm.assemble(seq[:ksize]), sep='\n') assert partitioner.n_components == 1 - @pytest.mark.parametrize("n_reads", list(range(100, 1001, 100))) - def test_one_component_from_reads(self, random_sequence, n_reads): + @pytest.mark.parametrize("n_reads", [100, 500, 1000]) + def test_one_component_from_reads(self, random_sequence, ksize, n_reads): seq = random_sequence() - seq_reads = list(reads(seq, dbg_cover=True, N=n_reads)) + seq_reads = list(reads(seq, ksize, dbg_cover=True, N=n_reads)) - G = khmer.Nodegraph(K, 1e6, 4) + G = khmer.Nodegraph(ksize, 1e6, 4) sp = StreamingPartitioner(G) for read in seq_reads: sp.consume(read) assert sp.n_components == 1 - @pytest.mark.parametrize("n_components", list(range(1, 10))) - def test_streaming_multicomponents(self, random_sequence, n_components): + @pytest.mark.parametrize("n_components", [3, 5, 10]) + def test_streaming_multicomponents(self, random_sequence, + ksize, n_components): '''Test with many components from reads, and check for memory leaks.''' seqs = [] for _ in range(n_components): @@ -240,14 +243,14 @@ def test_streaming_multicomponents(self, random_sequence, n_components): seq_reads = [] for seq in seqs: - seq_reads.extend(list(reads(seq, dbg_cover=True, N=100))) + seq_reads.extend(list(reads(seq, ksize, dbg_cover=True, N=100))) random.shuffle(seq_reads) - G = khmer.Nodegraph(K, 1e6, 4) + G = khmer.Nodegraph(ksize, 1e6, 4) sp = StreamingPartitioner(G) for read in seq_reads: - assert len(read) >= K + assert len(read) >= ksize sp.consume(read) assert sp.n_components == n_components @@ -257,14 +260,15 @@ def test_streaming_multicomponents(self, random_sequence, n_components): #assert sp.n_components == (comp._n_created - comp._n_destroyed) assert sp.n_consumed == len(seq_reads) - @pytest.mark.parametrize("n_components", list(range(1,101, 20))) + @pytest.mark.parametrize("n_components", [3, 5, 10]) @pytest.mark.parametrize("cov", [1,10,20]) - def test_write_components(self, random_sequence, cov, n_components, tmpdir): + def test_write_components(self, random_sequence, cov, + ksize, n_components, tmpdir): outfn = tmpdir.join('counts.csv') seqs = [] for _ in range(n_components): seqs.append(random_sequence(exclude=''.join(seqs))) - G = khmer.Countgraph(K, 1e6, 4) + G = khmer.Countgraph(ksize, 1e6, 4) sp = StreamingPartitioner(G) for seq in seqs: @@ -281,14 +285,15 @@ def test_write_components(self, random_sequence, cov, n_components, tmpdir): for row in results: assert abs(float(row[2])-float(cov)) < 2 - @pytest.mark.parametrize("n_components", [1, 10, 50, 100]) - def test_save_partitioner(self, random_sequence, n_components, tmpdir): + @pytest.mark.parametrize("n_components", [1, 3, 5, 10]) + def test_save_partitioner(self, random_sequence, ksize, + n_components, tmpdir): import json out_prefix = str(tmpdir.join('test_save')) seqs = [] for _ in range(n_components): seqs.append(random_sequence(exclude=''.join(seqs))) - G = khmer.Countgraph(K, 1e6, 4) + G = khmer.Countgraph(ksize, 1e6, 4) sp = StreamingPartitioner(G) for seq in seqs: sp.consume(seq) @@ -308,14 +313,16 @@ def test_save_partitioner(self, random_sequence, n_components, tmpdir): for comp in sp.components(): assert comp.component_id in result_comps - @pytest.mark.parametrize("n_components", [1, 10, 50, 100]) - def test_load_partitioner(self, random_sequence, n_components, tmpdir): + @pytest.mark.xfail + @pytest.mark.parametrize("n_components", [1, 3, 5, 10]) + def test_load_partitioner(self, random_sequence, ksize, + n_components, tmpdir): import json out_prefix = str(tmpdir.join('test_save')) seqs = [] for _ in range(n_components): seqs.append(random_sequence(exclude=''.join(seqs))) - G = khmer.Countgraph(K, 1e6, 4) + G = khmer.Countgraph(ksize, 1e6, 4) sp = StreamingPartitioner(G) for seq in seqs: sp.consume(seq) @@ -342,21 +349,21 @@ def teardown_method(self, method): # real memory leaks. gc.collect() - def test_one_paired_component(self, random_sequence): + def test_one_paired_component(self, ksize, random_sequence): first = random_sequence() second = random_sequence(exclude=first) - cg = khmer.Countgraph(K, 1e5, 4) + cg = khmer.Countgraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume_pair(first, second) assert sp.n_components == 1 - def test_two_paired_components_merge(self, random_sequence): + def test_two_paired_components_merge(self, ksize, random_sequence): comp1 = random_sequence() comp2 = random_sequence(exclude=comp1) - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(comp1) @@ -368,12 +375,12 @@ def test_two_paired_components_merge(self, random_sequence): sp.consume_pair(comp1, comp2) assert sp.n_components == 1 - def test_multi_paired_components_merge(self, random_sequence): + def test_multi_paired_components_merge(self, ksize, random_sequence): seq1 = random_sequence() seq2 = random_sequence(exclude=seq1) seq3 = random_sequence(exclude=seq1+seq2) - cg = khmer.Nodegraph(K, 1e5, 4) + cg = khmer.Nodegraph(ksize, 1e5, 4) sp = StreamingPartitioner(cg) sp.consume(seq1) From 03de927c2ef1da86de852320275fe6e3a54a2c8c Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 15 Nov 2017 02:00:27 -0800 Subject: [PATCH 160/185] basic triple fork test --- tests/test_linked_dbg.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_linked_dbg.py b/tests/test_linked_dbg.py index 2436720d31..dfe4325fe6 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_linked_dbg.py @@ -7,7 +7,7 @@ from khmer import forward_hash from . import khmer_tst_utils as utils from .khmer_tst_utils import _equals_rc, _contains_rc -from .graph_features import * +from .graph_structure_fixtures import * from khmer._oxli.graphlinks import StreamingCompactor from khmer import Nodegraph @@ -48,7 +48,8 @@ def compare_tip_with_cdbg(rts, compactor): # out_edge.sequence[K-1:]) -def test_compact_tip(right_tip_structure): +@using_ksize([21,25,31]) +def test_compact_tip(ksize, right_tip_structure): '''Should have no links. Need two junctions. ''' graph, contig, L, HDN, R, tip = right_tip_structure @@ -153,7 +154,19 @@ def test_compact_tip_x_merge(left_tip_structure, right_tip_structure): assert compactor.n_edges == 5 -def test_flanking_hdns(): +@using_ksize([21, 31]) +def test_compact_triple_fork(right_triple_fork_structure): + graph, core, L, HDN, R, top, bottom = right_triple_fork_structure + + compactor = StreamingCompactor(graph) + compactor.update(core) + compactor.report() + + assert compactor.n_nodes == 1 + assert compactor.n_edges == 4 + + +def test_compact_trivial_edge(): pass From 74a32c0f50543f1b45f9b4e9fb3dd1edc83aa69a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Wed, 15 Nov 2017 18:55:49 -0800 Subject: [PATCH 161/185] Functionalize the graph structure fixtures --- tests/graph_structure_fixtures.py | 368 +++++++++--------- tests/test_assembly.py | 62 +-- ...test_linked_dbg.py => test_compact_dbg.py} | 8 + 3 files changed, 231 insertions(+), 207 deletions(-) rename tests/{test_linked_dbg.py => test_compact_dbg.py} (93%) diff --git a/tests/graph_structure_fixtures.py b/tests/graph_structure_fixtures.py index dc558b579d..3c7919b8ac 100644 --- a/tests/graph_structure_fixtures.py +++ b/tests/graph_structure_fixtures.py @@ -58,13 +58,6 @@ def wrap(func): return func return wrap -''' -@pytest.fixture -def ksize(request): - print('ksize:', request.param) - return request.param -''' - def test_ksize(ksize): assert ksize == 21 @@ -271,7 +264,6 @@ def graph(request, ksize): return request.param(ksize, args.htable_size, args.num_htables) - def hdn_counts(sequence, graph): '''Get the degree distribution of nodes with degree more than 2. ''' @@ -292,15 +284,17 @@ def linear_structure(request, graph, ksize, random_sequence): sequence [0]→o→o~~o→o→[-1] ''' - sequence = random_sequence() - graph.consume(sequence) + def get(): + sequence = random_sequence() + graph.consume(sequence) - # Check for false positive neighbors in our graph - # Mark as an expected failure if any are found - if hdn_counts(sequence, graph): - request.applymarker(pytest.mark.xfail) + # Check for false positive neighbors in our graph + # Mark as an expected failure if any are found + if hdn_counts(sequence, graph): + request.applymarker(pytest.mark.xfail) - return graph, sequence + return graph, sequence + return get @pytest.fixture @@ -314,28 +308,30 @@ def right_tip_structure(request, graph, ksize, flank_coords, random_sequence): Where S is the start position of the high degreen node (HDN). That is, it has a single branch at the Sth K-mer. ''' - sequence = random_sequence() - S = flank_coords - if S < 0: - S = len(sequence) + S - # the HDN - HDN = Kmer(sequence[S:S + ksize], pos=S) - # left of the HDN - L = Kmer(sequence[S - 1:S - 1 + ksize], pos=S - 1) - # right of the HDN - R = Kmer(sequence[S + 1:S + 1 + ksize], pos=S + 1) - # the branch kmer - tip = Kmer(mutate_position(R, -1), - pos=R.pos) - - graph.consume(sequence) - graph.count(tip) - - # Check for false positive neighbors and mark as expected failure if found - if hdn_counts(sequence, graph) != {3: 1}: - request.applymarker(pytest.mark.xfail) - - return graph, sequence, L, HDN, R, tip + def get(): + sequence = random_sequence() + S = flank_coords + if S < 0: + S = len(sequence) + S + # the HDN + HDN = Kmer(sequence[S:S + ksize], pos=S) + # left of the HDN + L = Kmer(sequence[S - 1:S - 1 + ksize], pos=S - 1) + # right of the HDN + R = Kmer(sequence[S + 1:S + 1 + ksize], pos=S + 1) + # the branch kmer + tip = Kmer(mutate_position(R, -1), + pos=R.pos) + + graph.consume(sequence) + graph.count(tip) + + # Check for false positive neighbors and mark as expected failure if found + if hdn_counts(sequence, graph) != {3: 1}: + request.applymarker(pytest.mark.xfail) + + return graph, sequence, L, HDN, R, tip + return get @pytest.fixture @@ -352,39 +348,41 @@ def right_double_fork_structure(request, ksize, flank_coords, and B is the mutated base starting the branch. ''' - graph, core_sequence = linear_structure - print('\nCore Len:', len(core_sequence)) - branch_sequence = random_sequence(exclude=core_sequence) - print('Branch len:', len(branch_sequence)) - - # start position of the HDN - S = flank_coords - if S < 0: - S = len(core_sequence) + S - # the HDN - HDN = Kmer(core_sequence[S:S + ksize], pos=S) - # left of the HDN - L = Kmer(core_sequence[S - 1:S - 1 + ksize], pos=S - 1) - # right of the HDN - R = Kmer(core_sequence[S + 1:S + 1 + ksize], pos=S + 1) - # the branch sequence, mutated at position S+1 - branch_start = core_sequence[:R.pos] + mutate_position(R, -1) - branch_sequence = branch_start + branch_sequence - - graph.consume(core_sequence) - graph.consume(branch_sequence) - - # Check for false positive neighbors and mark as expected failure if found - core_hdns = hdn_counts(core_sequence, graph) - branch_hdns = hdn_counts(branch_sequence, graph) - - # the core and branch sequences should each have exactly - # ONE node of degree 3 (HDN) - if core_hdns != {3: 1} or branch_hdns != {3: 1}: - print(core_hdns, branch_hdns) - request.applymarker(pytest.mark.xfail) - - return graph, core_sequence, L, HDN, R, branch_sequence + def get(): + graph, core_sequence = linear_structure() + print('\nCore Len:', len(core_sequence)) + branch_sequence = random_sequence(exclude=core_sequence) + print('Branch len:', len(branch_sequence)) + + # start position of the HDN + S = flank_coords + if S < 0: + S = len(core_sequence) + S + # the HDN + HDN = Kmer(core_sequence[S:S + ksize], pos=S) + # left of the HDN + L = Kmer(core_sequence[S - 1:S - 1 + ksize], pos=S - 1) + # right of the HDN + R = Kmer(core_sequence[S + 1:S + 1 + ksize], pos=S + 1) + # the branch sequence, mutated at position S+1 + branch_start = core_sequence[:R.pos] + mutate_position(R, -1) + branch_sequence = branch_start + branch_sequence + + graph.consume(core_sequence) + graph.consume(branch_sequence) + + # Check for false positive neighbors and mark as expected failure if found + core_hdns = hdn_counts(core_sequence, graph) + branch_hdns = hdn_counts(branch_sequence, graph) + + # the core and branch sequences should each have exactly + # ONE node of degree 3 (HDN) + if core_hdns != {3: 1} or branch_hdns != {3: 1}: + print(core_hdns, branch_hdns) + request.applymarker(pytest.mark.xfail) + + return graph, core_sequence, L, HDN, R, branch_sequence + return get @pytest.fixture @@ -403,32 +401,35 @@ def right_triple_fork_structure(request, right_double_fork_structure, Where S is the start position of the high degreen node (HDN). ''' - - graph, core_sequence, L, HDN, R, top_sequence = right_double_fork_structure - bottom_branch = random_sequence(exclude=core_sequence + top_sequence) - print(len(core_sequence), len(top_sequence), len(bottom_branch)) - - # the branch sequence, mutated at position S+1 - # choose a base not already represented at that position - bases = {'A', 'C', 'G', 'T'} - mutated = random.choice(list(bases - {R[-1], top_sequence[R.pos + ksize - 1]})) - - bottom_sequence = core_sequence[:HDN.pos + ksize] + mutated + bottom_branch - - graph.consume(bottom_sequence) - - # Check for false positive neighbors and mark as expected failure if found - core_hdns = hdn_counts(core_sequence, graph) - top_hdns = hdn_counts(top_sequence, graph) - bottom_hdns = hdn_counts(bottom_sequence, graph) - - # the core, top, and bottom sequences should each have exactly - # ONE node of degree 4 (HDN) - if not (core_hdns == top_hdns == bottom_hdns == {4: 1}): - print(core_hdns, top_hdns, bottom_hdns) - request.applymarker(pytest.mark.xfail) - - return graph, core_sequence, L, HDN, R, top_sequence, bottom_sequence + + def get(): + graph, core_sequence, L, HDN, R, top_sequence = \ + right_double_fork_structure() + bottom_branch = random_sequence(exclude=core_sequence + top_sequence) + print(len(core_sequence), len(top_sequence), len(bottom_branch)) + + # the branch sequence, mutated at position S+1 + # choose a base not already represented at that position + bases = {'A', 'C', 'G', 'T'} + mutated = random.choice(list(bases - {R[-1], top_sequence[R.pos + ksize - 1]})) + + bottom_sequence = core_sequence[:HDN.pos + ksize] + mutated + bottom_branch + + graph.consume(bottom_sequence) + + # Check for false positive neighbors and mark as expected failure if found + core_hdns = hdn_counts(core_sequence, graph) + top_hdns = hdn_counts(top_sequence, graph) + bottom_hdns = hdn_counts(bottom_sequence, graph) + + # the core, top, and bottom sequences should each have exactly + # ONE node of degree 4 (HDN) + if not (core_hdns == top_hdns == bottom_hdns == {4: 1}): + print(core_hdns, top_hdns, bottom_hdns) + request.applymarker(pytest.mark.xfail) + + return graph, core_sequence, L, HDN, R, top_sequence, bottom_sequence + return get @pytest.fixture @@ -443,24 +444,26 @@ def left_tip_structure(request, graph, ksize, flank_coords, random_sequence): Where S is the start position of the HDN. ''' - sequence = random_sequence() - S = flank_coords - if S < 0: - S = len(sequence) + S - tip = Kmer(mutate_position(sequence[S - 1:S - 1 + ksize], 0), - pos=S - 1 + ksize) - HDN = Kmer(sequence[S:S + ksize], pos=S) - L = Kmer(sequence[S - 1:S - 1 + ksize], pos=S - 1) - R = Kmer(sequence[S + 1:S + 1 + ksize], pos=S + 1) - - graph.consume(sequence) - graph.count(tip) - - # Check for false positive neighbors and mark as expected failure if found - if hdn_counts(sequence, graph) != {3: 1}: - request.applymarker(pytest.mark.xfail) - - return graph, sequence, L, HDN, R, tip + def get(): + sequence = random_sequence() + S = flank_coords + if S < 0: + S = len(sequence) + S + tip = Kmer(mutate_position(sequence[S - 1:S - 1 + ksize], 0), + pos=S - 1 + ksize) + HDN = Kmer(sequence[S:S + ksize], pos=S) + L = Kmer(sequence[S - 1:S - 1 + ksize], pos=S - 1) + R = Kmer(sequence[S + 1:S + 1 + ksize], pos=S + 1) + + graph.consume(sequence) + graph.count(tip) + + # Check for false positive neighbors and mark as expected failure if found + if hdn_counts(sequence, graph) != {3: 1}: + request.applymarker(pytest.mark.xfail) + + return graph, sequence, L, HDN, R, tip + return get @pytest.fixture @@ -476,38 +479,40 @@ def left_double_fork_structure(request, linear_structure, ksize, Where S is the start position of the high degreen node (HDN). ''' - graph, core_sequence = linear_structure - branch_sequence = random_sequence(exclude=core_sequence) - - # start position of the HDN - S = flank_coords - if S < 0: - S = len(core_sequence) + S - # the HDN - HDN = Kmer(core_sequence[S:S + ksize], pos=S) - # left of the HDN - L = Kmer(core_sequence[S - 1:S - 1 + ksize], pos=S - 1) - # right of the HDN - R = Kmer(core_sequence[S + 1:S + 1 + ksize], pos=S + 1) - # the branch sequence, mutated at position 0 in L, - # whih is equivalent to the K-1 prefix of HDN prepended with a new base - branch_start = mutate_position(L, 0) - branch_sequence = branch_sequence + \ - branch_start + core_sequence[L.pos + ksize:] - - graph.consume(core_sequence) - graph.consume(branch_sequence) - - # Check for false positive neighbors and mark as expected failure if found - core_hdns = hdn_counts(core_sequence, graph) - branch_hdns = hdn_counts(branch_sequence, graph) - - # the core and branch sequences should each have exactly - # ONE node of degree 3 (HDN) - if not (core_hdns == branch_hdns == {3: 1}): - request.applymarker(pytest.mark.xfail) - - return graph, core_sequence, L, HDN, R, branch_sequence + def get(): + graph, core_sequence = linear_structure() + branch_sequence = random_sequence(exclude=core_sequence) + + # start position of the HDN + S = flank_coords + if S < 0: + S = len(core_sequence) + S + # the HDN + HDN = Kmer(core_sequence[S:S + ksize], pos=S) + # left of the HDN + L = Kmer(core_sequence[S - 1:S - 1 + ksize], pos=S - 1) + # right of the HDN + R = Kmer(core_sequence[S + 1:S + 1 + ksize], pos=S + 1) + # the branch sequence, mutated at position 0 in L, + # whih is equivalent to the K-1 prefix of HDN prepended with a new base + branch_start = mutate_position(L, 0) + branch_sequence = branch_sequence + \ + branch_start + core_sequence[L.pos + ksize:] + + graph.consume(core_sequence) + graph.consume(branch_sequence) + + # Check for false positive neighbors and mark as expected failure if found + core_hdns = hdn_counts(core_sequence, graph) + branch_hdns = hdn_counts(branch_sequence, graph) + + # the core and branch sequences should each have exactly + # ONE node of degree 3 (HDN) + if not (core_hdns == branch_hdns == {3: 1}): + request.applymarker(pytest.mark.xfail) + + return graph, core_sequence, L, HDN, R, branch_sequence + return get @pytest.fixture @@ -531,50 +536,61 @@ def snp_bubble_structure(request, linear_structure, ksize): so we bring the rightmost SNP a tad left. ''' - graph, wildtype_sequence = linear_structure - S = int(len(wildtype_sequence) / 2) - snp_sequence = mutate_position(wildtype_sequence, S + ksize) - HDN_L = Kmer(wildtype_sequence[S:S + ksize], pos=S) - HDN_R = Kmer(wildtype_sequence[S + ksize + 1:S + 2 * ksize + 1], pos=S + - ksize + 1) - - graph.consume(wildtype_sequence) - graph.consume(snp_sequence) + def get(): + graph, wildtype_sequence = linear_structure() + S = int(len(wildtype_sequence) / 2) + snp_sequence = mutate_position(wildtype_sequence, S + ksize) + HDN_L = Kmer(wildtype_sequence[S:S + ksize], pos=S) + HDN_R = Kmer(wildtype_sequence[S + ksize + 1:S + 2 * ksize + 1], pos=S + + ksize + 1) + + graph.consume(wildtype_sequence) + graph.consume(snp_sequence) + + # Check for false positive neighbors and mark as expected failure if found + w_hdns = hdn_counts(wildtype_sequence, graph) + snp_hdns = hdn_counts(snp_sequence, graph) + if not (w_hdns == snp_hdns == {3: 2}): + print(w_hdns, snp_hdns) + print(HDN_L, HDN_R) + print(wildtype_sequence[HDN_L.pos + ksize + 1]) + print(snp_sequence[HDN_L.pos + ksize + 1]) + request.applymarker(pytest.mark.xfail) + + return graph, wildtype_sequence, snp_sequence, HDN_L, HDN_R + return get - # Check for false positive neighbors and mark as expected failure if found - w_hdns = hdn_counts(wildtype_sequence, graph) - snp_hdns = hdn_counts(snp_sequence, graph) - if not (w_hdns == snp_hdns == {3: 2}): - print(w_hdns, snp_hdns) - print(HDN_L, HDN_R) - print(wildtype_sequence[HDN_L.pos + ksize + 1]) - print(snp_sequence[HDN_L.pos + ksize + 1]) - request.applymarker(pytest.mark.xfail) - return graph, wildtype_sequence, snp_sequence, HDN_L, HDN_R +@pytest.fixture +def tandem_forks(request, left_fork_structure, right_fork_structure): + pass @pytest.fixture(params=[2, 3, 4, 5, 6, 7, 8]) def tandem_repeat_structure(request, linear_structure): - graph, sequence = linear_structure + def get(): + graph, sequence = linear_structure() - tandem_repeats = sequence * request.param - graph.consume(tandem_repeats) + tandem_repeats = sequence * request.param + graph.consume(tandem_repeats) - if hdn_counts(tandem_repeats, graph): - request.applymarker(pytest.mark.xfail) + if hdn_counts(tandem_repeats, graph): + request.applymarker(pytest.mark.xfail) - return graph, sequence, tandem_repeats + return graph, sequence, tandem_repeats + return get @pytest.fixture def circular_linear_structure(request, linear_structure): - graph, sequence = linear_structure + def get(): + graph, sequence = linear_structure() - sequence += sequence + sequence += sequence - if hdn_counts(sequence, graph): - request.applymarker(pytest.mark.xfail) + if hdn_counts(sequence, graph): + request.applymarker(pytest.mark.xfail) - return graph, sequence + return graph, sequence + return get diff --git a/tests/test_assembly.py b/tests/test_assembly.py index ae58aeadaf..0ee050f183 100644 --- a/tests/test_assembly.py +++ b/tests/test_assembly.py @@ -62,7 +62,7 @@ class TestNonBranching: def test_all_start_positions(self, ksize, linear_structure, assembler): # assemble entire contig, starting from wherever - graph, contig = linear_structure + graph, contig = linear_structure() asm = assembler(graph) for start in range(0, len(contig), 150): @@ -71,7 +71,7 @@ def test_all_start_positions(self, ksize, linear_structure, assembler): def test_all_left_to_beginning(self, ksize, linear_structure, assembler): # assemble directed left - graph, contig = linear_structure + graph, contig = linear_structure() asm = assembler(graph) for start in range(0, len(contig), 150): @@ -81,7 +81,7 @@ def test_all_left_to_beginning(self, ksize, linear_structure, assembler): def test_all_right_to_end(self, ksize, linear_structure, assembler): # assemble directed right - graph, contig = linear_structure + graph, contig = linear_structure() asm = assembler(graph) for start in range(0, len(contig), 150): @@ -91,7 +91,7 @@ def test_all_right_to_end(self, ksize, linear_structure, assembler): def test_circular(self, ksize, circular_linear_structure, assembler): - graph, contig = circular_linear_structure + graph, contig = circular_linear_structure() asm = assembler(graph) path = asm.assemble_right(contig[:ksize]) @@ -99,7 +99,7 @@ def test_circular(self, ksize, circular_linear_structure, assembler): assert utils._equals_rc(path, contig[:len(path)]) def test_hash_as_seed(self, ksize, linear_structure, assembler): - graph, contig = linear_structure + graph, contig = linear_structure() asm = assembler(graph) left = graph.hash(contig[:ksize]) @@ -110,7 +110,7 @@ class TestCompactingAssembler: def test_beginning_to_branch_right(self, ksize, right_tip_structure): # assemble from beginning of contig, up until branch point - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = CompactingAssembler(graph) path = asm.assemble(contig[0:ksize]) @@ -121,26 +121,26 @@ def test_end_to_branch_right(self, ksize, right_tip_structure): # in the LinearAsembler, this would continue all the way # to the beginning. The CompactingAssembler does an extra # check of the node degree in the reverse direction. - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = CompactingAssembler(graph) class TestLinearAssembler_RightBranching: def test_branch_point(self, ksize, right_tip_structure): - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() assert graph.kmer_degree(HDN) == 3 def test_beginning_to_branch(self, ksize, right_tip_structure): # assemble from beginning of contig, up until branch point - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.LinearAssembler(graph) def test_beginning_to_branch_revcomp(self, ksize, right_tip_structure): # assemble from beginning of contig, up until branch point # starting from rev comp - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.LinearAssembler(graph) path = asm.assemble(revcomp(contig[0:ksize])) @@ -149,7 +149,7 @@ def test_beginning_to_branch_revcomp(self, ksize, right_tip_structure): def test_left_of_branch_to_beginning(self, ksize, right_tip_structure): # start from HDN (left of branch) - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.LinearAssembler(graph) path = asm.assemble(L) @@ -158,7 +158,7 @@ def test_left_of_branch_to_beginning(self, ksize, right_tip_structure): def test_left_of_branch_to_beginning_revcomp(self, ksize, right_tip_structure): # start from revcomp of HDN (left of branch) - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.LinearAssembler(graph) path = asm.assemble(revcomp(L)) @@ -170,7 +170,7 @@ def test_right_of_branch_outwards_to_ends(self, ksize, right_tip_structure): # Should get the *entire* original contig, as the assembler # will move left relative to the branch, and not consider it # as a high degree node - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.LinearAssembler(graph) path = asm.assemble(R) @@ -179,7 +179,7 @@ def test_right_of_branch_outwards_to_ends(self, ksize, right_tip_structure): def test_end_to_beginning(self, ksize, right_tip_structure): # should have exact same behavior as right_of_branch_outwards - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.LinearAssembler(graph) path = asm.assemble(contig[-ksize:]) @@ -190,14 +190,14 @@ def test_end_to_beginning(self, ksize, right_tip_structure): class TestLinearAssembler_LeftBranching: def test_branch_point(self, ksize, left_tip_structure): - graph, contig, L, HDN, R, tip = left_tip_structure + graph, contig, L, HDN, R, tip = left_tip_structure() assert graph.kmer_degree(HDN) == 3 def test_end_to_branch(self, ksize, left_tip_structure): # assemble from end until branch point # should include HDN - graph, contig, L, HDN, R, tip = left_tip_structure + graph, contig, L, HDN, R, tip = left_tip_structure() asm = khmer.LinearAssembler(graph) path = asm.assemble(contig[-ksize:]) @@ -206,7 +206,7 @@ def test_end_to_branch(self, ksize, left_tip_structure): def test_branch_to_end(self, ksize, left_tip_structure): # assemble from branch point until end - graph, contig, L, HDN, R, tip = left_tip_structure + graph, contig, L, HDN, R, tip = left_tip_structure() asm = khmer.LinearAssembler(graph) path = asm.assemble(HDN) @@ -216,7 +216,7 @@ def test_branch_to_end(self, ksize, left_tip_structure): def test_from_branch_to_ends_with_stopbf(self, ksize, left_tip_structure): # block the tip with the stop_filter. should return a full length # contig. - graph, contig, L, HDN, R, tip = left_tip_structure + graph, contig, L, HDN, R, tip = left_tip_structure() stop_filter = khmer.Nodegraph(ksize, 1e5, 4) stop_filter.count(tip) @@ -231,7 +231,7 @@ def test_from_branch_to_ends_with_stopbf(self, ksize, left_tip_structure): def test_from_branch_to_ends_with_stopbf_revcomp(self, ksize, left_tip_structure): # block the tip with the stop_filter. should return a full length # contig. - graph, contig, L, HDN, R, tip = left_tip_structure + graph, contig, L, HDN, R, tip = left_tip_structure() stop_filter = khmer.Nodegraph(ksize, 1e5, 4) stop_filter.count(tip) @@ -245,7 +245,7 @@ def test_from_branch_to_ends_with_stopbf_revcomp(self, ksize, left_tip_structure def test_end_thru_tip_with_stopbf(self, ksize, left_tip_structure): # assemble up to branch point, and include introduced branch b/c # of stop bf - graph, contig, L, HDN, R, tip = left_tip_structure + graph, contig, L, HDN, R, tip = left_tip_structure() stop_filter = khmer.Nodegraph(ksize, 1e5, 4) stop_filter.count(L) # ...and block original path @@ -261,7 +261,7 @@ def test_end_thru_tip_with_stopbf(self, ksize, left_tip_structure): def test_single_node_flanked_by_hdns(self, ksize, left_tip_structure): # assemble single node flanked by high-degree nodes # we'll copy the main nodegraph before mutating it - graph, contig, L, HDN, R, tip = left_tip_structure + graph, contig, L, HDN, R, tip = left_tip_structure() asm = khmer.LinearAssembler(graph) graph.consume(mutate_position(contig, HDN.pos + ksize)) @@ -275,7 +275,7 @@ def test_single_node_flanked_by_hdns(self, ksize, left_tip_structure): class TestLabeledAssembler: def test_hash_as_seed(self, ksize, linear_structure): - graph, contig = linear_structure + graph, contig = linear_structure() lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) @@ -284,7 +284,7 @@ def test_hash_as_seed(self, ksize, linear_structure): def test_beginning_to_end_across_tip(self, ksize, right_tip_structure): # assemble entire contig, ignoring branch point b/c of labels - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) hdn = graph.find_high_degree_nodes(contig) @@ -301,7 +301,7 @@ def test_beginning_to_end_across_tip(self, ksize, right_tip_structure): def test_assemble_right_double_fork(self, ksize, right_double_fork_structure): # assemble two contigs from a double forked structure - graph, contig, L, HDN, R, branch = right_double_fork_structure + graph, contig, L, HDN, R, branch = right_double_fork_structure() lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) @@ -323,7 +323,7 @@ def test_assemble_right_double_fork(self, ksize, right_double_fork_structure): def test_assemble_right_triple_fork(self, ksize, right_triple_fork_structure): # assemble three contigs from a trip fork (graph, contig, L, HDN, R, - top_sequence, bottom_sequence) = right_triple_fork_structure + top_sequence, bottom_sequence) = right_triple_fork_structure() lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) @@ -347,7 +347,7 @@ def test_assemble_right_triple_fork(self, ksize, right_triple_fork_structure): def test_assemble_left_double_fork(self, ksize, left_double_fork_structure): # assemble entire contig + branch points b/c of labels; start from end - graph, contig, L, HDN, R, branch = left_double_fork_structure + graph, contig, L, HDN, R, branch = left_double_fork_structure() lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) @@ -375,7 +375,7 @@ def test_assemble_left_double_fork(self, ksize, left_double_fork_structure): def test_assemble_snp_bubble_single(self, ksize, snp_bubble_structure): # assemble entire contig + one of two paths through a bubble - graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure + graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure() lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) @@ -390,7 +390,7 @@ def test_assemble_snp_bubble_single(self, ksize, snp_bubble_structure): def test_assemble_snp_bubble_both(self, ksize, snp_bubble_structure): # assemble entire contig + both paths - graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure + graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure() lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) @@ -415,7 +415,7 @@ def test_assemble_snp_bubble_stopbf(self, ksize, snp_bubble_structure): # assemble one side of bubble, blocked with stop_filter, # when labels on both branches # stop_filter should trip a filter failure, negating the label spanning - graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure + graph, wildtype, mutant, HDN_L, HDN_R = snp_bubble_structure() stop_filter = khmer.Nodegraph(ksize, 1e5, 4) lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh, stop_filter=stop_filter) @@ -436,7 +436,7 @@ def test_assemble_snp_bubble_stopbf(self, ksize, snp_bubble_structure): # @pytest.mark.skip(reason='destroys your computer and then the world') def test_assemble_tandem_repeats(self, ksize, tandem_repeat_structure): # assemble one copy of a tandem repeat - graph, repeat, tandem_repeats = tandem_repeat_structure + graph, repeat, tandem_repeats = tandem_repeat_structure() lh = khmer.GraphLabels(graph) asm = khmer.SimpleLabeledAssembler(lh) paths = asm.assemble(repeat[:ksize]) @@ -451,7 +451,7 @@ class TestJunctionCountAssembler: def test_beginning_to_end_across_tip(self, ksize, right_tip_structure): # assemble entire contig, ignoring branch point b/c of labels - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.JunctionCountAssembler(graph) asm.consume(contig) asm.consume(contig) diff --git a/tests/test_linked_dbg.py b/tests/test_compact_dbg.py similarity index 93% rename from tests/test_linked_dbg.py rename to tests/test_compact_dbg.py index dfe4325fe6..d7039a36ef 100644 --- a/tests/test_linked_dbg.py +++ b/tests/test_compact_dbg.py @@ -52,6 +52,7 @@ def compare_tip_with_cdbg(rts, compactor): def test_compact_tip(ksize, right_tip_structure): '''Should have no links. Need two junctions. ''' + right_tip_structure = right_tip_structure() graph, contig, L, HDN, R, tip = right_tip_structure compactor = StreamingCompactor(graph) @@ -83,6 +84,7 @@ def test_compact_tip(ksize, right_tip_structure): def test_compact_tip_double_update(right_tip_structure): + right_tip_structure = right_tip_structure() graph, contig, L, HDN, R, tip = right_tip_structure compactor = StreamingCompactor(graph) @@ -97,6 +99,7 @@ def test_compact_tip_double_update(right_tip_structure): def test_compact_tip_revcomp_update(right_tip_structure): + right_tip_structure = right_tip_structure() graph, contig, L, HDN, R, tip = right_tip_structure compactor = StreamingCompactor(graph) @@ -112,7 +115,9 @@ def test_compact_tip_revcomp_update(right_tip_structure): def test_compact_two_tip_islands(left_tip_structure, right_tip_structure): + right_tip_structure = right_tip_structure() graph, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure + left_tip_structure = left_tip_structure() _, contig_l, L_l, HDN_l, R_l, tip_l = left_tip_structure compactor = StreamingCompactor(graph) @@ -130,7 +135,9 @@ def test_compact_two_tip_islands(left_tip_structure, right_tip_structure): def test_compact_tip_x_merge(left_tip_structure, right_tip_structure): + right_tip_structure = right_tip_structure() graph, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure + left_tip_structure = left_tip_structure() _, contig_l, L_l, HDN_l, R_l, tip_l = left_tip_structure contig_merge = contig_l + contig_r @@ -156,6 +163,7 @@ def test_compact_tip_x_merge(left_tip_structure, right_tip_structure): @using_ksize([21, 31]) def test_compact_triple_fork(right_triple_fork_structure): + right_triple_fork_structure = right_triple_fork_structure() graph, core, L, HDN, R, top, bottom = right_triple_fork_structure compactor = StreamingCompactor(graph) From 42f837dbdc342d966d0dbf56c0dc617eb4c0a74f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 16 Nov 2017 14:42:27 -0800 Subject: [PATCH 162/185] add tandem hdn fixture --- tests/graph_structure_fixtures.py | 42 +++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/tests/graph_structure_fixtures.py b/tests/graph_structure_fixtures.py index 3c7919b8ac..785dacd4c4 100644 --- a/tests/graph_structure_fixtures.py +++ b/tests/graph_structure_fixtures.py @@ -562,8 +562,46 @@ def get(): @pytest.fixture -def tandem_forks(request, left_fork_structure, right_fork_structure): - pass +def tandem_triple_forks(request, right_triple_fork_structure, + random_sequence, ksize, flank_coords): + + def get(): + rtfs = right_triple_fork_structure() + graph, core, L, HDN, R, top_l, bottom_l = rtfs + S_l = flank_coords + if S_l < 0: + S_l = len(core) + S_l + S_r = S_l + 1 + + # top sequence for new HDN + top_r = random_sequence() + new_HDN = R + new_R = Kmer(core[S_r + 1:S_r + 1 + ksize], pos=S_r+1) + top_r_start = core[:new_R.pos] + mutate_position(new_R, -1) + top_r = top_r_start + top_r + + graph.consume(top_r) + + # now the bottom sequence for new HDN + bases = {'A', 'C', 'G', 'T'} + mutated = random.choice(list(bases - {new_R[-1], top_r[new_R.pos + ksize - 1]})) + bottom_r = random_sequence() + bottom_r = core[:new_HDN.pos + ksize] + mutated + bottom_r + + graph.consume(bottom_r) + + exp_2_hdns = [hdn_counts(s, graph) for s in (top_r, bottom_r, core)] + exp_1_hdn = [hdn_counts(s, graph) for s in (top_l, bottom_l)] + + if not all(map(lambda c: c == {4:2}, exp_2_hdns)) or \ + not all(map(lambda c: c == {4:1}, exp_1_hdn)): + + print(exp_2_hdns, exp_1_hdns) + request.applymarker(pytest.mark.xfail) + + return graph, core, L, HDN, new_HDN, new_R, top_l, bottom_l, top_r, bottom_r + + return get @pytest.fixture(params=[2, 3, 4, 5, 6, 7, 8]) From 2a070c523ba6af0c3b5cbb3c8037f34c2c2d477f Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 16 Nov 2017 14:42:38 -0800 Subject: [PATCH 163/185] addtrivial edge test --- tests/test_compact_dbg.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/test_compact_dbg.py b/tests/test_compact_dbg.py index d7039a36ef..b357b3ec85 100644 --- a/tests/test_compact_dbg.py +++ b/tests/test_compact_dbg.py @@ -174,9 +174,42 @@ def test_compact_triple_fork(right_triple_fork_structure): assert compactor.n_edges == 4 -def test_compact_trivial_edge(): - pass +@pytest.mark.parametrize('random_sequence', [100, 200], indirect=True) +def test_compact_trivial_edge(tandem_triple_forks, ksize): + ttf = tandem_triple_forks() + graph, core, L, HDN_l, HDN_r, R, top_l, bottom_l, top_r, bottom_r = ttf + + print('Core:', core[HDN_l.pos:], '\nHDN_l:', HDN_l, '\nHDN_r:', HDN_r, + '\ntop_l:', top_l[HDN_l.pos:HDN_l.pos+2*ksize], + '\nbottom_l:', bottom_l[HDN_l.pos:HDN_l.pos+2*ksize], + '\ntop_r:', top_r[HDN_r.pos:HDN_r.pos+2*ksize], + '\nbottom_r:', bottom_r[HDN_r.pos:HDN_r.pos+2*ksize]) + br = '=' * 20 + graph.reset() + compactor = StreamingCompactor(graph) + print(br, 'ADD CORE', br) + compactor.consume_and_update(core) + assert compactor.n_nodes == 0 + print(br, 'ADD top_l', br) + compactor.consume_and_update(top_l) + assert compactor.n_nodes == 1 + assert compactor.n_edges == 3 + + print(br, 'ADD bottom_l', br) + compactor.consume_and_update(bottom_l) + assert compactor.n_nodes == 1 + assert compactor.n_edges == 4 + + print(br, 'ADD top_r', br) + compactor.consume_and_update(top_r) + assert compactor.n_nodes == 2 + assert compactor.n_edges == 6 + + print(br, 'ADD bottom_r', br) + compactor.consume_and_update(bottom_r) + assert compactor.n_nodes == 2 + assert compactor.n_edges == 7 def test_compact_tip_split_merge(): pass From efcf1f30110d0b94af9d8bdda983382e7b603a47 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 16 Nov 2017 14:48:05 -0800 Subject: [PATCH 164/185] fix trivial edge meta deduction, names --- include/oxli/links.hh | 81 +++++++++++++++++++++----------------- khmer/_oxli/graphlinks.pxd | 8 ++-- khmer/_oxli/graphlinks.pyx | 8 ++-- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 6ad48b3596..178ce2653b 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -55,6 +55,7 @@ Contact: khmer-project@idyll.org #include "kmer_filters.hh" #include "traversal.hh" #include "assembler.hh" +#include "alphabets.hh" # define DEBUG_LINKS @@ -85,22 +86,22 @@ typedef std::unordered_map HashIDMap; enum compact_edge_meta_t { - IS_FULL_EDGE, - IS_TIP, - IS_ISLAND, - IS_TRIVIAL + FULL, + TIP, + ISLAND, + TRIVIAL }; inline const char * edge_meta_repr(compact_edge_meta_t meta) { switch(meta) { - case IS_FULL_EDGE: - return "FULL EDGE"; - case IS_TIP: + case FULL: + return "FULL"; + case TIP: return "TIP"; - case IS_ISLAND: + case ISLAND: return "ISLAND"; - case IS_TRIVIAL: + case TRIVIAL: return "TRIVIAL"; } } @@ -119,7 +120,7 @@ public: UHashSet tags; CompactEdge(uint64_t in_node_id, uint64_t out_node_id) : - in_node_id(in_node_id), out_node_id(out_node_id), meta(IS_FULL_EDGE) {} + in_node_id(in_node_id), out_node_id(out_node_id), meta(FULL) {} CompactEdge(uint64_t in_node_id, uint64_t out_node_id, compact_edge_meta_t meta) : in_node_id(in_node_id), out_node_id(out_node_id), meta(meta) {} @@ -202,14 +203,15 @@ public: pdebug("new compact edge: \n left=" << std::to_string(left_id) << std::endl << " right=" << std::to_string(right_id) + << std::endl << " meta=" << edge_meta_repr(edge_meta) << std::endl << " sequence =" << edge_sequence << std::endl << " rc_sequence=" << _revcomp(edge_sequence) - << std::endl << " start =" << edge_sequence.substr(0, _ksize) - << std::endl << " rc_start=" << _revcomp(edge_sequence.substr(0, _ksize)) + << std::endl << " start =" << edge_sequence.substr(0, _ksize+1) + << std::endl << " rc_start=" << _revcomp(edge_sequence.substr(0, _ksize+1)) << std::endl << " end =" - << edge_sequence.substr(edge_sequence.length()-_ksize, _ksize) + << edge_sequence.substr(edge_sequence.length()-_ksize-1, _ksize+1) << std::endl << " rc_end =" - << _revcomp(edge_sequence.substr(edge_sequence.length()-_ksize, _ksize))); + << _revcomp(edge_sequence.substr(edge_sequence.length()-_ksize-1, _ksize+1))); edge->sequence = edge_sequence; n_compact_edges++; @@ -305,13 +307,14 @@ public: } bool delete_edge(CompactEdge* edge) { + bool deleted = false; if (delete_in_edge(edge)) { - return true; + deleted = true; } if (delete_out_edge(edge)) { - return true; + deleted = true; } - return false; + return deleted; } bool delete_in_edge(CompactEdge* edge) { @@ -391,17 +394,17 @@ public: std::string edges_repr() { std::ostringstream os; os << *this << std::endl << "\tin_edges:" << std::endl; - for (const char b : "ACGT") { + for (auto b : alphabets::DNA_SIMPLE) { CompactEdge* e = get_in_edge(b); if (e != nullptr) { os << "\t " << b << "=" << *e << std::endl; } } os << "\tout_edges:" << std::endl; - for (const char b : "ACGT") { + for (auto b : alphabets::DNA_SIMPLE) { CompactEdge* e = get_out_edge(b); if (e != nullptr) { - os << "\t -" << b << "=" << *e << std::endl; + os << "\t " << b << "=" << *e << std::endl; } } return os.str(); @@ -626,11 +629,11 @@ public: compact_edge_meta_t deduce_meta(CompactNode* in, CompactNode* out) { compact_edge_meta_t edge_meta; if (in == nullptr && out == nullptr) { - edge_meta = IS_ISLAND; + edge_meta = ISLAND; } else if ((out == nullptr) != (in == nullptr)) { - edge_meta = IS_TIP; + edge_meta = TIP; } else { - edge_meta = IS_FULL_EDGE; + edge_meta = FULL; } return edge_meta; } @@ -695,7 +698,7 @@ public: << sequence << " and other node ID=" << ((other_node != nullptr) ? other_node->node_id : NULL_ID)); bool edge_valid = true; - if (edge->meta == IS_TIP) { + if (edge->meta == TIP) { if (other_node != nullptr) { edge_valid = false; } @@ -704,7 +707,7 @@ public: edge->sequence.length() == sequence.length())) { edge_valid = false; } - } else if (edge->meta == IS_FULL_EDGE) { + } else if (edge->meta == FULL) { if (other_node == nullptr) { edge_valid = false; } else { @@ -744,14 +747,14 @@ public: char out_base = segment_seq[segment_seq.length()-_ksize+1]; switch(edge_meta) { - case IS_FULL_EDGE: + case FULL: // then we merged two tips pdebug("merge TIPs"); break; - case IS_TIP: + case TIP: pdebug("extend TIP"); break; - case IS_ISLAND: + case ISLAND: pdebug("created or extended ISLAND"); break; } @@ -835,6 +838,8 @@ public: lcursor.push_filter(edges.get_tag_stopper(tag_pair, found_tag)); std::string segment_seq = cassem._assemble_directed(lcursor) + root_back; + pdebug("assembled segment: " << segment_seq << " length: " << + segment_seq.length()); // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; @@ -843,12 +848,13 @@ public: CompactNode* left_node = nodes.get_node_by_kmer(lcursor.cursor); CompactEdge* left_out_edge = nullptr; if (left_node != nullptr) { + pdebug("found existing left node"); nodes.get_edge_from_right(left_node, left_out_edge, segment_seq); } // validate edge leaving root if it exists if (segment_edge != nullptr && left_out_edge != nullptr) { - + pdebug("found edges leaving root and left node"); if (segment_edge == left_out_edge && validate_segment(root_node, left_node, @@ -866,6 +872,7 @@ public: n_updates += nodes.unlink_edge(left_out_edge); edges.delete_edge(left_out_edge); } else if (segment_edge != nullptr) { + pdebug("found end leaving root node"); if (validate_segment(root_node, left_node, segment_edge, segment_seq)) { continue; @@ -887,11 +894,11 @@ public: // construct the compact edge compact_edge_meta_t edge_meta = (left_node == nullptr) - ? IS_TIP : IS_FULL_EDGE; - edge_meta = (segment_seq.length() == _ksize + 1) - ? IS_TRIVIAL : edge_meta; + ? TIP : FULL; + edge_meta = (segment_seq.length() == _ksize + 1 && edge_meta == FULL) + ? TRIVIAL : edge_meta; - if (edge_meta == IS_FULL_EDGE || edge_meta == IS_TRIVIAL) { + if (edge_meta == FULL || edge_meta == TRIVIAL) { // left side includes HDN, right side does not segment_edge = edges.build_edge(left_node->node_id, @@ -967,11 +974,11 @@ public: pdebug("segment sequence length=" << segment_seq.length()); compact_edge_meta_t edge_meta = (right_node == nullptr) ? - IS_TIP : IS_FULL_EDGE; - edge_meta = (segment_seq.length() == _ksize + 1) - ? IS_TRIVIAL : edge_meta; + TIP : FULL; + edge_meta = (segment_seq.length() == _ksize + 1 && edge_meta == FULL) + ? TRIVIAL : edge_meta; - if (edge_meta == IS_FULL_EDGE) { + if (edge_meta == FULL || edge_meta == TRIVIAL) { segment_edge = edges.build_edge(root_node->node_id, right_node->node_id, edge_meta, diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 5a993aa54d..609cbfab8d 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -23,10 +23,10 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: ctypedef umap[HashIntoType, uint64_t] HashIDMap ctypedef enum compact_edge_meta_t: - IS_FULL_EDGE - IS_TIP - IS_ISLAND - IS_TRIVIAL + FULL + TIP + ISLAND + TRIVIAL cdef const char * edge_meta_repr(compact_edge_meta_t) diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 02d37c7372..7014b5ab15 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -21,13 +21,13 @@ cdef class CompactEdge: @property def edge_type(self): cdef compact_edge_meta_t meta = deref(self._ce_this).meta - if meta == IS_FULL_EDGE: + if meta == FULL: return 'FULL' - elif meta == IS_TIP: + elif meta == TIP: return 'TIP' - elif meta == IS_ISLAND: + elif meta == ISLAND: return 'ISLAND' - elif meta == IS_TRIVIAL: + elif meta == TRIVIAL: return 'TRIVIAL' else: raise ValueError('Malformed edge metadata') From af0574488ff2de2fae4488f18b4f72df96d7f5a7 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 16 Nov 2017 17:25:22 -0800 Subject: [PATCH 165/185] Fix check for matching canonical orientations between edge sequence and root node after edge assembly --- include/oxli/links.hh | 48 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 178ce2653b..55627526a6 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -306,6 +306,10 @@ public: return lhs.node_id == rhs.node_id; } + std::string rc_sequence() const { + return _revcomp(sequence); + } + bool delete_edge(CompactEdge* edge) { bool deleted = false; if (delete_in_edge(edge)) { @@ -385,6 +389,7 @@ public: const CompactNode& node) { stream << ""; @@ -512,6 +517,14 @@ public: return n_deletes; } + bool is_rc_from_left(CompactNode* v, std::string& sequence) const { + const char * node_kmer = v->sequence.c_str(); + const char * _sequence = sequence.c_str(); + return strncmp(node_kmer, + _sequence + sequence.size()-_ksize+1, + _ksize - 1) != 0; + } + bool get_pivot_from_left(CompactNode* v, std::string& sequence, char& pivot_base) const { @@ -559,6 +572,17 @@ public: } } + bool is_rc_from_right(CompactNode* v, + std::string& sequence) const { + /* Check if sequence shared same canonical + * orientation with v from "right," assuming + * sequence does NOT include v + */ + const char * node_kmer = v->sequence.c_str(); + const char * _sequence = sequence.c_str(); + return strncmp(node_kmer+1, _sequence, _ksize-1) != 0; + } + bool get_pivot_from_right(CompactNode* v, std::string& sequence, char& pivot_base) const { @@ -815,13 +839,11 @@ public: KmerQueue neighbors; while(!induced_hdns.empty()) { Kmer root_kmer = *induced_hdns.begin(); - std::string root_kmer_seq = root_kmer.get_string_rep(_ksize); - char root_front = root_kmer_seq.front(); - char root_back = root_kmer_seq.back(); - induced_hdns.erase(root_kmer); CompactNode* root_node = nodes.get_node_by_kmer(root_kmer); + char root_front = root_node->sequence.front(); + char root_back = root_node->sequence.back(); pdebug("searching from induced HDN: " << root_node->edges_repr()); // check left (in) edges @@ -836,8 +858,12 @@ public: bool found_tag = false; lcursor.push_filter(edges.get_tag_stopper(tag_pair, found_tag)); - std::string segment_seq = cassem._assemble_directed(lcursor) - + root_back; + std::string segment_seq = cassem._assemble_directed(lcursor); + if (nodes.is_rc_from_left(root_node, segment_seq)) { + segment_seq = segment_seq + complement(root_front); + } else { + segment_seq = segment_seq + root_back; + } pdebug("assembled segment: " << segment_seq << " length: " << segment_seq.length()); @@ -848,7 +874,7 @@ public: CompactNode* left_node = nodes.get_node_by_kmer(lcursor.cursor); CompactEdge* left_out_edge = nullptr; if (left_node != nullptr) { - pdebug("found existing left node"); + pdebug("found existing left node: " << *left_node); nodes.get_edge_from_right(left_node, left_out_edge, segment_seq); } @@ -930,8 +956,12 @@ public: bool found_tag = false; rcursor.push_filter(edges.get_tag_stopper(tag_pair, found_tag)); - std::string segment_seq = root_front + cassem._assemble_directed(rcursor); - + std::string segment_seq = cassem._assemble_directed(rcursor); + if (nodes.is_rc_from_right(root_node, segment_seq)) { + segment_seq = complement(root_back) + segment_seq; + } else { + segment_seq = root_front + segment_seq; + } // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; nodes.get_edge_from_right(root_node, segment_edge, segment_seq); From 92df38bdbda67aecad5d01d5e61b207950606272 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 16 Nov 2017 17:28:37 -0800 Subject: [PATCH 166/185] Add some docs to the confusing orientation functions --- include/oxli/links.hh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 55627526a6..588812bff5 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -518,6 +518,10 @@ public: } bool is_rc_from_left(CompactNode* v, std::string& sequence) const { + /* Check if sequence shares same canonical orientation with + * v when coming from graph left, assuming sequence + * does NOT include v. + */ const char * node_kmer = v->sequence.c_str(); const char * _sequence = sequence.c_str(); return strncmp(node_kmer, @@ -528,6 +532,10 @@ public: bool get_pivot_from_left(CompactNode* v, std::string& sequence, char& pivot_base) const { + /* Check if sequence shared same canonical + * orientation with v from graph left, assuming + * sequence does NOT include v + */ const char * node_kmer = v->sequence.c_str(); const char * _segment = sequence.c_str(); pivot_base = _segment[sequence.size()-_ksize-1]; @@ -575,7 +583,7 @@ public: bool is_rc_from_right(CompactNode* v, std::string& sequence) const { /* Check if sequence shared same canonical - * orientation with v from "right," assuming + * orientation with v from graph right, assuming * sequence does NOT include v */ const char * node_kmer = v->sequence.c_str(); @@ -586,6 +594,10 @@ public: bool get_pivot_from_right(CompactNode* v, std::string& sequence, char& pivot_base) const { + /* Find the "pivot base" between sequence and v + * when sequence is from graph right, assuming + * v contained in sequence + */ const char * node_kmer = v->sequence.c_str(); const char * _segment = sequence.c_str(); pivot_base = _segment[_ksize]; From f55e367cc5aaf34559ec68a39de9dfd7abaadb2c Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 16 Nov 2017 18:01:18 -0800 Subject: [PATCH 167/185] Better trivial edge test granularity --- tests/test_compact_dbg.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_compact_dbg.py b/tests/test_compact_dbg.py index b357b3ec85..1a4a839ad4 100644 --- a/tests/test_compact_dbg.py +++ b/tests/test_compact_dbg.py @@ -211,5 +211,19 @@ def test_compact_trivial_edge(tandem_triple_forks, ksize): assert compactor.n_nodes == 2 assert compactor.n_edges == 7 + nodes = list(compactor.sequence_nodes(core)) + node_1, node_2 = nodes + trivial, node_2_out = list(node_2.in_edges()), list(node_2.out_edges()) + if len(trivial) != 1: + trivial, node_2_out = node_2_out, trivial + _, trivial = trivial[0] + + assert trivial.edge_type == 'TRIVIAL' + assert len(trivial) == ksize + 1 + + assert HDN_l in trivial.sequence + assert HDN_r in trivial.sequence + + def test_compact_tip_split_merge(): pass From d67927da6190b939f1efb7c29914c5dd8cdc032e Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 16 Nov 2017 18:05:12 -0800 Subject: [PATCH 168/185] expose CompactNode.degree --- khmer/_oxli/graphlinks.pyx | 4 ++++ tests/test_compact_dbg.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 7014b5ab15..74ce6c7791 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -81,6 +81,10 @@ cdef class CompactNode: def in_degree(self): return deref(self._cn_this).in_degree() + @property + def degree(self): + return deref(self._cn_this).degree() + @property def ID(self): return deref(self._cn_this).node_id diff --git a/tests/test_compact_dbg.py b/tests/test_compact_dbg.py index 1a4a839ad4..50f5b932a5 100644 --- a/tests/test_compact_dbg.py +++ b/tests/test_compact_dbg.py @@ -223,6 +223,8 @@ def test_compact_trivial_edge(tandem_triple_forks, ksize): assert HDN_l in trivial.sequence assert HDN_r in trivial.sequence + assert node_1.degree == 4 + assert node_2.degree == 4 def test_compact_tip_split_merge(): From 228e61cd1153d063195b3cd255aff27497cae553 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 17 Nov 2017 14:53:14 -0800 Subject: [PATCH 169/185] Add linear merge --- include/oxli/links.hh | 117 ++++++++++++++++++++++++------------- khmer/_oxli/graphlinks.pxd | 6 +- khmer/_oxli/graphlinks.pyx | 4 ++ tests/test_compact_dbg.py | 35 ++++++++++- 4 files changed, 118 insertions(+), 44 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 588812bff5..43fc086e80 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -178,6 +178,7 @@ class CompactEdgeFactory : public KmerFactory { protected: uint64_t n_compact_edges; + uint64_t _n_updates; uint32_t tag_density; TagEdgeMap tags_to_edges; @@ -186,7 +187,8 @@ protected: public: CompactEdgeFactory(WordLength K) : - KmerFactory(K), n_compact_edges(0) { + KmerFactory(K), n_compact_edges(0), + _n_updates(0) { tag_density = DEFAULT_TAG_DENSITY; } @@ -195,6 +197,10 @@ public: return n_compact_edges; } + uint64_t n_updates() const { + return _n_updates; + } + CompactEdge* build_edge(uint64_t left_id, uint64_t right_id, compact_edge_meta_t edge_meta, std::string edge_sequence) { @@ -215,6 +221,7 @@ public: edge->sequence = edge_sequence; n_compact_edges++; + _n_updates++; return edge; } @@ -227,6 +234,7 @@ public: } delete edge; n_compact_edges--; + _n_updates++; } } @@ -427,15 +435,20 @@ protected: // linear storage for CompactNodes CompactNodeVector compact_nodes; uint64_t n_compact_nodes; + uint64_t _n_updates; public: CompactNodeFactory(WordLength K) : - KmerFactory(K), n_compact_nodes(0) {} + KmerFactory(K), n_compact_nodes(0), + _n_updates(0) {} uint64_t n_nodes() const { return n_compact_nodes; } + uint64_t n_updates() const { + return _n_updates; + } // protected linear creation of CompactNode // they should never be deleted, so this is straightforward @@ -448,6 +461,7 @@ public: v = &(compact_nodes.back()); v->sequence = _revhash(hdn, _ksize); kmer_id_map[hdn] = v->node_id; + _n_updates++; //pdebug("Allocate: " << *v); } return v; @@ -499,22 +513,20 @@ public: return nodes; } - uint8_t unlink_edge(CompactEdge* edge) { + void unlink_edge(CompactEdge* edge) { pdebug("unlink edge " << *edge); CompactNode *left, *right; left = get_node_by_id(edge->in_node_id); right = get_node_by_id(edge->out_node_id); - uint8_t n_deletes = 0; if (left != nullptr) { // be lazy for now and use bidirectional delete left->delete_edge(edge); - n_deletes++; + _n_updates++; } if (right != nullptr) { right->delete_edge(edge); - n_deletes++; + _n_updates++; } - return n_deletes; } bool is_rc_from_left(CompactNode* v, std::string& sequence) const { @@ -551,17 +563,19 @@ public: } } - bool add_edge_from_left(CompactNode* v, CompactEdge* e) const { + bool add_edge_from_left(CompactNode* v, CompactEdge* e) { char pivot_base; if (!get_pivot_from_left(v, e->sequence, pivot_base)) { // same canonical orientation pdebug("add in edge " << *e << " to node " << *v << " from " << pivot_base); v->add_in_edge(pivot_base, e); + _n_updates++; return false; } else { // must have opposite canonical orientation pdebug("add out edge " << *e << " to node " << *v << " from " << pivot_base); v->add_out_edge(pivot_base, e); + _n_updates++; return true; } } @@ -611,15 +625,17 @@ public: } } - bool add_edge_from_right(CompactNode* v, CompactEdge* e) const { + bool add_edge_from_right(CompactNode* v, CompactEdge* e) { char pivot_base; if (!get_pivot_from_right(v, e->sequence, pivot_base)) { pdebug("add out edge " << *e << " to node " << *v << " from " << pivot_base); v->add_out_edge(pivot_base, e); + _n_updates++; return false; } else { pdebug("add in edge " << *e << " to node " << *v << " from " << pivot_base); v->add_in_edge(pivot_base, e); + _n_updates++; return true; } } @@ -662,7 +678,7 @@ public: { } - compact_edge_meta_t deduce_meta(CompactNode* in, CompactNode* out) { + compact_edge_meta_t deduce_edge_meta(CompactNode* in, CompactNode* out) { compact_edge_meta_t edge_meta; if (in == nullptr && out == nullptr) { edge_meta = ISLAND; @@ -682,6 +698,10 @@ public: return edges.n_edges(); } + uint64_t n_updates() const { + return nodes.n_updates() + edges.n_updates(); + } + void report() const { std::cout << std::endl << "REPORT: StreamingCompactor(@" << this << " with " << "Hashgraph @" << graph.get() << ")" << std::endl; @@ -763,7 +783,9 @@ public: /* Update a compact dbg where there are no induced * HDNs - uint64_t update_compact_dbg_linear(std::string& sequence) { + */ + uint64_t update_compact_dbg_linear(const std::string& sequence) { + uint64_t n_ops_before = n_updates(); Kmer root_kmer = graph->build_kmer(sequence.substr(0, _ksize)); CompactingAT lcursor(graph.get(), root_kmer); @@ -777,29 +799,48 @@ public: CompactNode *left_node = nullptr, *right_node = nullptr; left_node = nodes.get_node_by_kmer(lcursor.cursor); right_node = nodes.get_node_by_kmer(rcursor.cursor); + + CompactEdge *left_edge = nullptr, *right_edge = nullptr; + if (left_node != nullptr) { + nodes.get_edge_from_right(left_node, left_edge, segment_seq); + } + if (right_node != nullptr) { + nodes.get_edge_from_left(right_node, right_edge, segment_seq); + } + + if (left_edge != nullptr) { + nodes.unlink_edge(left_edge); + edges.delete_edge(left_edge); + } + if (right_edge != nullptr) { + nodes.unlink_edge(right_edge); + edges.delete_edge(right_edge); + } compact_edge_meta_t edge_meta = deduce_edge_meta(left_node, right_node); - char in_base = segment_seq[_ksize-1]; - char out_base = segment_seq[segment_seq.length()-_ksize+1]; - - switch(edge_meta) { - case FULL: - // then we merged two tips - pdebug("merge TIPs"); - break; - case TIP: - pdebug("extend TIP"); - break; - case ISLAND: - pdebug("created or extended ISLAND"); - break; + if (edge_meta == ISLAND) { // don't deal with islands for now + return n_updates() - n_ops_before; + } + uint64_t left_id, right_id; + left_id = (left_node != nullptr) ? left_node->node_id : NULL_ID; + right_id = (right_node != nullptr) ? right_node->node_id : NULL_ID; + CompactEdge *new_edge = edges.build_edge(left_id, right_id, + edge_meta, segment_seq); + if (left_node != nullptr) { + nodes.add_edge_from_right(left_node, new_edge); } + if (right_node != nullptr) { + nodes.add_edge_from_left(right_node, new_edge); + } + + return n_updates() - n_ops_before; } - */ + uint64_t update_compact_dbg(const std::string& sequence) { pdebug("update cDBG from " << sequence); n_sequences_added++; + uint64_t n_ops_before = n_updates(); // first gather up all k-mers that could have been disturbed -- // k-mers in the read, and the neighbors of the flanking nodes @@ -819,7 +860,6 @@ public: // find the induced HDNs in the disturbed k-mers KmerSet induced_hdns; - uint64_t n_updates = 0; while(!disturbed_kmers.empty()) { Kmer kmer = disturbed_kmers.back(); disturbed_kmers.pop_back(); @@ -829,12 +869,10 @@ public: if(l_degree > 1 || r_degree > 1) { pdebug("found HDN... " << kmer); CompactNode* hdn = nodes.get_or_build_node(kmer); - if (hdn->count == 1) { + if (hdn->count == 1) { // just created induced_hdns.insert(kmer); - n_updates++; } else if (hdn->degree() != (l_degree + r_degree)) { induced_hdns.insert(kmer); - n_updates++; } } } @@ -842,8 +880,9 @@ public: /* If there are no induced HDNs, we must have extended * a tip or merged two tips into a linear segment */ - // handle_merge() - + if (induced_hdns.size() == 0) { + return update_compact_dbg_linear(sequence); + } /* Update from all induced HDNs */ @@ -907,7 +946,7 @@ public: } else if (left_out_edge != nullptr) { // there was no edge from root, must be bad pdebug("edge from left invalid, delete"); - n_updates += nodes.unlink_edge(left_out_edge); + nodes.unlink_edge(left_out_edge); edges.delete_edge(left_out_edge); } else if (segment_edge != nullptr) { pdebug("found end leaving root node"); @@ -916,7 +955,7 @@ public: continue; } else { pdebug("edge from root invalid, delete"); - n_updates += nodes.unlink_edge(segment_edge); + nodes.unlink_edge(segment_edge); edges.delete_edge(segment_edge); } } @@ -951,8 +990,6 @@ public: segment_seq); } - - n_updates++; nodes.add_edge_from_left(root_node, segment_edge); } @@ -1001,7 +1038,7 @@ public: } else if (right_in_edge != nullptr) { // there was no edge from root, must be bad pdebug("edge from left invalid, delete"); - n_updates += nodes.unlink_edge(right_in_edge); + nodes.unlink_edge(right_in_edge); edges.delete_edge(right_in_edge); } else if (segment_edge != nullptr) { if (validate_segment(root_node, right_node, @@ -1009,7 +1046,7 @@ public: continue; } else { pdebug("edge from root invalid, delete"); - n_updates += nodes.unlink_edge(segment_edge); + nodes.unlink_edge(segment_edge); edges.delete_edge(segment_edge); } } @@ -1033,14 +1070,12 @@ public: segment_seq); } - - n_updates++; nodes.add_edge_from_right(root_node, segment_edge); } } - return n_updates; + return n_updates() - n_ops_before; } // update_compact_dbg diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 609cbfab8d..0055cbf0b0 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -52,6 +52,8 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpCompactEdgeFactory(WordLength) uint64_t n_edges() + uint64_t n_updates() + CpCompactEdge* build_edge(uint64_t, uint64_t, compact_edge_meta_t, string) void delete_edge(CpCompactEdge*) @@ -89,6 +91,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: cdef cppclass CpCompactNodeFactory "oxli::CompactNodeFactory" (CpKmerFactory): CpCompactNodeFactory(WordLength) uint64_t n_nodes() + uint64_t n_updates() CpCompactNode* build_node(CpKmer) CpCompactNode* get_node_by_kmer(HashIntoType) @@ -96,7 +99,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpCompactNode* get_or_build_node(CpKmer) vector[CpCompactNode*] get_nodes(const string&) - uint8_t unlink_edge(CpCompactEdge*) + void unlink_edge(CpCompactEdge*) bool get_pivot_from_left(CpCompactNode*, string&, char&) bool add_edge_from_left(CpCompactNode*, CpCompactEdge*) @@ -113,6 +116,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: void report() uint64_t n_nodes() uint64_t n_edges() + uint64_t n_updates() CpCompactNode* get_node_by_kmer(HashIntoType) CpCompactNode* get_node_by_id(uint64_t) diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 74ce6c7791..b667319bf9 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -164,3 +164,7 @@ cdef class StreamingCompactor: @property def n_edges(self): return deref(self._sc_this).n_edges() + + @property + def n_updates(self): + return deref(self._sc_this).n_updates() diff --git a/tests/test_compact_dbg.py b/tests/test_compact_dbg.py index 50f5b932a5..385198a369 100644 --- a/tests/test_compact_dbg.py +++ b/tests/test_compact_dbg.py @@ -227,5 +227,36 @@ def test_compact_trivial_edge(tandem_triple_forks, ksize): assert node_2.degree == 4 -def test_compact_tip_split_merge(): - pass +def test_compact_tip_linear_merge(left_tip_structure, right_tip_structure, + ksize): + right_tip_structure = right_tip_structure() + graph, contig_r, L_r, HDN_r, R_r, tip_r = right_tip_structure + left_tip_structure = left_tip_structure() + _, contig_l, L_l, HDN_l, R_l, tip_l = left_tip_structure + + contig_merge = contig_l[-ksize:] + contig_r[0:ksize] + graph.reset() + + compactor = StreamingCompactor(graph) + + compactor.consume(str(tip_l)) + print(compactor.consume_and_update(contig_l), + 'cDBG updates from left') + compactor.report() + compare_tip_with_cdbg(left_tip_structure, compactor) + assert compactor.n_nodes == 1 + assert compactor.n_edges == 3 + + compactor.consume(str(tip_r)) + print(compactor.consume_and_update(contig_r), + 'cDBG updates from right') + compactor.report() + compare_tip_with_cdbg(right_tip_structure, compactor) + assert compactor.n_nodes == 2 + assert compactor.n_edges == 6 + + print(compactor.consume_and_update(contig_merge), + 'cDBG updates from linear merge') + + assert compactor.n_nodes == 2 + assert compactor.n_edges == 5 From 69a4eb61faf18ed9e2edea245ee07b31392d523a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 21 Nov 2017 17:47:32 -0800 Subject: [PATCH 170/185] Add GML output and Edge map --- include/oxli/links.hh | 75 +++++++++++++++++++++++++++----------- khmer/_oxli/graphlinks.pxd | 31 +++++++++------- khmer/_oxli/graphlinks.pyx | 4 ++ src/oxli/links.cc | 71 ++++++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 35 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index 43fc086e80..bd4b5974e2 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -74,15 +74,17 @@ Contact: khmer-project@idyll.org namespace oxli { +typedef uint64_t id_t; #define NULL_ID ULLONG_MAX using std::make_shared; using std::shared_ptr; -typedef std::pair HashIDPair; +typedef std::pair HashIDPair; typedef std::unordered_set UHashSet; typedef std::vector HashVector; -typedef std::unordered_map HashIDMap; +typedef std::unordered_map HashIDMap; +typedef std::unordered_set IDSet; enum compact_edge_meta_t { @@ -113,17 +115,21 @@ class CompactEdge { public: - const uint64_t in_node_id; // left and right HDN IDs - const uint64_t out_node_id; + const id_t in_node_id; // left and right HDN IDs + const id_t out_node_id; + const id_t edge_id; compact_edge_meta_t meta; std::string sequence; UHashSet tags; - CompactEdge(uint64_t in_node_id, uint64_t out_node_id) : - in_node_id(in_node_id), out_node_id(out_node_id), meta(FULL) {} + CompactEdge(id_t in_node_id, id_t out_node_id, id_t edge_id) : + in_node_id(in_node_id), out_node_id(out_node_id), + meta(FULL), edge_id(edge_id) {} - CompactEdge(uint64_t in_node_id, uint64_t out_node_id, compact_edge_meta_t meta) : - in_node_id(in_node_id), out_node_id(out_node_id), meta(meta) {} + CompactEdge(id_t in_node_id, id_t out_node_id, id_t edge_id, + compact_edge_meta_t meta) : + in_node_id(in_node_id), out_node_id(out_node_id), + meta(meta), edge_id(edge_id) {} void add_tags(UHashSet& new_tags) { for (auto tag: new_tags) { @@ -169,12 +175,14 @@ public: typedef std::vector CompactEdgeVector; typedef std::unordered_map TagEdgeMap; +typedef std::unordered_map IDEdgeMap; typedef std::pair TagEdgePair; typedef std::set TagEdgePairSet; typedef std::set CompactEdgeSet; +class CompactNodeFactory; class CompactEdgeFactory : public KmerFactory { - + friend class CompactNodeFactory; protected: uint64_t n_compact_edges; @@ -182,7 +190,7 @@ protected: uint32_t tag_density; TagEdgeMap tags_to_edges; - CompactEdgeVector compact_edges; + IDEdgeMap compact_edges; public: @@ -201,11 +209,13 @@ public: return _n_updates; } - CompactEdge* build_edge(uint64_t left_id, uint64_t right_id, + CompactEdge* build_edge(id_t left_id, id_t right_id, compact_edge_meta_t edge_meta, std::string edge_sequence) { - CompactEdge* edge = new CompactEdge(left_id, right_id, edge_meta); + CompactEdge* edge = new CompactEdge(left_id, right_id, + _n_updates, edge_meta); + compact_edges[_n_updates] = edge; pdebug("new compact edge: \n left=" << std::to_string(left_id) << std::endl << " right=" << std::to_string(right_id) @@ -225,6 +235,14 @@ public: return edge; } + CompactEdge* get_edge_by_id(id_t id) { + auto search = compact_edges.find(id); + if (search != compact_edges.end()) { + return search->second; + } + return nullptr; + } + void delete_edge(CompactEdge * edge) { //pdebug("attempt edge delete @" << edge); if (edge != nullptr) { @@ -232,18 +250,24 @@ public: for (auto tag: edge->tags) { tags_to_edges.erase(tag); } + compact_edges.erase(edge->edge_id); delete edge; n_compact_edges--; _n_updates++; } } - void delete_edge(UHashSet& tags) { + void delete_edge_by_id(id_t id) { + CompactEdge* e = get_edge_by_id(id); + delete_edge(e); + } + + void delete_edge_by_tag(UHashSet& tags) { CompactEdge* edge = get_edge(tags); delete_edge(edge); } - void delete_edge(HashIntoType tag) { + void delete_edge_by_tag(HashIntoType tag) { CompactEdge* edge = get_edge(tag); delete_edge(edge); } @@ -287,6 +311,9 @@ public: return stopper; } + + void write_gml(const std::string filename, + const CompactNodeFactory& nodes) const; }; @@ -296,17 +323,17 @@ class CompactNode { public: Kmer kmer; uint32_t count; - const uint64_t node_id; + const id_t node_id; std::string sequence; bool direction; CompactEdge* in_edges[4] = {nullptr, nullptr, nullptr, nullptr}; CompactEdge* out_edges[4] = {nullptr, nullptr, nullptr, nullptr}; - CompactNode(Kmer kmer, uint64_t node_id) : + CompactNode(Kmer kmer, id_t node_id) : kmer(kmer), count(0), node_id(node_id), direction(kmer.is_forward()) {} - CompactNode(Kmer kmer, std::string sequence, uint64_t node_id) : + CompactNode(Kmer kmer, std::string sequence, id_t node_id) : kmer(kmer), count(0), sequence(sequence), node_id(node_id), direction(kmer.is_forward()) {} @@ -427,7 +454,7 @@ public: typedef std::vector CompactNodeVector; class CompactNodeFactory : public KmerFactory { - + friend class CompactEdgeFactory; protected: // map from HDN hashes to CompactNode IDs @@ -470,13 +497,13 @@ public: CompactNode* get_node_by_kmer(HashIntoType hdn) { auto search = kmer_id_map.find(hdn); if (search != kmer_id_map.end()) { - uint64_t ID = search->second; + id_t ID = search->second; return &(compact_nodes[ID]); } return nullptr; } - CompactNode* get_node_by_id(uint64_t id) { + CompactNode* get_node_by_id(id_t id) { if (id >= compact_nodes.size()) { return nullptr; } @@ -715,7 +742,7 @@ public: return nodes.get_node_by_kmer(hdn); } - CompactNode* get_node_by_id(uint64_t id) { + CompactNode* get_node_by_id(id_t id) { return nodes.get_node_by_id(id); } @@ -821,7 +848,7 @@ public: if (edge_meta == ISLAND) { // don't deal with islands for now return n_updates() - n_ops_before; } - uint64_t left_id, right_id; + id_t left_id, right_id; left_id = (left_node != nullptr) ? left_node->node_id : NULL_ID; right_id = (right_node != nullptr) ? right_node->node_id : NULL_ID; CompactEdge *new_edge = edges.build_edge(left_id, right_id, @@ -1079,6 +1106,10 @@ public: } // update_compact_dbg + void write_gml(const std::string filename) const { + edges.write_gml(filename, nodes); + } + }; diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 0055cbf0b0..8896b8bc09 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -17,10 +17,11 @@ cdef extern from "oxli/links.hh": cdef extern from "oxli/links.hh" namespace "oxli" nogil: - ctypedef pair[HashIntoType, uint64_t] HashIDPair + ctypedef uint64_t id_t + ctypedef pair[HashIntoType, id_t] HashIDPair ctypedef uset[HashIntoType] UHashSet ctypedef vector[HashIntoType] HashVector - ctypedef umap[HashIntoType, uint64_t] HashIDMap + ctypedef umap[HashIntoType, id_t] HashIDMap ctypedef enum compact_edge_meta_t: FULL @@ -31,14 +32,15 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: cdef const char * edge_meta_repr(compact_edge_meta_t) cdef cppclass CpCompactEdge "oxli::CompactEdge": - uint64_t in_node_id - uint64_t out_node_id + const id_t in_node_id + const id_t out_node_id + const id_t edge_id UHashSet tags compact_edge_meta_t meta string sequence - CpCompactEdge(uint64_t, uint64_t) - CpComapctEdge(uint64_t, uint64_t, compact_edge_meta_t) + CpCompactEdge(id_t, id_t) + CpComapctEdge(id_t, id_t, compact_edge_meta_t) string rc_sequence() void add_tags(UHashSet&) @@ -54,7 +56,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: uint64_t n_edges() uint64_t n_updates() - CpCompactEdge* build_edge(uint64_t, uint64_t, compact_edge_meta_t, + CpCompactEdge* build_edge(id_t, id_t, compact_edge_meta_t, string) void delete_edge(CpCompactEdge*) void delete_edge(UHashSet&) @@ -62,18 +64,19 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpCompactEdge* get_edge(HashIntoType) bool get_tag_edge_pair(HashIntoType, TagEdgePair&) CpCompactEdge* get_edge(UHashSet&) - cdef cppclass CpCompactNode "oxli::CompactNode": CpKmer kmer uint32_t count - const uint64_t node_id + const id_t node_id string sequence CpCompactEdge* in_edges[4] CpCompactEdge* out_edges[4] - CpCompactNode(CpKmer) + CpCompactNode(CpKmer, id_t) + CpCompactNode(CpKmer, string, id_t) + void add_in_edge(const char, CpCompactEdge*) bool delete_in_edge(CpCompactEdge*) CpCompactEdge* get_in_edge(const char) @@ -95,7 +98,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: CpCompactNode* build_node(CpKmer) CpCompactNode* get_node_by_kmer(HashIntoType) - CpCompactNode* get_node_by_id(uint64_t) + CpCompactNode* get_node_by_id(id_t) CpCompactNode* get_or_build_node(CpKmer) vector[CpCompactNode*] get_nodes(const string&) @@ -119,17 +122,19 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: uint64_t n_updates() CpCompactNode* get_node_by_kmer(HashIntoType) - CpCompactNode* get_node_by_id(uint64_t) + CpCompactNode* get_node_by_id(id_t) vector[CpCompactNode*] get_nodes(const string&) CpCompactEdge* get_edge(HashIntoType) - bool get_tag_edge_pair(uint64_t, TagEdgePair&) + bool get_tag_edge_pair(id_t, TagEdgePair&) CpCompactEdge* get_edge(UHashSet&) uint64_t update_compact_dbg(const string&) uint64_t consume_sequence(const string&) uint64_t consume_sequence_and_update(const string&) + void write_gml(string) + cdef class CompactNode: cdef CpCompactNode* _cn_this diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index b667319bf9..6c4f0bbb37 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -168,3 +168,7 @@ cdef class StreamingCompactor: @property def n_updates(self): return deref(self._sc_this).n_updates() + + def write_gml(self, str filename): + cdef string _filename = _bstring(filename) + deref(self._sc_this).write_gml(_filename) diff --git a/src/oxli/links.cc b/src/oxli/links.cc index c6906a108f..1b2407d20d 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -4,3 +4,74 @@ using namespace oxli; +namespace oxli { + +void CompactEdgeFactory::write_gml(const std::string filename, + const CompactNodeFactory& nodes) const { + + std::ofstream file; + file.open(filename); + pdebug("opened " << filename); + file << "graph 1" << std::endl << "[" << std::endl; + + pdebug("writing " << nodes.n_nodes() << " nodes"); + for (auto node : nodes.compact_nodes) { + file << " node [" << std::endl; + file << " id " << std::to_string(node.node_id) << std::endl; + file << " kmer \"" << node.sequence << "\"" << std::endl; + file << " count \"" << std::to_string(node.count) << "\"" << std::endl; + file << " ]" << std::endl; + } + + pdebug("writing " << compact_edges.size() << " edges"); + for (auto edge_pair : compact_edges) { + + id_t edge_id = edge_pair.first; + CompactEdge* edge = edge_pair.second; + + file << " edge [" << std::endl; + file << " id " << std::to_string(edge->edge_id) << std::endl; + + id_t in_id, out_id; + bool in_null = false, out_null = false; + if (edge->in_node_id == NULL_ID) { + in_id = NULL_ID - edge->edge_id; + in_null = true; + } else { + in_id = edge->in_node_id; + } + if(edge->out_node_id == NULL_ID) { + out_id = NULL_ID - edge->edge_id; + out_null = true; + } else { + out_id = edge->out_node_id; + } + + file << " source " << std::to_string(in_id) << std::endl; + file << " target " << std::to_string(out_id) << std::endl; + file << " sequence \"" << edge->sequence << "\"" << std::endl; + file << " ]" << std::endl; + + // dummy nodes for tips + if (in_null) { + file << " node [" << std::endl; + file << " id " << std::to_string(in_id) << std::endl; + file << " label \"null\"" << std::endl; + file << " ]" << std::endl; + } + + if (out_null) { + file << " node [" << std::endl; + file << " id " << std::to_string(out_id) << std::endl; + file << " label \"null\"" << std::endl; + file << " ]" << std::endl; + } + } + + file << "]"; + + file.close(); + pdebug("closed file"); +} + +}; From c0ef2294750c8c6215984306fc5707871dba9c48 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Dec 2017 16:09:18 -0800 Subject: [PATCH 171/185] add write_fasta --- include/oxli/links.hh | 11 +++++++-- khmer/_oxli/graphlinks.pxd | 1 + khmer/_oxli/graphlinks.pyx | 6 ++++- src/oxli/links.cc | 46 ++++++++++++++++++++++++++++++++------ 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index bd4b5974e2..b114215d2c 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -195,6 +195,7 @@ protected: public: CompactEdgeFactory(WordLength K) : + KmerFactory(K), n_compact_edges(0), _n_updates(0) { @@ -314,6 +315,8 @@ public: void write_gml(const std::string filename, const CompactNodeFactory& nodes) const; + void write_fasta(const std::string filename) const; + }; @@ -573,13 +576,13 @@ public: char& pivot_base) const { /* Check if sequence shared same canonical * orientation with v from graph left, assuming - * sequence does NOT include v + * sequence includes v */ const char * node_kmer = v->sequence.c_str(); const char * _segment = sequence.c_str(); pivot_base = _segment[sequence.size()-_ksize-1]; if (strncmp(node_kmer, - _segment+(sequence.size())-_ksize, + _segment+sequence.size()-_ksize, _ksize-1) == 0) { // same canonical orientation return false; @@ -1110,6 +1113,10 @@ public: edges.write_gml(filename, nodes); } + void write_fasta(const std::string filename) const { + edges.write_fasta(filename); + } + }; diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 8896b8bc09..1fd76b2d00 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -134,6 +134,7 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: uint64_t consume_sequence_and_update(const string&) void write_gml(string) + void write_fasta(string) cdef class CompactNode: diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 6c4f0bbb37..3ae3330b64 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -1,7 +1,7 @@ from cython.operator cimport dereference as deref from libcpp.memory cimport make_shared -from khmer._oxli.utils cimport _bstring +from khmer._oxli.utils cimport _bstring, _ustring from khmer._oxli.sequence cimport Alphabets @@ -172,3 +172,7 @@ cdef class StreamingCompactor: def write_gml(self, str filename): cdef string _filename = _bstring(filename) deref(self._sc_this).write_gml(_filename) + + def write_fasta(self, str filename): + cdef string _filename = _bstring(filename) + deref(self._sc_this).write_fasta(_filename) diff --git a/src/oxli/links.cc b/src/oxli/links.cc index 1b2407d20d..e5bac7cf84 100644 --- a/src/oxli/links.cc +++ b/src/oxli/links.cc @@ -12,7 +12,7 @@ void CompactEdgeFactory::write_gml(const std::string filename, std::ofstream file; file.open(filename); pdebug("opened " << filename); - file << "graph 1" << std::endl << "[" << std::endl; + file << "graph" << std::endl << "[" << std::endl; pdebug("writing " << nodes.n_nodes() << " nodes"); for (auto node : nodes.compact_nodes) { @@ -23,49 +23,59 @@ void CompactEdgeFactory::write_gml(const std::string filename, file << " ]" << std::endl; } + uint32_t edge_offset = INT_MAX / 2; pdebug("writing " << compact_edges.size() << " edges"); for (auto edge_pair : compact_edges) { - id_t edge_id = edge_pair.first; + id_t edge_id = edge_pair.first + edge_offset; CompactEdge* edge = edge_pair.second; file << " edge [" << std::endl; - file << " id " << std::to_string(edge->edge_id) << std::endl; + file << " id " << std::to_string(edge_id) << std::endl; id_t in_id, out_id; bool in_null = false, out_null = false; if (edge->in_node_id == NULL_ID) { - in_id = NULL_ID - edge->edge_id; + in_id = INT_MAX - edge_id; in_null = true; } else { in_id = edge->in_node_id; } if(edge->out_node_id == NULL_ID) { - out_id = NULL_ID - edge->edge_id; + out_id = INT_MAX - edge_id; out_null = true; } else { out_id = edge->out_node_id; } + if (in_null && out_null) { + std::cerr << "in and out nodes NULL_ID, something weird with " + << edge->edge_id << std::endl; + } + file << " source " << std::to_string(in_id) << std::endl; file << " target " << std::to_string(out_id) << std::endl; file << " sequence \"" << edge->sequence << "\"" << std::endl; + file << " Length " << edge->sequence.length() << std::endl; + file << " meta \"" << edge_meta_repr(edge->meta) << "\"" << std::endl; file << " ]" << std::endl; // dummy nodes for tips + /* if (in_null) { file << " node [" << std::endl; file << " id " << std::to_string(in_id) << std::endl; - file << " label \"null\"" << std::endl; + file << " label \"null_" << std::to_string(in_id) << "\"" << std::endl; file << " ]" << std::endl; } if (out_null) { file << " node [" << std::endl; file << " id " << std::to_string(out_id) << std::endl; - file << " label \"null\"" << std::endl; + file << " label \"null_" << std::to_string(out_id) << "\"" << std::endl; file << " ]" << std::endl; } + */ } file << "]"; @@ -74,4 +84,26 @@ void CompactEdgeFactory::write_gml(const std::string filename, pdebug("closed file"); } + +void CompactEdgeFactory::write_fasta(const std::string filename) const { + std::ofstream file; + file.open(filename); + pdebug("opened " << filename); + for (auto edge_pair : compact_edges) { + + id_t edge_id = edge_pair.first; + CompactEdge* edge = edge_pair.second; + file << ">" << "edge_id=" << edge_id; + file << " len=" << edge->sequence.length(); + file << " type=" << edge_meta_repr(edge->meta); + file << " src=" << edge->in_node_id; + file << " tgt=" << edge->out_node_id; + file << std::endl; + file << edge->sequence; + file << std::endl; + } + + file.close(); +} + }; From ca759917987202152b3115f1bc8e8dc32e92bded Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Dec 2017 16:10:12 -0800 Subject: [PATCH 172/185] Fix edge case with reads intersecting non-induced HDNs --- include/oxli/links.hh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/include/oxli/links.hh b/include/oxli/links.hh index b114215d2c..3ed4b12a80 100644 --- a/include/oxli/links.hh +++ b/include/oxli/links.hh @@ -57,8 +57,7 @@ Contact: khmer-project@idyll.org #include "assembler.hh" #include "alphabets.hh" - -# define DEBUG_LINKS +#define DEBUG_LINKS # ifdef DEBUG_LINKS # define pdebug(x) do { std::cout << std::endl << "@ " << __FILE__ <<\ ":" << __FUNCTION__ << ":" <<\ @@ -483,7 +482,7 @@ public: // protected linear creation of CompactNode // they should never be deleted, so this is straightforward CompactNode* build_node(Kmer hdn) { - //pdebug("new compact node from " << hdn); + pdebug("new compact node from " << hdn); CompactNode * v = get_node_by_kmer(hdn); if (v == nullptr) { compact_nodes.emplace_back(hdn, n_compact_nodes); @@ -492,7 +491,7 @@ public: v->sequence = _revhash(hdn, _ksize); kmer_id_map[hdn] = v->node_id; _n_updates++; - //pdebug("Allocate: " << *v); + pdebug("Allocate: " << *v); } return v; } @@ -815,6 +814,7 @@ public: * HDNs */ uint64_t update_compact_dbg_linear(const std::string& sequence) { + pdebug("no induced HDNs, update linear..."); uint64_t n_ops_before = n_updates(); Kmer root_kmer = graph->build_kmer(sequence.substr(0, _ksize)); @@ -890,6 +890,7 @@ public: // find the induced HDNs in the disturbed k-mers KmerSet induced_hdns; + KmerSet disturbed_hdns; while(!disturbed_kmers.empty()) { Kmer kmer = disturbed_kmers.back(); disturbed_kmers.pop_back(); @@ -903,6 +904,8 @@ public: induced_hdns.insert(kmer); } else if (hdn->degree() != (l_degree + r_degree)) { induced_hdns.insert(kmer); + } else { + disturbed_hdns.insert(kmer); } } } @@ -910,8 +913,10 @@ public: /* If there are no induced HDNs, we must have extended * a tip or merged two tips into a linear segment */ - if (induced_hdns.size() == 0) { + if (induced_hdns.size() == 0 && disturbed_hdns.size() == 0) { return update_compact_dbg_linear(sequence); + } else if (induced_hdns.size() == 0) { + induced_hdns.insert(disturbed_hdns.begin(), disturbed_hdns.end()); } /* Update from all induced HDNs @@ -1006,8 +1011,6 @@ public: ? TRIVIAL : edge_meta; if (edge_meta == FULL || edge_meta == TRIVIAL) { - // left side includes HDN, right side does not - segment_edge = edges.build_edge(left_node->node_id, root_node->node_id, edge_meta, @@ -1030,6 +1033,7 @@ public: Kmer neighbor = neighbors.back(); neighbors.pop_back(); rcursor.cursor = neighbor; + pdebug("right neighbor: " << neighbor.repr(_ksize)); TagEdgePair tag_pair; bool found_tag = false; @@ -1041,6 +1045,8 @@ public: } else { segment_seq = root_front + segment_seq; } + pdebug("assembled segment: " << segment_seq << " length: " << + segment_seq.length()); // first check for a segment going this direction from root CompactEdge* segment_edge = nullptr; nodes.get_edge_from_right(root_node, segment_edge, segment_seq); @@ -1081,7 +1087,6 @@ public: } } - pdebug("segment sequence length=" << segment_seq.length()); compact_edge_meta_t edge_meta = (right_node == nullptr) ? TIP : FULL; edge_meta = (segment_seq.length() == _ksize + 1 && edge_meta == FULL) From d1d4c5c7c660c418c04b07ce82562a92bb0dc9ba Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Dec 2017 16:10:34 -0800 Subject: [PATCH 173/185] wrap node and edge factories --- khmer/_oxli/graphlinks.pxd | 14 ++++++++++++ khmer/_oxli/graphlinks.pyx | 47 ++++++++++++++++++++++++++++++++++++++ tests/test_compact_dbg.py | 36 ++++++++++++++++++++++++++++- 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/graphlinks.pxd index 1fd76b2d00..a36caadd0a 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/graphlinks.pxd @@ -104,10 +104,12 @@ cdef extern from "oxli/links.hh" namespace "oxli" nogil: void unlink_edge(CpCompactEdge*) + bool is_rc_from_left(CpCompactNode* v, string&) bool get_pivot_from_left(CpCompactNode*, string&, char&) bool add_edge_from_left(CpCompactNode*, CpCompactEdge*) bool get_edge_from_left(CpCompactNode*, CpCompactEdge* &, string&) + bool is_rc_from_right(CpCompactNode* v, string&) bool get_pivot_from_right(CpCompactNode*, string&, char&) bool add_edge_from_right(CpCompactNode*, CpCompactEdge*) bool get_edge_from_right(CpCompactNode*, CpCompactEdge* &, string&) @@ -145,6 +147,12 @@ cdef class CompactNode: cdef CompactNode _wrap(CpCompactNode*) +cdef class CompactNodeFactory: + cdef CpCompactNodeFactory * _cnf_this + @staticmethod + cdef CompactNodeFactory _wrap(CpCompactNodeFactory*) + + cdef class CompactEdge: cdef CpCompactEdge* _ce_this @@ -152,6 +160,12 @@ cdef class CompactEdge: cdef CompactEdge _wrap(CpCompactEdge*) +cdef class CompactEdgeFactory: + cdef CpCompactEdgeFactory* _cef_this + @staticmethod + cdef CompactEdgeFactory _wrap(CpCompactEdgeFactory*) + + cdef class StreamingCompactor: cdef shared_ptr[CpHashgraph] _graph diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/graphlinks.pyx index 3ae3330b64..ca967cf198 100644 --- a/khmer/_oxli/graphlinks.pyx +++ b/khmer/_oxli/graphlinks.pyx @@ -54,6 +54,15 @@ cdef class CompactEdge: return str(self) +cdef class CompactEdgeFactory: + + @staticmethod + cdef CompactEdgeFactory _wrap(CpCompactEdgeFactory* _this): + cdef CompactEdgeFactory factory = CompactEdgeFactory() + factory._cef_this = _this + return factory + + cdef class CompactNode: def __cinit__(self): @@ -127,6 +136,44 @@ cdef class CompactNode: self.out_degree, self.sequence) + +cdef class CompactNodeFactory: + + @staticmethod + def new(WordLength ksize): + cdef CpCompactNodeFactory* factory = new CpCompactNodeFactory(ksize) + return CompactNodeFactory._wrap(factory) + + @staticmethod + cdef CompactNodeFactory _wrap(CpCompactNodeFactory* _this): + cdef CompactNodeFactory factory = CompactNodeFactory() + factory._cnf_this = _this + return factory + + def build_node(self, Kmer kmer): + cdef CpCompactNode* _node = \ + deref(self._cnf_this).build_node(deref(kmer._this.get())) + return CompactNode._wrap(_node) + + def get_pivot_from_left(self, CompactNode node, str sequence): + cdef string _sequence = _bstring(sequence) + cdef char pivot + cdef bool pivoted + pivoted = deref(self._cnf_this).get_pivot_from_left(node._cn_this, + _sequence, + pivot) + return (pivot).decode('UTF-8'), pivoted + + def get_pivot_from_right(self, CompactNode node, str sequence): + cdef string _sequence = _bstring(sequence) + cdef char pivot + cdef bool pivoted + pivoted = deref(self._cnf_this).get_pivot_from_right(node._cn_this, + _sequence, + pivot) + return (pivot).decode('UTF-8'), pivoted + + cdef class StreamingCompactor: def __cinit__(self, Hashgraph graph): diff --git a/tests/test_compact_dbg.py b/tests/test_compact_dbg.py index 385198a369..fb68a89180 100644 --- a/tests/test_compact_dbg.py +++ b/tests/test_compact_dbg.py @@ -9,7 +9,9 @@ from .khmer_tst_utils import _equals_rc, _contains_rc from .graph_structure_fixtures import * -from khmer._oxli.graphlinks import StreamingCompactor +from khmer._oxli.graphlinks import (StreamingCompactor, CompactNode, + CompactNodeFactory) +from khmer._oxli.hashing import Kmer as CyKmer from khmer import Nodegraph import pytest @@ -18,6 +20,38 @@ def teardown(): utils.cleanup() +def test_get_pivot_from_right(ksize, linear_structure): + graph, sequence = linear_structure() + print(sequence) + factory = CompactNodeFactory.new(ksize) + kmer = CyKmer(sequence[:ksize]) + node = factory.build_node(kmer) + print(node) + + if kmer.is_forward: + assert factory.get_pivot_from_right(node, sequence) == \ + (sequence[ksize], False) + else: + assert factory.get_pivot_from_right(node, sequence) == \ + (revcomp(sequence[ksize]), True) + + +def test_get_pivot_from_left(ksize, linear_structure): + graph, sequence = linear_structure() + print(sequence) + factory = CompactNodeFactory.new(ksize) + kmer = CyKmer(sequence[-ksize:]) + node = factory.build_node(kmer) + print(node) + + if kmer.is_forward: + assert factory.get_pivot_from_left(node, sequence) == \ + (sequence[-ksize-1], False) + else: + assert factory.get_pivot_from_left(node, sequence) == \ + (revcomp(sequence[-ksize-1]), True) + + def compare_tip_with_cdbg(rts, compactor): graph, contig, L, HDN, R, tip = rts From d5bb0693ecc00d98594c5351199734d02f564a7e Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 8 Dec 2017 16:28:12 -0800 Subject: [PATCH 174/185] fix cythonization in setup --- setup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 704385e140..3aa9eb9c0a 100755 --- a/setup.py +++ b/setup.py @@ -67,9 +67,10 @@ HAS_CYTHON = False try: import Cython - from Cython.Build import cythonize + from Cython.Distutils import Extension as CyExtension HAS_CYTHON = True except ImportError: + from setuptools import Extension as CyExtension pass cy_ext = 'pyx' if HAS_CYTHON else 'cpp' @@ -224,13 +225,14 @@ def build_dir(): "define_macros": [("VERSION", versioneer.get_version()), ] } + if HAS_CYTHON: + CY_EXTENSION_MOD_DICT['cython_directives'] = CY_OPTS + ext_name = "khmer._oxli.{0}".format( splitext(os.path.basename(cython_ext))[0]) - EXTENSION_MODS.append(Extension(ext_name, + EXTENSION_MODS.append(CyExtension(ext_name, ** CY_EXTENSION_MOD_DICT)) -EXTENSION_MODS = cythonize(EXTENSION_MODS, - compiler_directives=CY_OPTS) SCRIPTS = [] SCRIPTS.extend([path_join("scripts", script) From ac53da76edce52e76cff8b1254e0267713271b18 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Dec 2017 18:33:52 -0800 Subject: [PATCH 175/185] update queue arg --- khmer/_oxli/partitioning.pxd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/khmer/_oxli/partitioning.pxd b/khmer/_oxli/partitioning.pxd index e2b7aa7f5b..666ef7aed0 100644 --- a/khmer/_oxli/partitioning.pxd +++ b/khmer/_oxli/partitioning.pxd @@ -77,11 +77,11 @@ cdef extern from "oxli/partitioning.hh" namespace "oxli": uint64_t seed_sequence(string&, TagVector&, KmerQueue&, set[HashIntoType]&) except +MemoryError - void find_connected_tags(queue[CpKmer]&, + void find_connected_tags(KmerQueue&, TagVector&, set[HashIntoType]&) except +MemoryError - void find_connected_tags(queue[CpKmer]&, + void find_connected_tags(KmerQueue&, TagVector&, set[HashIntoType]&, bool) except +MemoryError From 302bb167eba2419076b6607d128fa2389aa5cc62 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Tue, 9 Jan 2018 16:19:26 -0800 Subject: [PATCH 176/185] Update streaming diginorm tests for FastxParser --- tests/test_scripts.py | 44 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 1ebe0d2107..4b54171ea3 100755 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -2100,32 +2100,22 @@ def execute_streaming_diginorm(ifilename): This is not directly executed but is run by the tests themselves ''' # Get temp filenames, etc. - fifo = utils.get_temp_filename('fifo') - in_dir = os.path.dirname(fifo) - script = 'normalize-by-median.py' - args = ['-C', '1', '-k', '17', '-o', 'outfile', fifo] - - # make a fifo to simulate streaming - os.mkfifo(fifo) - - # FIFOs MUST BE OPENED FOR READING BEFORE THEY ARE WRITTEN TO - # If this isn't done, they will BLOCK and things will hang. - thread = threading.Thread(target=utils.runscript, - args=(script, args, in_dir)) - thread.start() - ifile = io.open(ifilename, 'rb') - fifofile = io.open(fifo, 'wb') - # read binary to handle compressed files - chunk = ifile.read(8192) - while len(chunk) > 0: - fifofile.write(chunk) - chunk = ifile.read(8192) - - fifofile.close() + script = os.path.join(utils.scriptpath(), + 'normalize-by-median.py') + infile = utils.copy_test_data(ifilename) + in_dir = os.path.dirname(infile) + args = '-C 1 -k 17 -o outfile -' + cmd = 'cat {infile} | {script} {args}'.format(infile=infile, + script=script, + args=args) + (status, out, err) = utils.run_shell_cmd(cmd, in_directory=in_dir) - thread.join() + if status != 0: + print(out) + print(err) + assert status == 0, status - return in_dir + '/outfile' + return os.path.join(in_dir, 'outfile') def _execute_load_graph_streaming(filename): @@ -2139,7 +2129,7 @@ def _execute_load_graph_streaming(filename): infile = utils.copy_test_data(filename) in_dir = os.path.dirname(infile) - args = '-x 1e7 -N 2 -k 20 out -' + args = '-x 1e7 -N 2 -k 20 out /dev/stdin' cmd = 'cat {infile} | {scripts}/load-graph.py {args}'.format( infile=infile, scripts=scripts, args=args) @@ -2188,6 +2178,7 @@ def test_screed_streaming_ufq(): assert seqs[0].startswith('CAGGCGCCCACCACCGTGCCCTCCAACCTGATGGT') +@pytest.mark.known_failing def test_screed_streaming_bzipfq(): # bzip compressed fq o = execute_streaming_diginorm(utils.get_test_data('100-reads.fq.bz2')) @@ -2196,6 +2187,7 @@ def test_screed_streaming_bzipfq(): assert seqs[0].startswith('CAGGCGCCCACCACCGTGCCCTCCAACCTGATGGT'), seqs +@pytest.mark.known_failing def test_screed_streaming_bzipfa(): # bzip compressed fa o = execute_streaming_diginorm( @@ -2206,7 +2198,6 @@ def test_screed_streaming_bzipfa(): assert seqs[0].startswith('GGTTGACGGGGCTCAGGGGG') -@pytest.mark.known_failing def test_screed_streaming_gzipfq(): # gzip compressed fq o = execute_streaming_diginorm(utils.get_test_data('100-reads.fq.gz')) @@ -2215,7 +2206,6 @@ def test_screed_streaming_gzipfq(): assert seqs[0].startswith('CAGGCGCCCACCACCGTGCCCTCCAACCTG') -@pytest.mark.known_failing def test_screed_streaming_gzipfa(): o = execute_streaming_diginorm( utils.get_test_data('test-abund-read-2.fa.gz')) From 495c1b0aa2e2630938b5fd3f141beda0a755da75 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 11 Jan 2018 11:44:34 -0800 Subject: [PATCH 177/185] cythonized script does not work with stdout and err injection --- khmer/_oxli/app.pyx | 5 +++-- khmer/_oxli/legacy_partitioning.pxd | 2 +- tests/test_banding.py | 23 +++++++++++++---------- tests/test_counttable.py | 16 +++++++++------- tests/test_scripts.py | 3 ++- 5 files changed, 28 insertions(+), 21 deletions(-) diff --git a/khmer/_oxli/app.pyx b/khmer/_oxli/app.pyx index 937ec73ced..928e141d15 100644 --- a/khmer/_oxli/app.pyx +++ b/khmer/_oxli/app.pyx @@ -5,7 +5,8 @@ import json import os import sys -from khmer.khmer_args import build_counting_args, create_countgraph +from khmer.khmer_args import (build_counting_args, create_countgraph, + sanitize_help) from khmer.khmer_logger import (configure_logging, log_info, log_error, log_warn) @@ -48,7 +49,7 @@ cdef class PartitioningApp: parser.add_argument('--output-interval', default=0, type=int) parser.add_argument('--tag-density', default=None, type=int) - return parser.parse_args(args) + return sanitize_help(parser).parse_args(args) def write_results(self, folder, n, new_kmers): filename = os.path.join(folder, '{0}.csv'.format(n)) diff --git a/khmer/_oxli/legacy_partitioning.pxd b/khmer/_oxli/legacy_partitioning.pxd index 5ea499fcb4..2b4c8e262f 100644 --- a/khmer/_oxli/legacy_partitioning.pxd +++ b/khmer/_oxli/legacy_partitioning.pxd @@ -74,7 +74,7 @@ cdef extern from "oxli/subset.hh" nogil: unsigned long long repartition_largest_partition(unsigned int, unsigned int, unsigned int, - CpCountgraph&) + CpCountgraph&) except +oxli_raise_py_error void repartition_a_partition(const HashIntoTypeSet &) except +oxli_raise_py_error void _clear_partition(PartitionID, HashIntoTypeSet &) void _merge_other(HashIntoType, PartitionID, PartitionPtrMap &) diff --git a/tests/test_banding.py b/tests/test_banding.py index 3728ba0d8b..b274c57cb0 100755 --- a/tests/test_banding.py +++ b/tests/test_banding.py @@ -36,14 +36,16 @@ import screed import khmer from . import khmer_tst_utils as utils +from .graph_structure_fixtures import using_ksize import pytest -@pytest.mark.parametrize('ksize,memory,epsilon,numbands', [ - (21, 5e6, 1, 2), - (21, 5e6, 1, 4), - (21, 5e6, 1, 8), - (21, 5e6, 1, 16), +@using_ksize(21) +@pytest.mark.parametrize('memory,epsilon,numbands', [ + (5e6, 1, 2), + (5e6, 1, 4), + (5e6, 1, 8), + (5e6, 1, 16), ]) def test_banding_in_memory(ksize, memory, epsilon, numbands): """ @@ -82,11 +84,12 @@ def test_banding_in_memory(ksize, memory, epsilon, numbands): assert min(nonzeros) == 1 -@pytest.mark.parametrize('ksize,memory,numbands', [ - (21, 5e6, 3), - (21, 5e6, 11), - (21, 5e6, 23), - (21, 5e6, 29), +@using_ksize(21) +@pytest.mark.parametrize('memory,numbands', [ + (5e6, 3), + (5e6, 11), + (5e6, 23), + (5e6, 29), ]) def test_banding_to_disk(ksize, memory, numbands): """ diff --git a/tests/test_counttable.py b/tests/test_counttable.py index 1873668a35..2b2dea3a8b 100755 --- a/tests/test_counttable.py +++ b/tests/test_counttable.py @@ -38,6 +38,7 @@ import pytest from . import khmer_tst_utils as utils +from .graph_structure_fixtures import using_ksize def test_get_kmer_hashes(): @@ -61,13 +62,14 @@ def test_kmer_revcom_hash(kmer): assert a.hash(kmer) == a.hash(khmer.reverse_complement(kmer)) -@pytest.mark.parametrize('ksize,sketch_allocator', [ - (21, khmer.Nodetable), - (21, khmer.Counttable), - (21, khmer.SmallCounttable), - (49, khmer.Nodetable), - (49, khmer.Counttable), - (49, khmer.SmallCounttable), +@using_ksize([21,49]) +@pytest.mark.parametrize('sketch_allocator', [ + (khmer.Nodetable), + (khmer.Counttable), + (khmer.SmallCounttable), + (khmer.Nodetable), + (khmer.Counttable), + (khmer.SmallCounttable), ]) def test_reverse_hash(ksize, sketch_allocator): multiplier = int(ksize / len('GATTACA')) diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 4b54171ea3..5a678f47b0 100755 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -2866,9 +2866,10 @@ def test_unique_kmers_multiple_inputs(): if entry.endswith('.py')]) def test_version_and_basic_citation(scriptname): with open(os.path.join(utils.scriptpath(), scriptname)) as script: + print(script) line = script.readline() line = script.readline() - if 'khmer' in line: + if 'khmer' in line and '_oxli.app' not in line: # check citation information appears when using --info status, out, err = utils.runscript(scriptname, ["--info"]) assert status == 0, status From 9957b2c037cfe0a7ce11d685d89f1eb2bb165710 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 11 Jan 2018 12:25:38 -0800 Subject: [PATCH 178/185] convet load-graph to FastxParser --- oxli/functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oxli/functions.py b/oxli/functions.py index c79c475f83..de93da82b0 100755 --- a/oxli/functions.py +++ b/oxli/functions.py @@ -37,6 +37,7 @@ import threading import khmer.utils +from khmer._oxli.parsing import FastxParser def build_graph(ifilenames, graph, num_threads=1, tags=False): @@ -54,7 +55,7 @@ def build_graph(ifilenames, graph, num_threads=1, tags=False): eat = graph.consume_seqfile for _, ifile in enumerate(ifilenames): - rparser = khmer.ReadParser(ifile) + rparser = FastxParser(ifile) threads = [] for _ in range(num_threads): From 2d9e79de4bdb823d3e97d3e84925f2bfa9b5c39e Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 11 Jan 2018 12:26:10 -0800 Subject: [PATCH 179/185] switch streaming input to dash instead of stdin --- tests/test_scripts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 5a678f47b0..ad815bf33d 100755 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -973,6 +973,7 @@ def test_partition_graph_no_big_traverse(): assert x[0] == 4, x # should be four partitions, broken at knot. +@pytest.mark.xfail(reason='Deprecated legacy partitioning.') def test_partition_find_knots_execute(): graphbase = _make_graph(utils.get_test_data('random-20-a.fa')) @@ -989,6 +990,7 @@ def test_partition_find_knots_execute(): assert os.path.exists(stoptags_file) +@pytest.mark.xfail(reason='Deprecated legacy partitioning.') def test_partition_find_knots_existing_stoptags(): graphbase = _make_graph(utils.get_test_data('random-20-a.fa')) @@ -2129,7 +2131,7 @@ def _execute_load_graph_streaming(filename): infile = utils.copy_test_data(filename) in_dir = os.path.dirname(infile) - args = '-x 1e7 -N 2 -k 20 out /dev/stdin' + args = '-x 1e7 -N 2 -k 20 out -' cmd = 'cat {infile} | {scripts}/load-graph.py {args}'.format( infile=infile, scripts=scripts, args=args) From bbf7c4b2a91faa1febc5df19370a75c2fe70c651 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 11 Jan 2018 12:35:50 -0800 Subject: [PATCH 180/185] update assembly tests from master for ksize fixture --- tests/test_assembly.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_assembly.py b/tests/test_assembly.py index f48b4a8a0c..c285cf2ffa 100755 --- a/tests/test_assembly.py +++ b/tests/test_assembly.py @@ -137,16 +137,16 @@ def test_beginning_to_branch(self, ksize, right_tip_structure): graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.LinearAssembler(graph) - def test_assemble_takes_hash(self, right_tip_structure): + def test_assemble_takes_hash(self, ksize, right_tip_structure): # assemble from beginning of contig, up until branch point - graph, contig, L, HDN, R, tip = right_tip_structure + graph, contig, L, HDN, R, tip = right_tip_structure() asm = khmer.LinearAssembler(graph) - path = asm.assemble(graph.hash(contig[0:K])) + path = asm.assemble(graph.hash(contig[0:ksize])) - assert len(path) == HDN.pos + K + assert len(path) == HDN.pos + ksize assert utils._equals_rc(path, contig[:len(path)]) - def test_beginning_to_branch_revcomp(self, right_tip_structure): + def test_beginning_to_branch_revcomp(self, ksize, right_tip_structure): # assemble from beginning of contig, up until branch point # starting from rev comp graph, contig, L, HDN, R, tip = right_tip_structure() From 5d62b3e707fb5a304f7c7b333f6a3d1e014eb2fe Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 11 Jan 2018 13:48:20 -0800 Subject: [PATCH 181/185] Change links and graphlinks filenames to cdbg basename --- include/oxli/{links.hh => cdbg.hh} | 8 ++++---- khmer/_oxli/{graphlinks.pxd => cdbg.pxd} | 4 ++-- khmer/_oxli/{graphlinks.pyx => cdbg.pyx} | 0 setup.py | 4 ++-- src/oxli/Makefile | 6 ++++-- src/oxli/{links.cc => cdbg.cc} | 2 +- tests/test_compact_dbg.py | 6 ++---- 7 files changed, 15 insertions(+), 15 deletions(-) rename include/oxli/{links.hh => cdbg.hh} (99%) rename khmer/_oxli/{graphlinks.pxd => cdbg.pxd} (98%) rename khmer/_oxli/{graphlinks.pyx => cdbg.pyx} (100%) rename src/oxli/{links.cc => cdbg.cc} (99%) diff --git a/include/oxli/links.hh b/include/oxli/cdbg.hh similarity index 99% rename from include/oxli/links.hh rename to include/oxli/cdbg.hh index 3ed4b12a80..2724b7e748 100644 --- a/include/oxli/links.hh +++ b/include/oxli/cdbg.hh @@ -34,8 +34,8 @@ LICENSE (END) Contact: khmer-project@idyll.org */ -#ifndef LINKS_HH -#define LINKS_HH +#ifndef CDBG_HH +#define CDBG_HH #include #include @@ -57,8 +57,8 @@ Contact: khmer-project@idyll.org #include "assembler.hh" #include "alphabets.hh" -#define DEBUG_LINKS -# ifdef DEBUG_LINKS +#define DEBUG_CDBG +# ifdef DEBUG_CDBG # define pdebug(x) do { std::cout << std::endl << "@ " << __FILE__ <<\ ":" << __FUNCTION__ << ":" <<\ __LINE__ << std::endl << x << std::endl;\ diff --git a/khmer/_oxli/graphlinks.pxd b/khmer/_oxli/cdbg.pxd similarity index 98% rename from khmer/_oxli/graphlinks.pxd rename to khmer/_oxli/cdbg.pxd index a36caadd0a..043139f2e3 100644 --- a/khmer/_oxli/graphlinks.pxd +++ b/khmer/_oxli/cdbg.pxd @@ -12,10 +12,10 @@ from khmer._oxli.hashing cimport CpKmer, Kmer, CpKmerFactory from khmer._oxli.graphs cimport CpHashgraph, Hashgraph, Nodegraph, Countgraph -cdef extern from "oxli/links.hh": +cdef extern from "oxli/cdbg.hh": cdef uint64_t NULL_ID -cdef extern from "oxli/links.hh" namespace "oxli" nogil: +cdef extern from "oxli/cdbg.hh" namespace "oxli" nogil: ctypedef uint64_t id_t ctypedef pair[HashIntoType, id_t] HashIDPair diff --git a/khmer/_oxli/graphlinks.pyx b/khmer/_oxli/cdbg.pyx similarity index 100% rename from khmer/_oxli/graphlinks.pyx rename to khmer/_oxli/cdbg.pyx diff --git a/setup.py b/setup.py index ca5ed14240..820e3a5b8c 100755 --- a/setup.py +++ b/setup.py @@ -166,7 +166,7 @@ def build_dir(): "khmer", "kmer_hash", "hashtable", "labelhash", "hashgraph", "hllcounter", "oxli_exception", "read_aligner", "subset", "read_parsers", "kmer_filters", "traversal", "assembler", "alphabets", "storage", - "partitioning", "gmap", "hist", "links"]) + "partitioning", "gmap", "hist", "cdbg"]) SOURCES = [path_join("src", "khmer", bn + ".cc") for bn in [ "_cpy_khmer", "_cpy_utils", "_cpy_readparsers" @@ -175,7 +175,7 @@ def build_dir(): "read_parsers", "kmer_hash", "hashtable", "hashgraph", "labelhash", "subset", "read_aligner", "oxli", "hllcounter", "traversal", "kmer_filters", "assembler", "alphabets", - "storage", "partitioning", "links"]) + "storage", "partitioning", "cdbg"]) SOURCES.extend(path_join("third-party", "smhasher", bn + ".cc") for bn in [ "MurmurHash3"]) diff --git a/src/oxli/Makefile b/src/oxli/Makefile index 23bf96a3aa..b06ea96c6a 100644 --- a/src/oxli/Makefile +++ b/src/oxli/Makefile @@ -243,7 +243,8 @@ LIBOXLI_OBJS= \ alphabets.o \ murmur3.o \ storage.o \ - partitioning.o + partitioning.o \ + cdbg.o PRECOMILE_OBJS ?= PRECLEAN_TARGS ?= @@ -282,7 +283,8 @@ HEADERS= \ storage.hh \ partitioning.hh \ gmap.hh \ - hist.hh + hist.hh \ + cdbg.hh OXLI_HEADERS = $(addprefix ../../include/oxli/,$(HEADERS)) diff --git a/src/oxli/links.cc b/src/oxli/cdbg.cc similarity index 99% rename from src/oxli/links.cc rename to src/oxli/cdbg.cc index e5bac7cf84..0dc1cd68c0 100644 --- a/src/oxli/links.cc +++ b/src/oxli/cdbg.cc @@ -1,6 +1,6 @@ #include -#include "oxli/links.hh" +#include "oxli/cdbg.hh" using namespace oxli; diff --git a/tests/test_compact_dbg.py b/tests/test_compact_dbg.py index fb68a89180..0b47eab373 100644 --- a/tests/test_compact_dbg.py +++ b/tests/test_compact_dbg.py @@ -9,8 +9,8 @@ from .khmer_tst_utils import _equals_rc, _contains_rc from .graph_structure_fixtures import * -from khmer._oxli.graphlinks import (StreamingCompactor, CompactNode, - CompactNodeFactory) +from khmer._oxli.cdbg import (StreamingCompactor, CompactNode, + CompactNodeFactory) from khmer._oxli.hashing import Kmer as CyKmer from khmer import Nodegraph import pytest @@ -84,8 +84,6 @@ def compare_tip_with_cdbg(rts, compactor): @using_ksize([21,25,31]) def test_compact_tip(ksize, right_tip_structure): - '''Should have no links. Need two junctions. - ''' right_tip_structure = right_tip_structure() graph, contig, L, HDN, R, tip = right_tip_structure From cf9e6c349641959e7f257af70b452533ac877ba0 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 26 Jan 2018 17:12:56 -0800 Subject: [PATCH 182/185] unidirectional mrmur --- Makefile | 1 + include/oxli/kmer_hash.hh | 1 + src/oxli/kmer_hash.cc | 7 +++++++ 3 files changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 1d6c5298d9..bc5f065c2e 100644 --- a/Makefile +++ b/Makefile @@ -290,6 +290,7 @@ install-liboxli: liboxli mkdir -p $(PREFIX)/include/khmer cp -r include/khmer/_cpy_*.hh $(PREFIX)/include/khmer/ cp include/oxli/oxli_exception_convert.hh $(PREFIX)/include/oxli/ + cp third-party/rollinghash/*.h $(PREFIX)/include/oxli/ # Runs a test of liboxli libtest: FORCE diff --git a/include/oxli/kmer_hash.hh b/include/oxli/kmer_hash.hh index fdab1c10af..03e846942b 100644 --- a/include/oxli/kmer_hash.hh +++ b/include/oxli/kmer_hash.hh @@ -116,6 +116,7 @@ HashIntoType _hash_murmur(const std::string& kmer, const WordLength k, HashIntoType& h, HashIntoType& r); HashIntoType _hash_murmur_forward(const std::string& kmer, const WordLength k); +uint64_t _hash_murmur_uni(const std::string& sequence); // Cyclic hash, a rolling hash that is irreversible HashIntoType _hash_cyclic(const std::string& kmer, const WordLength k); diff --git a/src/oxli/kmer_hash.cc b/src/oxli/kmer_hash.cc index 8378ee1936..38ea1247d0 100644 --- a/src/oxli/kmer_hash.cc +++ b/src/oxli/kmer_hash.cc @@ -207,6 +207,13 @@ HashIntoType _hash_murmur_forward(const std::string& kmer, const WordLength k) return h; } +uint64_t _hash_murmur_uni(const std::string& sequence) { + uint64_t out[2]; + uint64_t seed = 0; + MurmurHash_x64_128((void*)sequence.c_str(), sequence.length(), seed, &out); + return out[0]; +} + HashIntoType _hash_cyclic(const std::string& kmer, const WordLength k) { HashIntoType h = 0; From c9456ce8ada65313d142b1d278e54d913b3ec183 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 22 Feb 2018 11:27:55 -0800 Subject: [PATCH 183/185] fix murmurhash call --- src/oxli/kmer_hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oxli/kmer_hash.cc b/src/oxli/kmer_hash.cc index 38ea1247d0..921eb529ea 100644 --- a/src/oxli/kmer_hash.cc +++ b/src/oxli/kmer_hash.cc @@ -210,7 +210,7 @@ HashIntoType _hash_murmur_forward(const std::string& kmer, const WordLength k) uint64_t _hash_murmur_uni(const std::string& sequence) { uint64_t out[2]; uint64_t seed = 0; - MurmurHash_x64_128((void*)sequence.c_str(), sequence.length(), seed, &out); + MurmurHash3_x64_128((void*)sequence.c_str(), sequence.length(), seed, &out); return out[0]; } From 141f2219737232f1beef265056ce8b02f9b0728c Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Thu, 22 Feb 2018 11:28:20 -0800 Subject: [PATCH 184/185] include memory header in storage.hh for export --- include/oxli/storage.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/include/oxli/storage.hh b/include/oxli/storage.hh index ca380d9e65..4cf9cee80f 100644 --- a/include/oxli/storage.hh +++ b/include/oxli/storage.hh @@ -40,6 +40,7 @@ Contact: khmer-project@idyll.org #include #include +#include #include #include using MuxGuard = std::lock_guard; From f6e9efbb4a8892e07d52e50da3c4275a08e7f252 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Mon, 26 Feb 2018 21:05:51 -0800 Subject: [PATCH 185/185] Improve Cyclic and Murmur forward hash impls --- src/oxli/kmer_hash.cc | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/oxli/kmer_hash.cc b/src/oxli/kmer_hash.cc index 921eb529ea..1edd1c33eb 100644 --- a/src/oxli/kmer_hash.cc +++ b/src/oxli/kmer_hash.cc @@ -199,21 +199,24 @@ HashIntoType _hash_murmur(const std::string& kmer, const WordLength k, HashIntoType _hash_murmur_forward(const std::string& kmer, const WordLength k) { - HashIntoType h = 0; - HashIntoType r = 0; - - oxli::_hash_murmur(kmer, k, h, r); + uint64_t out[2]; + uint64_t seed = 0; + MurmurHash3_x64_128((void*)kmer.c_str(), k, seed, &out); + return out[0]; - return h; } -uint64_t _hash_murmur_uni(const std::string& sequence) { + +HashIntoType _hash_murmur_forward(const std::string& kmer) +{ uint64_t out[2]; uint64_t seed = 0; - MurmurHash3_x64_128((void*)sequence.c_str(), sequence.length(), seed, &out); + MurmurHash3_x64_128((void*)kmer.c_str(), kmer.length(), seed, &out); return out[0]; + } + HashIntoType _hash_cyclic(const std::string& kmer, const WordLength k) { HashIntoType h = 0; @@ -258,11 +261,11 @@ HashIntoType _hash_cyclic(const std::string& kmer, const WordLength k, HashIntoType _hash_cyclic_forward(const std::string& kmer, const WordLength k) { - HashIntoType h = 0; - HashIntoType r = 0; - - oxli::_hash_cyclic(kmer, k, h, r); - return h; + CyclicHash hasher(k); + for (WordLength i = 0; i < k; ++i) { + hasher.eat(kmer[i]); + } + return hasher.hashvalue; }