From dcdb15650a4b448a8dd0a5b76f8b5d3cd36f0885 Mon Sep 17 00:00:00 2001 From: Masajiro Iwasaki Date: Wed, 19 Jul 2023 15:04:18 +0900 Subject: [PATCH] QBG internal improvements --- VERSION | 2 +- lib/NGT/Clustering.h | 43 +- lib/NGT/Command.cpp | 6 +- lib/NGT/Common.h | 134 +-- lib/NGT/GraphOptimizer.h | 2 + lib/NGT/NGTQ/Capi.cpp | 19 +- lib/NGT/NGTQ/HierarchicalKmeans.cpp | 641 ++++++++++++ lib/NGT/NGTQ/HierarchicalKmeans.h | 831 +++++++-------- lib/NGT/NGTQ/Optimizer.cpp | 251 +++-- lib/NGT/NGTQ/Optimizer.h | 89 +- lib/NGT/NGTQ/QbgCli.cpp | 452 ++++++--- lib/NGT/NGTQ/QbgCli.h | 36 +- lib/NGT/NGTQ/QuantizedBlobGraph.h | 1446 ++++++++++++++++----------- lib/NGT/NGTQ/QuantizedGraph.cpp | 18 +- lib/NGT/NGTQ/QuantizedGraph.h | 14 +- lib/NGT/NGTQ/Quantizer.h | 803 ++++++++++++--- lib/NGT/ObjectRepository.h | 2 +- lib/NGT/ObjectSpace.h | 54 +- lib/NGT/Optimizer.h | 8 +- lib/NGT/defines.h.in | 1 + python/src/ngtpy.cpp | 202 ++-- samples/qbg-capi/qbg-capi.cpp | 3 +- samples/qg-capi/qg-capi.cpp | 3 +- 23 files changed, 3431 insertions(+), 1629 deletions(-) create mode 100644 lib/NGT/NGTQ/HierarchicalKmeans.cpp diff --git a/VERSION b/VERSION index a14da29..7ec1d6d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.16 +2.1.0 diff --git a/lib/NGT/Clustering.h b/lib/NGT/Clustering.h index b377852..ed05643 100644 --- a/lib/NGT/Clustering.h +++ b/lib/NGT/Clustering.h @@ -81,6 +81,7 @@ namespace NGT { class Cluster { public: + Cluster():radius(0.0) {} Cluster(std::vector &c):centroid(c), radius(0.0) {} Cluster(const Cluster &c) { *this = c; } Cluster &operator=(const Cluster &c) { @@ -95,8 +96,8 @@ namespace NGT { double radius; }; - Clustering(InitializationMode im = InitializationModeHead, ClusteringType ct = ClusteringTypeKmeansWithNGT, size_t mi = 10000, size_t nc = 0): - clusteringType(ct), initializationMode(im), numberOfClusters(nc), maximumIteration(mi) { initialize(); } + Clustering(InitializationMode im = InitializationModeHead, ClusteringType ct = ClusteringTypeKmeansWithNGT, size_t mi = 10000, size_t nc = 0, bool s = true): + clusteringType(ct), initializationMode(im), numberOfClusters(nc), maximumIteration(mi), silence(s) { initialize(); } void initialize() { epsilonFrom = 0.12; @@ -208,8 +209,9 @@ namespace NGT { } } if ((numberOfClusters != 0) && (clusters.size() < numberOfClusters)) { - std::cerr << "initial cluster data are not enough. " << clusters.size() << ":" << numberOfClusters << std::endl; - exit(1); + std::stringstream msg; + msg << "initial cluster data are not enough. " << clusters.size() << ":" << numberOfClusters; + NGTThrowException(msg); } } #if !defined(NGT_CLUSTER_NO_AVX) @@ -247,6 +249,33 @@ namespace NGT { } #endif // !defined(NGT_AVX_DISABLED) && defined(__AVX__) + static void + clearMembers(std::vector &clusters) { + for (auto &cluster : clusters) { + cluster.members.clear(); + } + } + + static size_t + removeEmptyClusters(std::vector &clusters) { + size_t count = 0; + auto dst = clusters.begin(); + for (auto src = clusters.begin(); src != clusters.end(); ++src) { + if ((*src).members.size() == 0) { + count++; + continue; + } + if (dst != src) { + *dst = std::move(*src); + } + ++dst; + } + if (count != 0) { + clusters.resize(clusters.size() - count); + } + return count; + } + static double distanceL2(std::vector &vector1, std::vector &vector2) { return sqrt(sumOfSquares(&vector1[0], &vector2[0], vector1.size())); @@ -661,10 +690,13 @@ namespace NGT { } static void - saveClusters(const std::string &file, std::vector &clusters) + saveClusters(const std::string &file, std::vector &clusters, bool skipEmptyClusters = false) { std::ofstream os(file); for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) { + if (skipEmptyClusters && (*cit).members.size() == 0) { + continue; + } std::vector &v = (*cit).centroid; for (auto it = v.begin(); it != v.end(); ++it) { os << std::setprecision(9) << (*it); @@ -1042,6 +1074,7 @@ namespace NGT { float epsilonStep; size_t resultSizeCoefficient; vector diffHistory; + bool silence; }; } diff --git a/lib/NGT/Command.cpp b/lib/NGT/Command.cpp index c83f815..58bbad9 100644 --- a/lib/NGT/Command.cpp +++ b/lib/NGT/Command.cpp @@ -280,7 +280,7 @@ using namespace std; if (searchParameters.querySize > 0 && queryCount >= searchParameters.querySize) { break; } - NGT::Object *object = index.allocateObject(line, " \t"); + NGT::Object *object = index.allocateObject(line, " \t,"); queryCount++; size_t step = searchParameters.step == 0 ? UINT_MAX : searchParameters.step; for (size_t n = 0; n <= step; n++) { @@ -373,6 +373,8 @@ using namespace std; if (searchParameters.outputMode[0] == 'e') { stream << "# Average Query Time (msec)=" << totalTime * 1000.0 / (double)queryCount << endl; stream << "# Number of queries=" << queryCount << endl; + stream << "# VM size=" << NGT::Common::getProcessVmSizeStr() << std::endl; + stream << "# Peak VM size=" << NGT::Common::getProcessVmPeakStr() << std::endl; stream << "# End of Evaluation" << endl; if (searchParameters.outputMode == "e+") { @@ -510,7 +512,7 @@ using namespace std; while(getline(is, line)) { count++; vector tokens; - NGT::Common::tokenize(line, tokens, "\t "); + NGT::Common::tokenize(line, tokens, "\t, "); if (tokens.size() == 0 || tokens[0].size() == 0) { continue; } diff --git a/lib/NGT/Common.h b/lib/NGT/Common.h index 2c22c0a..6b5b925 100644 --- a/lib/NGT/Common.h +++ b/lib/NGT/Common.h @@ -234,6 +234,69 @@ namespace NGT { char **argV; }; + class Timer { + public: + Timer():time(0) {} + void reset() { time = 0; ntime = 0; } + + void start() { + struct timespec res; + clock_getres(CLOCK_REALTIME, &res); + reset(); + clock_gettime(CLOCK_REALTIME, &startTime); + } + + void restart() { + clock_gettime(CLOCK_REALTIME, &startTime); + } + + void stop() { + clock_gettime(CLOCK_REALTIME, &stopTime); + sec = stopTime.tv_sec - startTime.tv_sec; + nsec = stopTime.tv_nsec - startTime.tv_nsec; + if (nsec < 0) { + sec -= 1; + nsec += 1000000000L; + } + time += (double)sec + (double)nsec / 1000000000.0; + ntime += sec * 1000000000L + nsec; + } + + void add(Timer &t) { + time += t.time; + ntime += t.ntime; + } + + friend std::ostream &operator<<(std::ostream &os, Timer &t) { + auto time = t.time; + if (time < 1.0) { + time *= 1000.0; + os << std::setprecision(6) << time << " (ms)"; + return os; + } + if (time < 60.0) { + os << std::setprecision(6) << time << " (s)"; + return os; + } + time /= 60.0; + if (time < 60.0) { + os << std::setprecision(6) << time << " (m)"; + return os; + } + time /= 60.0; + os << std::setprecision(6) << time << " (h)"; + return os; + } + + struct timespec startTime; + struct timespec stopTime; + + int64_t sec; + int64_t nsec; + int64_t ntime; // nano second + double time; // second + }; + class Common { public: static void tokenize(const std::string &str, std::vector &token, const std::string seps) { @@ -289,13 +352,13 @@ namespace NGT { for (idx = 0; idx < tokens.size(); idx++) { if (tokens[idx].size() == 0) { std::stringstream msg; - msg << "Common::extractVecotFromText: No data. " << textLine; + msg << "Common::extractVecot: No data. " << textLine; NGTThrowException(msg); } char *e; double v = ::strtod(tokens[idx].c_str(), &e); if (*e != 0) { - std::cerr << "ObjectSpace::readText: Warning! Not numerical value. [" << e << "]" << std::endl; + std::cerr << "Common::extractVector: Warning! Not numerical value. [" << e << "] " << std::endl; break; } object.push_back(v); @@ -342,7 +405,7 @@ namespace NGT { } size = round(size * 100) / 100; std::stringstream str; - str << size << unit; + str << size << " " << unit; return str.str(); } static std::string getProcessVmSizeStr() { return sizeToString(getProcessVmSize()); } @@ -1319,7 +1382,7 @@ namespace NGT { } - size_t size() { return vectorSize; } + size_t size() const { return vectorSize; } public: void extend(SharedMemoryAllocator &allocator) { @@ -2176,6 +2239,7 @@ namespace NGT { public: Container(Object &o, ObjectID i):object(o), id(i) {} Container(Container &c):object(c.object), id(c.id) {} + bool isEmptyObject() { return &object == 0; } Object &object; ObjectID id; }; @@ -2213,7 +2277,7 @@ namespace NGT { useAllNodesInLeaf = false; expectedAccuracy = -1.0; } - void setSize(size_t s) { size = s; }; + void setSize(size_t s) { size = s; } void setResults(ObjectDistances *r) { result = r; } void setRadius(Distance r) { radius = r; } void setEpsilon(float e) { explorationCoefficient = e + 1.0; } @@ -2240,7 +2304,6 @@ namespace NGT { bool useAllNodesInLeaf; size_t visitCount; float expectedAccuracy; - private: ObjectDistances *result; }; @@ -2300,64 +2363,5 @@ namespace NGT { InsertContainer(Object &f, ObjectID i):Container(f, i) {} }; - class Timer { - public: - Timer():time(0) {} - - void reset() { time = 0; ntime = 0; } - - void start() { - struct timespec res; - clock_getres(CLOCK_REALTIME, &res); - reset(); - clock_gettime(CLOCK_REALTIME, &startTime); - } - - void restart() { - clock_gettime(CLOCK_REALTIME, &startTime); - } - - void stop() { - clock_gettime(CLOCK_REALTIME, &stopTime); - sec = stopTime.tv_sec - startTime.tv_sec; - nsec = stopTime.tv_nsec - startTime.tv_nsec; - if (nsec < 0) { - sec -= 1; - nsec += 1000000000L; - } - time += (double)sec + (double)nsec / 1000000000.0; - ntime += sec * 1000000000L + nsec; - } - - friend std::ostream &operator<<(std::ostream &os, Timer &t) { - auto time = t.time; - if (time < 1.0) { - time *= 1000.0; - os << std::setprecision(6) << time << " (ms)"; - return os; - } - if (time < 60.0) { - os << std::setprecision(6) << time << " (s)"; - return os; - } - time /= 60.0; - if (time < 60.0) { - os << std::setprecision(6) << time << " (m)"; - return os; - } - time /= 60.0; - os << std::setprecision(6) << time << " (h)"; - return os; - } - - struct timespec startTime; - struct timespec stopTime; - - int64_t sec; - int64_t nsec; - int64_t ntime; // nano second - double time; // second - }; - } // namespace NGT diff --git a/lib/NGT/GraphOptimizer.h b/lib/NGT/GraphOptimizer.h index 2e5d21c..046e0a4 100644 --- a/lib/NGT/GraphOptimizer.h +++ b/lib/NGT/GraphOptimizer.h @@ -14,6 +14,8 @@ // limitations under the License. // +#pragma once + #include "GraphReconstructor.h" #include "Optimizer.h" diff --git a/lib/NGT/NGTQ/Capi.cpp b/lib/NGT/NGTQ/Capi.cpp index 35c041f..84bbff2 100644 --- a/lib/NGT/NGTQ/Capi.cpp +++ b/lib/NGT/NGTQ/Capi.cpp @@ -318,8 +318,9 @@ bool qbg_build_index(const char *index_path, QBGBuildParameters *parameters, QBG hierarchicalKmeans.numOfSecondClusters = parameters->number_of_second_clusters; hierarchicalKmeans.numOfThirdClusters = parameters->number_of_third_clusters; hierarchicalKmeans.numOfObjects = 0; - hierarchicalKmeans.threeLayerClustering = true; - hierarchicalKmeans.silence = true; + //-/hierarchicalKmeans.threeLayerClustering = true; + hierarchicalKmeans.clusteringType = QBG::HierarchicalKmeans::ClusteringTypeThreeLayer; + hierarchicalKmeans.verbose = false; try { hierarchicalKmeans.clustering(index_path); @@ -341,16 +342,16 @@ bool qbg_build_index(const char *index_path, QBGBuildParameters *parameters, QBG optimizer.iteration = parameters->rotation_iteration; optimizer.clusterIteration = parameters->subvector_iteration; optimizer.clusterSizeConstraint = false; - optimizer.nOfMatrices = parameters->number_of_matrices; - optimizer.seedStartObjectSizeRate = 0.1; - optimizer.seedStep = 2; + optimizer.numberOfMatrices = parameters->number_of_matrices; + optimizer.seedNumberOfSteps = 2; + optimizer.seedStep = 10; optimizer.reject = 0.9; optimizer.timelimit = 24 * 2; optimizer.timelimit *= 60.0 * 60.0; optimizer.rotation = parameters->rotation; optimizer.repositioning = parameters->repositioning; optimizer.globalType = QBG::Optimizer::GlobalTypeNone; - optimizer.silence = true; + optimizer.verbose = false; try { auto nthreads = omp_get_max_threads(); @@ -363,8 +364,8 @@ bool qbg_build_index(const char *index_path, QBGBuildParameters *parameters, QBG } try { - auto silence = true; - QBG::Index::build(index_path, silence); + auto verbose = false; + QBG::Index::build(index_path, verbose); } catch (NGT::Exception &err) { std::stringstream ss; ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); @@ -406,7 +407,7 @@ static bool qbg_search_index_(QBG::Index* pindex, std::vector &query, QBG sc.setEdgeSize(param.number_of_edges); sc.setGraphExplorationSize(param.number_of_explored_blobs); - pindex->searchBlobGraph(sc); + pindex->search(sc); return true; } diff --git a/lib/NGT/NGTQ/HierarchicalKmeans.cpp b/lib/NGT/NGTQ/HierarchicalKmeans.cpp new file mode 100644 index 0000000..2f1dc88 --- /dev/null +++ b/lib/NGT/NGTQ/HierarchicalKmeans.cpp @@ -0,0 +1,641 @@ +// +// Copyright (C) 2021 Yahoo Japan Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +#include "NGT/NGTQ/QuantizedBlobGraph.h" +#include "NGT/NGTQ/HierarchicalKmeans.h" + +QBG::HierarchicalKmeans::HierarchicalKmeans(QBG::BuildParameters ¶m) { +#ifdef NGTQ_QBG + setParameters(param.hierarchicalClustering); +#endif +} +QBG::HierarchicalKmeans::HierarchicalKmeans(QBG::HierarchicalClusteringParameters &hierarchicalClustering) { + setParameters(hierarchicalClustering); +} + +void QBG::HierarchicalKmeans::setParameters(QBG::HierarchicalClusteringParameters &hierarchicalClustering) { +#ifdef NGTQ_QBG + maxSize = hierarchicalClustering.maxSize; + numOfObjects = hierarchicalClustering.numOfObjects; + numOfClusters = hierarchicalClustering.numOfClusters; + numOfTotalClusters = hierarchicalClustering.numOfTotalClusters; + numOfTotalBlobs = hierarchicalClustering.numOfTotalBlobs; + clusterID = hierarchicalClustering.clusterID; + + initMode = hierarchicalClustering.initMode; + + numOfRandomObjects = hierarchicalClustering.numOfRandomObjects; + + numOfFirstObjects = hierarchicalClustering.numOfFirstObjects; + numOfFirstClusters = hierarchicalClustering.numOfFirstClusters; + numOfSecondObjects = hierarchicalClustering.numOfSecondObjects; + numOfSecondClusters = hierarchicalClustering.numOfSecondClusters; + numOfThirdClusters = hierarchicalClustering.numOfThirdClusters; + numOfThirdObjects = hierarchicalClustering.numOfThirdObjects; + extractCentroid = hierarchicalClustering.extractCentroid; + + clusteringType = hierarchicalClustering.clusteringType; + epsilonExplorationSize = hierarchicalClustering.epsilonExplorationSize; + expectedRecall = hierarchicalClustering.expectedRecall; + verbose = hierarchicalClustering.verbose; +#endif +} + +void QBG::HierarchicalKmeans::initialize() { +#ifdef NGTQ_QBG + HierarchicalClusteringParameters params; + setParameters(params); +#endif +} + +#ifdef NGTQ_QBG +void QBG::HierarchicalKmeans::treeBasedTopdownClustering(std::string prefix, QBG::Index &index, uint32_t rootID, std::vector &object, std::vector &nodes, NGT::Clustering &clustering) { + auto &quantizer = static_cast&>(index.getQuantizer()); + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + QBGObjectList &objectList = quantizer.objectList; + NGT::Timer timer; + timer.start(); + std::vector batch; + std::vector> exceededLeaves; + size_t nleaves = 1; + size_t nOfThreads = 32; + for (size_t id = 1; id <= numOfObjects; id++) { + if (id % (numOfObjects / 100) == 0) { + timer.stop(); + std::cerr << "# of processed objects=" << id << " " << id * 100 / numOfObjects << "% " << timer << " # of leaves=" << nleaves << std::endl; + timer.start(); + } + batch.push_back(id); + if (batch.size() > 100000) { + size_t kmeansBatchSize = nleaves < nOfThreads ? nleaves : nOfThreads; + hierarchicalKmeansBatch(batch, exceededLeaves, rootID, object, objectList, objectSpace, nodes, + clustering, maxSize, nleaves, kmeansBatchSize); + + } + } + hierarchicalKmeansBatch(batch, exceededLeaves, rootID, object, objectList, objectSpace, nodes, + clustering, maxSize, nleaves, 0); + + if (numOfTotalClusters != 0) { + NGT::Timer timer; + timer.start(); + size_t numOfLeaves = 0; + for (auto node : nodes) { + if (node->leaf) { + numOfLeaves++; + } + } + std::cerr << "# of nodes=" << nodes.size() << std::endl; + std::cerr << "# of leaves=" << numOfLeaves << std::endl; + std::cerr << "clustering for quantization." << std::endl; + hierarchicalKmeansWithNumberOfClustersInParallel(numOfTotalClusters, numOfObjects, numOfLeaves, + objectList, objectSpace, nodes, initMode); + if (numOfTotalBlobs != 0) { + NGT::Timer timer; + timer.start(); + size_t numOfLeaves = 0; + for (auto node : nodes) { + if (node->leaf) { + numOfLeaves++; + } + } + std::cerr << "# of leaves=" << numOfLeaves << ":" << numOfTotalClusters << std::endl; + if (numOfLeaves != numOfTotalClusters) { + std::cerr << "# of leaves is invalid " << numOfLeaves << ":" << numOfTotalClusters << std::endl; + abort(); + } + { + std::ofstream of(prefix + QBG::Index::getSecondCentroidSuffix()); + extractCentroids(of, nodes); + } + std::vector qNodeIDs; + for (uint32_t nid = 0; nid < nodes.size(); nid++) { + if (nodes[nid]->leaf) { + qNodeIDs.push_back(nid); + } + } + std::cerr << "clustering to make blobs." << std::endl; + hierarchicalKmeansWithNumberOfClustersInParallel(numOfTotalBlobs, numOfObjects, numOfTotalClusters, + objectList, objectSpace, nodes, initMode); + { + std::ofstream of(prefix + QBG::Index::get3rdTo2ndSuffix()); + extractBtoQIndex(of, nodes, qNodeIDs); + } + } + } + +} + +void QBG::HierarchicalKmeans::threeLayerClustering(std::string prefix, QBG::Index &index) { + auto &quantizer = static_cast&>(index.getQuantizer()); + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + { + std::cerr << "Three layer clustering..." << std::endl; + std::cerr << "HiearchicalKmeans::clustering: # of clusters=" << numOfThirdClusters << ":" << index.getQuantizer().property.globalCentroidLimit << std::endl; + if (index.getQuantizer().objectList.size() <= 1) { + NGTThrowException("HierarchicelKmeans: No objects"); + } + if (numOfThirdClusters == 0) { + if (index.getQuantizer().property.globalCentroidLimit == 0) { + numOfThirdClusters = index.getQuantizer().objectList.size() / 1000; + numOfThirdClusters = numOfThirdClusters == 0 ? 1 : numOfThirdClusters; + numOfThirdClusters = numOfThirdClusters > 1000000 ? 1000000 : numOfThirdClusters; + } else { + numOfThirdClusters = index.getQuantizer().property.globalCentroidLimit; + } + } + if (numOfThirdClusters != 0 && index.getQuantizer().property.globalCentroidLimit != 0 && + numOfThirdClusters != index.getQuantizer().property.globalCentroidLimit) { + } + auto &quantizer = static_cast&>(index.getQuantizer()); + QBGObjectList &objectList = quantizer.objectList; + if (numOfObjects == 0) { + numOfObjects = objectList.size() - 1; + } + if (numOfThirdObjects > numOfObjects) { + numOfThirdObjects = numOfObjects; + } + + if (numOfThirdClusters == 0 || numOfObjects == 0) { + NGTThrowException("numOfThirdClusters or numOfObjects are zero"); + } + numOfThirdObjects = numOfThirdObjects == 0 ? numOfObjects : numOfThirdObjects; + numOfSecondClusters = numOfSecondClusters == 0 ? numOfThirdClusters : numOfSecondClusters; + numOfFirstClusters = numOfFirstClusters == 0 ? static_cast(sqrt(numOfSecondClusters)) : numOfFirstClusters; + numOfSecondObjects = numOfSecondObjects == 0 ? numOfSecondClusters * 100 : numOfSecondObjects; + numOfSecondObjects = numOfSecondObjects > numOfObjects ? numOfObjects : numOfSecondObjects; + numOfFirstObjects = numOfFirstObjects == 0 ? numOfFirstClusters * 2000 : numOfFirstObjects; + numOfFirstObjects = numOfFirstObjects > numOfSecondObjects ? numOfSecondObjects : numOfFirstObjects; + + + if (numOfFirstObjects < numOfFirstClusters) { + std::stringstream msg; + msg << "# of objects for the first should be larger than # of the first clusters. " << numOfFirstObjects << ":" << numOfFirstClusters; + NGTThrowException(msg); + } + if (numOfFirstClusters > numOfSecondClusters) { + std::stringstream msg; + msg << "# of the first clusters should be larger than or equal to # of the second clusters. " << numOfFirstClusters << ":" << numOfSecondClusters; + NGTThrowException(msg); + } + if (numOfSecondClusters > numOfThirdClusters) { + std::stringstream msg; + msg << "# of the third clusters should be larger than or equal to # of the second clusters. " << numOfSecondClusters << ":" << numOfThirdClusters; + NGTThrowException(msg); + } + if (numOfFirstClusters > numOfSecondClusters) { + std::stringstream msg; + msg << "# of the second clusters should be larger than # of the first clusters. " << numOfFirstClusters << ":" << numOfSecondClusters; + NGTThrowException(msg); + } + + NGT::Clustering firstClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 300); + float clusterSizeConstraint = 5.0; + firstClustering.setClusterSizeConstraintCoefficient(clusterSizeConstraint); + std::cerr << "size constraint=" << clusterSizeConstraint << std::endl; + std::vector> vectors; + vectors.reserve(numOfFirstObjects); + std::cerr << "loading objects..." << std::endl; + NGT::Timer timer; + std::vector obj; + for (size_t id = 1; id <= numOfFirstObjects; id++) { + if (id % 1000000 == 0) { + std::cerr << "# of prcessed objects is " << id << std::endl; + } + if (!objectList.get(id, obj, &objectSpace)) { + std::stringstream msg; + msg << "qbg: Cannot get object. ID=" << id; + NGTThrowException(msg); + } + vectors.push_back(obj); + } + std::cerr << "loading objects time=" << timer << std::endl; + std::cerr << "clustering for the first... (" << vectors.size() << "->" << numOfFirstClusters << ") " << std::endl; + std::vector firstClusters; + + timer.start(); + firstClustering.kmeans(vectors, numOfFirstClusters, firstClusters); + timer.stop(); + std::cerr << "end of clustering for the first. # of clusters=" << firstClusters.size() << " time=" << timer << std::endl; + + std::vector> otherVectors; + timer.start(); + std::cerr << "assign for the second. (" << numOfFirstObjects << "->" << numOfSecondObjects << ")..." << std::endl; + assign(firstClusters, numOfFirstObjects + 1, numOfSecondObjects, objectSpace, objectList); + timer.stop(); + std::cerr << "end of assign for the second. time=" << timer << ", vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + + std::cerr << "subclustering for the second (" << numOfSecondClusters << ")..." << std::endl; + std::vector secondClusters; + timer.start(); + subclustering(firstClusters, numOfSecondClusters, numOfSecondObjects, objectSpace, objectList, initMode, secondClusters); + timer.stop(); + std::cerr << "end of subclustering for the second. time=" << timer << ", vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + timer.start(); + NGT::Clustering::clearMembers(secondClusters); + std::cerr << "assign for the third. (" << 1 << "->" << numOfThirdObjects << ")..." << std::endl; + assignWithNGT(secondClusters, 1, numOfThirdObjects, objectSpace, objectList, epsilonExplorationSize, expectedRecall); + { + size_t noOfRemovedClusters = NGT::Clustering::removeEmptyClusters(secondClusters); + if (noOfRemovedClusters != 0) { + std::cerr << "Clustering: Warning. # of removed clusters=" << noOfRemovedClusters << std::endl; + } + } + timer.stop(); + std::cerr << "end of assign for the third. time=" << timer << ", vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + std::cerr << "subclustering for the third (" << numOfThirdClusters << ")..." << std::endl; + std::vector> thirdClusters; + timer.start(); + subclustering(secondClusters, numOfThirdClusters, numOfThirdObjects, objectSpace, objectList, initMode, thirdClusters); + timer.stop(); + std::cerr << "end of subclustering for the third. time=" << timer << ", vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + + std::vector thirdFlatClusters; + flattenClusters(secondClusters, thirdClusters, numOfThirdClusters, thirdFlatClusters); + + timer.start(); + NGT::Clustering::clearMembers(thirdFlatClusters); + std::cerr << "assign all for the third (" << 1 << "-" << numOfObjects << ")..." << std::endl; + assignWithNGT(thirdFlatClusters, 1, numOfObjects, objectSpace, objectList, epsilonExplorationSize, expectedRecall); + timer.stop(); + std::cerr << "end of assign all for the third. time=" << timer << ", vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + + { + std::vector bqindex; + size_t idx = 0; + for (size_t idx1 = 0; idx1 < thirdClusters.size(); idx1++) { + for (size_t idx2 = 0; idx2 < thirdClusters[idx1].size(); idx2++, idx++) { + if (thirdClusters[idx1][idx2].members.size() == 0) { + std::stringstream msg; + msg << "Fatal error! found an empty cluster in thirdClusters."; + NGTThrowException(msg); + } + if (thirdFlatClusters[idx].members.size() == 0) { + std::cerr << "warning. found an empty cluster in thirdFlatClusters. " << idx << std::endl; + } else { + bqindex.push_back(idx1); + } + } + } + std::cerr << "save the 3rd to the 2nd index..." << std::endl; + NGT::Clustering::saveVector(prefix + QBG::Index::get3rdTo2ndSuffix(), bqindex); + } + + std::cerr << "save quantization centroid" << std::endl; + NGT::Clustering::saveClusters(prefix + QBG::Index::getSecondCentroidSuffix(), secondClusters); + + std::cerr << "save the third centroid..." << std::endl; + auto skipEmptyClusters = true; + NGT::Clustering::saveClusters(prefix + QBG::Index::getThirdCentroidSuffix(), thirdFlatClusters, skipEmptyClusters); + { + std::vector cindex(numOfObjects); + size_t idx = 0; + for (size_t cidx = 0; cidx < thirdFlatClusters.size(); cidx++) { + if (thirdFlatClusters[cidx].members.size() == 0) { + continue; + } + for (auto mit = thirdFlatClusters[cidx].members.begin(); mit != thirdFlatClusters[cidx].members.end(); ++mit) { + size_t vid = (*mit).vectorID; + cindex[vid] = idx; + } + idx++; + } + std::cerr << "save index... " << cindex.size() << std::endl; + NGT::Clustering::saveVector(prefix + QBG::Index::getObjTo3rdSuffix(), cindex); + } + std::cerr << "end of clustering" << std::endl; + return; + } + +} + +void QBG::HierarchicalKmeans::twoPlusLayerClustering(std::string prefix, QBG::Index &index) { + auto &quantizer = static_cast&>(index.getQuantizer()); + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + { + std::cerr << "Two layer clustering..." << std::endl; + std::cerr << "HiearchicalKmeans::clustering: # of clusters=" << numOfThirdClusters << ":" << index.getQuantizer().property.globalCentroidLimit << std::endl; + if (index.getQuantizer().objectList.size() <= 1) { + NGTThrowException("HierarchicelKmeans: No objects"); + } + if (numOfThirdClusters == 0) { + if (index.getQuantizer().property.globalCentroidLimit == 0) { + numOfThirdClusters = index.getQuantizer().objectList.size() / 1000; + numOfThirdClusters = numOfThirdClusters == 0 ? 1 : numOfThirdClusters; + numOfThirdClusters = numOfThirdClusters > 1000000 ? 1000000 : numOfThirdClusters; + } else { + numOfThirdClusters = index.getQuantizer().property.globalCentroidLimit; + } + } + if (numOfThirdClusters != 0 && index.getQuantizer().property.globalCentroidLimit != 0 && + numOfThirdClusters != index.getQuantizer().property.globalCentroidLimit) { + } + auto &quantizer = static_cast&>(index.getQuantizer()); + QBGObjectList &objectList = quantizer.objectList; + if (numOfObjects == 0) { + numOfObjects = objectList.size() - 1; + } + if (numOfThirdObjects > numOfObjects) { + numOfThirdObjects = numOfObjects; + } + + std::cerr << "The first layer. " << numOfFirstClusters << ":" << numOfFirstObjects << std::endl; + if (numOfThirdClusters == 0 || numOfObjects == 0) { + NGTThrowException("numOfThirdClusters or numOfObjects are zero"); + } + numOfThirdObjects = numOfThirdObjects == 0 ? numOfObjects : numOfThirdObjects; + numOfSecondClusters = numOfSecondClusters == 0 ? numOfThirdClusters : numOfSecondClusters; + numOfFirstClusters = numOfFirstClusters == 0 ? static_cast(sqrt(numOfSecondClusters)) : numOfFirstClusters; + numOfSecondObjects = numOfSecondObjects == 0 ? numOfSecondClusters * 100 : numOfSecondObjects; + numOfSecondObjects = numOfSecondObjects > numOfObjects ? numOfObjects : numOfSecondObjects; + numOfFirstObjects = numOfFirstObjects == 0 ? numOfFirstClusters * 2000 : numOfFirstObjects; + numOfFirstObjects = numOfFirstObjects > numOfSecondObjects ? numOfSecondObjects : numOfFirstObjects; + if (numOfFirstObjects < numOfFirstClusters) { + std::stringstream msg; + msg << "# of objects for the first should be larger than # of the first clusters. " << numOfFirstObjects << ":" << numOfFirstClusters; + NGTThrowException(msg); + } + if (numOfFirstClusters > numOfSecondClusters) { + std::stringstream msg; + msg << "# of the first clusters should be larger than or equal to # of the second clusters. " << numOfFirstClusters << ":" << numOfSecondClusters; + NGTThrowException(msg); + } + if (numOfSecondClusters > numOfThirdClusters) { + std::stringstream msg; + msg << "# of the third clusters should be larger than or equal to # of the second clusters. " << numOfSecondClusters << ":" << numOfThirdClusters; + NGTThrowException(msg); + } + if (numOfFirstClusters > numOfSecondClusters) { + std::stringstream msg; + msg << "# of the second clusters should be larger than # of the first clusters. " << numOfFirstClusters << ":" << numOfSecondClusters; + NGTThrowException(msg); + } + + std::cerr << "Two layer clustering. " << numOfFirstClusters << ":" << numOfFirstObjects << "," << numOfSecondClusters << ":" << numOfSecondObjects << "," << numOfThirdClusters << ":" << numOfThirdObjects << " " << numOfObjects << std::endl; + + NGT::Clustering firstClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 300); + float clusterSizeConstraint = 5.0; + firstClustering.setClusterSizeConstraintCoefficient(clusterSizeConstraint); + std::cerr << "size constraint=" << clusterSizeConstraint << std::endl; + std::vector> vectors; + size_t reserveSize = numOfThirdClusters > numOfFirstObjects ? numOfThirdClusters : numOfFirstObjects; + vectors.reserve(reserveSize); + std::cerr << "loading objects..." << std::endl; + NGT::Timer timer; + std::vector obj; + for (size_t id = 1; id <= numOfFirstObjects; id++) { + if (id % 1000000 == 0) { + std::cerr << "# of prcessed objects is " << id << std::endl; + } + if (!objectList.get(id, obj, &objectSpace)) { + std::stringstream msg; + msg << "qbg: Cannot get object. ID=" << id; + NGTThrowException(msg); + } + vectors.push_back(obj); + } + std::cerr << "loading objects time=" << timer << " vmsize=" << NGT::Common::getProcessVmSize() << std::endl; + std::cerr << "kmeans clustering... " << vectors.size() << " to " << numOfFirstClusters << std::endl; + std::vector firstClusters; + + timer.start(); + firstClustering.kmeans(vectors, numOfFirstClusters, firstClusters); + timer.stop(); + std::cerr << "kmeans clustering. # of clusters=" << firstClusters.size() << " time=" << timer << " vmsize=" << NGT::Common::getProcessVmSize() << std::endl; + timer.start(); + std::cerr << "assign for the third (" << numOfFirstObjects << "-" << numOfThirdObjects << ")..." << std::endl; + assign(firstClusters, numOfFirstObjects + 1, numOfThirdObjects, objectSpace, objectList); + timer.stop(); + std::cerr << "assign for the third. time=" << timer << " vmsize=" << NGT::Common::getProcessVmSize() << std::endl; + + std::cerr << "subclustering for the third (" << numOfThirdClusters << ", " << numOfThirdObjects << ")..." << std::endl; + std::vector thirdFlatClusters; + timer.start(); + subclustering(firstClusters, numOfThirdClusters, numOfThirdObjects, objectSpace, objectList, initMode, thirdFlatClusters); + timer.stop(); + std::cerr << "subclustering for the third. time=" << timer << ", vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + + std::cerr << "assign all for the third (" << numOfThirdObjects << "-" << numOfObjects << ")..." << std::endl; + timer.start(); + assignWithNGT(thirdFlatClusters, numOfThirdObjects + 1, numOfObjects, objectSpace, objectList); + timer.stop(); + std::cerr << "assign all for the third. time=" << timer << ", vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + + std::vector secondClusters; + + std::cerr << "clustering for the second. # of objects=" << thirdFlatClusters.size() << " # of clusters=" << numOfSecondClusters << std::endl; + timer.start(); + vectors.clear(); + vectors.reserve(thirdFlatClusters.size()); + for (auto cit = thirdFlatClusters.begin(); cit != thirdFlatClusters.end(); ++cit) { + std::vector &v = (*cit).centroid; + vectors.push_back(v); + } + if (clusteringType == QBG::HierarchicalKmeans::ClusteringTypeTwoPlusOneLayer) { + std::cerr << "two + one layer clustering..." << std::endl; + NGT::Clustering clustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 1000); + clustering.kmeans(vectors, numOfSecondClusters, secondClusters); + } else if (clusteringType == QBG::HierarchicalKmeans::ClusteringTypeTwoPlusOneLayer) { + auto n = numOfSecondObjects / numOfSecondClusters; + std::cerr << "two + one layer clustering with NGT... " << n << std::endl; + NGT::Clustering clustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithNGT, n); + clustering.kmeans(vectors, numOfSecondClusters, secondClusters); + } else if (clusteringType == QBG::HierarchicalKmeans::ClusteringTypeTwoPlusTwoLayer) { + std::cerr << "two + two layer clustering..." << std::endl; + twoLayerClustering(vectors, numOfSecondClusters, secondClusters, 100); + } else { + std::stringstream msg; + msg << "Invalid clustering type:" << clusteringType << std::endl; + NGTThrowException(msg); + } + timer.stop(); + std::cerr << "clustering for the second. time=" << timer << " vmsize=" << NGT::Common::getProcessVmSize() << std::endl; + NGT::Clustering::saveClusters(prefix + QBG::Index::getSecondCentroidSuffix(), secondClusters); + + timer.start(); + std::vector thirdPermutedClusters; + thirdPermutedClusters.reserve(thirdFlatClusters.size()); + for (size_t sidx = 0; sidx < secondClusters.size(); sidx++) { + for (auto mit = secondClusters[sidx].members.begin(); mit != secondClusters[sidx].members.end(); ++mit) { + thirdPermutedClusters.emplace_back(thirdFlatClusters[(*mit).vectorID]); + (*mit).vectorID = thirdPermutedClusters.size() - 1; + } + } + timer.stop(); + std::cerr << "permuting cluster time=" << timer << " vmsize=" << NGT::Common::getProcessVmSize() << std::endl; + + std::vector cindex(numOfObjects); + for (size_t cidx = 0; cidx < thirdPermutedClusters.size(); cidx++) { + for (auto mit = thirdPermutedClusters[cidx].members.begin(); mit != thirdPermutedClusters[cidx].members.end(); ++mit) { + size_t vid = (*mit).vectorID; + cindex[vid] = cidx; + } + } + std::cerr << "save index... " << cindex.size() << std::endl; + NGT::Clustering::saveVector(prefix + QBG::Index::getObjTo3rdSuffix(), cindex); + + std::vector bqindex(thirdPermutedClusters.size()); + for (size_t idx1 = 0; idx1 < secondClusters.size(); idx1++) { + for (size_t idx2 = 0; idx2 < secondClusters[idx1].members.size(); idx2++) { + bqindex[secondClusters[idx1].members[idx2].vectorID] = idx1; + } + } + std::cerr << "save bqindex..." << std::endl; + NGT::Clustering::saveVector(prefix + QBG::Index::get3rdTo2ndSuffix(), bqindex); + std::cerr << "save the third centroid" << std::endl; + NGT::Clustering::saveClusters(prefix + QBG::Index::getThirdCentroidSuffix(), thirdPermutedClusters); + + } + + std::cerr << "end of clustering" << std::endl; + return; +} + +void QBG::HierarchicalKmeans::multiLayerClustering(QBG::Index &index, std::string prefix, std::string objectIDsFile) { + NGT::Clustering::ClusteringType clusteringType = NGT::Clustering::ClusteringTypeKmeansWithoutNGT; + + uint32_t rootID = 0; + std::vector nodes; + nodes.push_back(new HKLeafNode); + + std::vector object; + size_t iteration = 1000; + NGT::Clustering clustering(initMode, clusteringType, iteration, numOfClusters); + auto &quantizer = static_cast&>(index.getQuantizer()); + QBGObjectList &objectList = quantizer.objectList; + if (objectIDsFile.empty()) { + treeBasedTopdownClustering(prefix, index, rootID, object, nodes, clustering); + } else { + std::cerr << "Cluster ID=" << clusterID << std::endl; + if (clusterID < 0) { + std::stringstream msg; + msg << "Any target cluster ID is not specified."; + NGTThrowException(msg); + } + std::ifstream objectIDs(objectIDsFile); + if (!objectIDs) { + std::stringstream msg; + msg << "Cannot open the object id file. " << objectIDsFile; + NGTThrowException(msg); + } + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + uint32_t id = 1; + int32_t cid; + size_t ccount = 0; + while (objectIDs >> cid) { + std::cerr << cid << std::endl; + if (id % 100000 == 0) { + std::cerr << "# of processed objects=" << id << std::endl; + } + if (cid == -1) { + continue; + } + if (cid == clusterID) { + ccount++; + hierarchicalKmeans(id, rootID, object, objectList, objectSpace, nodes, clustering, maxSize); + } + id++; + } + } + size_t objectCount = 0; + if (prefix.empty()) { + objectCount = extractCentroids(std::cout, nodes); + } else { + { + std::ofstream of(prefix + QBG::Index::getThirdCentroidSuffix()); + objectCount = extractCentroids(of, nodes); + } + { + std::ofstream of(prefix + QBG::Index::getObjTo3rdSuffix()); + extractIndex(of, nodes, numOfObjects); + } + if (numOfFirstObjects > 0) { + std::ofstream btoqof(prefix + QBG::Index::get3rdTo2ndSuffix()); + std::ofstream qcof(prefix + QBG::Index::getSecondCentroidSuffix()); + extractBtoQAndQCentroid(btoqof, qcof, nodes, numOfThirdClusters); + } + if (numOfRandomObjects > 0) { + std::ofstream of(prefix + "_random_object.tsv"); + if (extractCentroid) { + extractRandomObjectsFromEachBlob(of, nodes, numOfObjects, numOfRandomObjects - 1, quantizer, extractCentroid); + } else { + extractRandomObjectsFromEachBlob(of, nodes, numOfObjects, numOfRandomObjects, quantizer, extractCentroid); + } + } + } + if (objectCount != numOfObjects) { + std::cerr << "# of objects is invalid. " << objectCount << ":" << numOfObjects << std::endl; + } +} + +void QBG::HierarchicalKmeans::clustering(std::string indexPath, std::string prefix, std::string objectIDsFile) { + NGT::StdOstreamRedirector redirector(!verbose); + redirector.begin(); + + std::cerr << "The specified params=FC:" << numOfFirstClusters << ":FO:" << numOfFirstObjects + << ",SC:" << numOfSecondClusters << ":SO:" << numOfSecondObjects + << ",TC:" << numOfThirdClusters << ":TO:" << numOfThirdObjects << ",O:" << numOfObjects << std::endl; + + bool readOnly = false; + QBG::Index index(indexPath, readOnly); + if (index.getQuantizer().objectList.size() <= 1) { + NGTThrowException("No objects in the index."); + } + if (clusteringType == QBG::HierarchicalKmeans::ClusteringTypeMultiLayer) { + try { + multiLayerClustering(index, prefix, objectIDsFile); + } catch(NGT::Exception &err) { + redirector.end(); + throw err; + } + } else { + std::cerr << "three layer clustering... " << std::endl; + try { + if (numOfObjects == 0) { + numOfObjects = index.getQuantizer().objectList.size() - 1; + } + if (prefix.empty()) { + std::cerr << "Prefix is not specified." << std::endl; + prefix = indexPath + "/" + QBG::Index::getWorkspaceName(); + try { + NGT::Index::mkdir(prefix); + } catch(...) {} + prefix +="/" + QBG::Index::getHierarchicalClusteringPrefix(); + std::cerr << prefix << " is used" << std::endl; + } + auto &quantizer = static_cast&>(index.getQuantizer()); + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + size_t paddedDimension = objectSpace.getPaddedDimension(); + size_t dimension = objectSpace.getDimension(); + if (paddedDimension != dimension) { + std::cerr << "HierarachicalKmeans: Warning! Dimensions are inconsistent. Dimension=" << paddedDimension << ":" << dimension << std::endl; + } + if (clusteringType == QBG::HierarchicalKmeans::ClusteringTypeThreeLayer) { + threeLayerClustering(prefix, index); + } else { + twoPlusLayerClustering(prefix, index); + } + } catch(NGT::Exception &err) { + redirector.end(); + throw err; + } + } + redirector.end(); +} + +#endif + diff --git a/lib/NGT/NGTQ/HierarchicalKmeans.h b/lib/NGT/NGTQ/HierarchicalKmeans.h index 2e51905..c5d6007 100644 --- a/lib/NGT/NGTQ/HierarchicalKmeans.h +++ b/lib/NGT/NGTQ/HierarchicalKmeans.h @@ -17,12 +17,26 @@ #pragma once #include "Quantizer.h" +#include "NGT/GraphOptimizer.h" namespace QBG { + class Index; + class BuildParameters; + class HierarchicalClusteringParameters; + class OptimizationParameters; + class HierarchicalKmeans { public: typedef NGTQ::Quantizer::ObjectList QBGObjectList; + enum ClusteringType { + ClusteringTypeMultiLayer = 0, + ClusteringTypeThreeLayer = 1, + ClusteringTypeTwoPlusOneLayer = 2, + ClusteringTypeTwoPlusOneLayerWithNGT = 3, + ClusteringTypeTwoPlusTwoLayer = 4 + }; + class HKNode { public: bool leaf; @@ -41,34 +55,12 @@ namespace QBG { std::vector>> children; }; - HierarchicalKmeans() { - silence = true; - } + HierarchicalKmeans() { initialize(); } + HierarchicalKmeans(QBG::BuildParameters ¶m); + HierarchicalKmeans(QBG::HierarchicalClusteringParameters &hierarchicalClustering); - HierarchicalKmeans(QBG::BuildParameters ¶m) { -#ifdef NGTQ_QBG - maxSize = param.hierarchicalClustering.maxSize; - numOfObjects = param.hierarchicalClustering.numOfObjects; - numOfClusters = param.hierarchicalClustering.numOfClusters; - numOfTotalClusters = param.hierarchicalClustering.numOfTotalClusters; - numOfTotalBlobs = param.hierarchicalClustering.numOfTotalBlobs; - clusterID = param.hierarchicalClustering.clusterID; - - initMode = param.hierarchicalClustering.initMode; - - numOfRandomObjects = param.hierarchicalClustering.numOfRandomObjects; - - numOfFirstObjects = param.hierarchicalClustering.numOfFirstObjects; - numOfFirstClusters = param.hierarchicalClustering.numOfFirstClusters; - numOfSecondObjects = param.hierarchicalClustering.numOfSecondObjects; - numOfSecondClusters = param.hierarchicalClustering.numOfSecondClusters; - numOfThirdClusters = param.hierarchicalClustering.numOfThirdClusters; - extractCentroid = param.hierarchicalClustering.extractCentroid; - - threeLayerClustering = param.hierarchicalClustering.threeLayerClustering; - silence = param.silence; -#endif - } + void setParameters(QBG::HierarchicalClusteringParameters &hierarchicalClustering); + void initialize(); static int32_t searchLeaf(std::vector &nodes, int32_t rootID, float *object) { auto nodeID = rootID; @@ -403,7 +395,7 @@ namespace QBG { leafID++; } } -} + } static void hierarchicalKmeans(uint32_t id, int32_t rootID, std::vector &object, @@ -643,7 +635,9 @@ namespace QBG { #ifndef MULTIPLE_OBJECT_LISTS void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, - NGT::Clustering::InitializationMode initMode, std::vector> &lowerClusters) { + NGT::Clustering::InitializationMode initMode, + std::vector> &lowerClusters, + size_t maximumIteration = 1000) { std::vector nPartialClusters(upperClusters.size()); auto numOfRemainingClusters = numOfLowerClusters; auto numOfRemainingVectors = numOfObjects; @@ -685,7 +679,7 @@ namespace QBG { std::cerr << "the sizes of members are not consistent" << std::endl; abort(); } - NGT::Clustering lowerClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 1000); + NGT::Clustering lowerClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, maximumIteration); lowerClustering.kmeans(partialVectors, nPartialClusters[idx], lowerClusters[idx]); if (nPartialClusters[idx] != lowerClusters[idx].size()) { std::cerr << "the sizes of cluster members are not consistent" << std::endl; @@ -704,8 +698,9 @@ namespace QBG { } void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, - NGT::Clustering::InitializationMode initMode, std::vector &flatLowerClusters) { - + NGT::Clustering::InitializationMode initMode, + std::vector &flatLowerClusters, + size_t maximumIteration = 1000) { std::vector> lowerClusters; subclustering(upperClusters, numOfLowerClusters, numOfObjects, objectSpace, objectList, initMode, lowerClusters); @@ -716,18 +711,27 @@ namespace QBG { #else static void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, - NGT::Clustering::InitializationMode initMode, std::vector> &lowerClusters) { + NGT::Clustering::InitializationMode initMode, + std::vector> &lowerClusters, + size_t maximumIteration = 1000) { + NGT::Timer timer; + timer.start(); std::vector nPartialClusters(upperClusters.size()); - auto numOfRemainingClusters = numOfLowerClusters; - auto numOfRemainingVectors = numOfObjects; + int numOfRemainingClusters = numOfLowerClusters; + int numOfRemainingVectors = numOfObjects; size_t ts = 0; for (size_t idx = 0; idx < upperClusters.size(); idx++) { - size_t ncs = round(static_cast(upperClusters[idx].members.size()) / numOfRemainingVectors * - numOfRemainingClusters); - ncs = ncs == 0 ? 1 : ncs; + float ncsf = static_cast(upperClusters[idx].members.size()) / + numOfRemainingVectors * + (numOfRemainingClusters - (upperClusters.size() - idx)); + ncsf += 1.0; + int ncs = round(ncsf); numOfRemainingVectors -= upperClusters[idx].members.size(); - if (numOfRemainingClusters >= ncs) { - numOfRemainingClusters -= ncs; + numOfRemainingClusters -= ncs; + if (numOfRemainingClusters < 0) { + std::stringstream msg; + msg << " subclustering: Internal error! " << numOfRemainingClusters << ":" << idx; + NGTThrowException(msg); } nPartialClusters[idx] = ncs; ts += ncs; @@ -737,47 +741,69 @@ namespace QBG { std::cerr << "numOfRemainingVectors=" << numOfRemainingVectors << std::endl; std::cerr << "upperClusters=" << upperClusters.size() << std::endl; std::cerr << "total=" << ts << ":" << numOfLowerClusters << std::endl; - if (ts < numOfLowerClusters || numOfRemainingClusters != 0) { - std::cerr << "subclustering: Internal error! " << std::endl; - exit(1); + std::cerr << "max iteration=" << maximumIteration << std::endl; + timer.stop(); + std::cerr << "time=" << timer << std::endl; + timer.restart(); + if (ts != numOfLowerClusters || numOfRemainingClusters != 0) { + std::stringstream msg; + msg << "subclustering: Internal error! " << ts << ":" << numOfLowerClusters + << ":" << numOfRemainingClusters << std::endl; + NGTThrowException(msg); } auto nthreads = omp_get_max_threads(); if (!objectList.openMultipleStreams(nthreads)) { - std::cerr << "Cannot open multiple streams." << std::endl; - abort(); + std::stringstream msg; + msg << "subclustering: Internal error! Cannot open multiple streams. " << nthreads; + NGTThrowException(msg); } lowerClusters.resize(upperClusters.size()); + std::vector counters(nthreads, 0); #pragma omp parallel for schedule(dynamic) for (size_t idx = 0; idx < upperClusters.size(); idx++) { std::vector> partialVectors; partialVectors.reserve(upperClusters[idx].members.size()); std::vector obj; auto threadid = omp_get_thread_num(); - //#pragma omp critical { for (auto &m : upperClusters[idx].members) { if (threadid >= nthreads) { - std::cerr << "inner fatal error. # of threads=" << nthreads << ":" << threadid << std::endl; - exit(1); + std::stringstream msg; + msg << "subclustering: inner fatal error. # of threads=" << nthreads << ":" << threadid; + NGTThrowException(msg); } if (!objectList.get(threadid, m.vectorID + 1, obj, &objectSpace)) { - std::cerr << "subclustering: Fatal error! cannot get!!!! " << m.vectorID + 1 << std::endl; - abort(); + std::stringstream msg; + msg << "subclustering: Fatal error! cannot get!!!! " << m.vectorID + 1; + NGTThrowException(msg); } partialVectors.push_back(obj); } } if (upperClusters[idx].members.size() != partialVectors.size()) { - std::cerr << "the sizes of members are not consistent" << std::endl; - abort(); + std::stringstream msg; + msg << "inner fatal error. the sizes of members are inconsistent. " << upperClusters[idx].members.size() << ":" << partialVectors.size() << ":" << idx; + NGTThrowException(msg); } - NGT::Clustering lowerClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 1000); + NGT::Clustering lowerClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, maximumIteration); lowerClustering.kmeans(partialVectors, nPartialClusters[idx], lowerClusters[idx]); if (nPartialClusters[idx] != lowerClusters[idx].size()) { - std::cerr << "the sizes of cluster members are not consistent" << std::endl; - abort(); + std::cerr << "Warning: the sizes of cluster members are inconsistent. " << nPartialClusters[idx] << ":" << lowerClusters[idx].size() << ":" << idx << std::endl; + } + counters[threadid]++; + { + size_t cnt = 0; + for (auto c : counters) { + cnt += c; + } + if (cnt % ((upperClusters.size() < 20 ? 20 : upperClusters.size()) / 20) == 0) { + timer.stop(); + std::cerr << "subclustering: " << cnt << " clusters (" + << (cnt * 100 / upperClusters.size()) << "%) have been processed. time=" << timer << std::endl; + timer.restart(); + } } } size_t nc = 0; @@ -793,8 +819,9 @@ namespace QBG { static void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, - NGT::Clustering::InitializationMode initMode, std::vector &flatLowerClusters) { - + NGT::Clustering::InitializationMode initMode, + std::vector &flatLowerClusters, + size_t maximumIteration = 1000) { std::vector> lowerClusters; subclustering(upperClusters, numOfLowerClusters, numOfObjects, objectSpace, objectList, initMode, lowerClusters); @@ -804,6 +831,125 @@ namespace QBG { #endif + static void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, + NGT::Clustering::InitializationMode initMode, + std::vector> &lowerClusters, + std::vector> &vectors, + size_t maximumIteration = 1000) { + std::vector nPartialClusters(upperClusters.size()); + int numOfRemainingClusters = numOfLowerClusters; + int numOfRemainingVectors = numOfObjects; + size_t ts = 0; + for (size_t idx = 0; idx < upperClusters.size(); idx++) { + float ncsf = static_cast(upperClusters[idx].members.size()) / + numOfRemainingVectors * + (numOfRemainingClusters - (upperClusters.size() - idx)); + ncsf += 1.0; + int ncs = round(ncsf); + numOfRemainingVectors -= upperClusters[idx].members.size(); + numOfRemainingClusters -= ncs; + if (numOfRemainingClusters < 0) { + std::stringstream msg; + msg << "subclustering: Internal error! " << numOfRemainingClusters << idx; + NGTThrowException(msg); + } + nPartialClusters[idx] = ncs; + ts += ncs; + } + + std::cerr << "numOfRemainingClusters=" << numOfRemainingClusters << std::endl; + std::cerr << "numOfRemainingVectors=" << numOfRemainingVectors << std::endl; + std::cerr << "upperClusters=" << upperClusters.size() << std::endl; + std::cerr << "total=" << ts << ":" << numOfLowerClusters << std::endl; + if (ts < numOfLowerClusters || numOfRemainingClusters != 0) { + std::stringstream msg; + msg << "subclustering: Internal error! " << ts << ":" << numOfLowerClusters + << ":" << numOfRemainingClusters << std::endl; + NGTThrowException(msg); + } + + auto nthreads = omp_get_max_threads(); + + lowerClusters.resize(upperClusters.size()); +#pragma omp parallel for schedule(dynamic) + for (size_t idx = 0; idx < upperClusters.size(); idx++) { + std::vector> partialVectors; + partialVectors.reserve(upperClusters[idx].members.size()); + auto threadid = omp_get_thread_num(); + { + for (auto &m : upperClusters[idx].members) { + if (threadid >= nthreads) { + std::cerr << "inner fatal error. # of threads=" << nthreads << ":" << threadid << std::endl; + exit(1); + } + std::vector &obj = vectors[m.vectorID]; + partialVectors.push_back(obj); + } + } + if (upperClusters[idx].members.size() != partialVectors.size()) { + std::stringstream msg; + msg << "the sizes of members are inconsistent. " << upperClusters[idx].members.size() << ":" << partialVectors.size() << ":" << idx; + NGTThrowException(msg); + } + NGT::Clustering lowerClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, maximumIteration); + lowerClustering.kmeans(partialVectors, nPartialClusters[idx], lowerClusters[idx]); + if (nPartialClusters[idx] != lowerClusters[idx].size()) { + std::cerr << "Warning: the sizes of cluster members are inconsistent. " << nPartialClusters[idx] << ":" << lowerClusters[idx].size() << ":" << idx << std::endl; + } + } + size_t nc = 0; + size_t mc = 0; + for (auto &cs : lowerClusters) { + nc += cs.size(); + for (auto &c : cs) { + mc += c.members.size(); + } + } + std::cerr << "# of clusters=" << nc << " # of members=" << mc << std::endl; + } + + static void subclustering(std::vector &upperClusters, size_t numOfLowerClusters, size_t numOfObjects, + NGT::Clustering::InitializationMode initMode, + std::vector &flatLowerClusters, + std::vector> &vectors, + size_t maximumIteration = 1000) { + std::vector> lowerClusters; + subclustering(upperClusters, numOfLowerClusters, numOfObjects, initMode, lowerClusters, vectors); + + flattenClusters(upperClusters, lowerClusters, numOfLowerClusters, flatLowerClusters); + + } + + static void assign(std::vector &clusters, size_t beginID, size_t endID, std::vector> &vectors) { + + size_t count = 0; +#pragma omp parallel for + for (size_t id = beginID; id <= endID; id++) { + std::vector &obj = vectors[id]; + float min = std::numeric_limits::max(); + int minidx = -1; + for (size_t cidx = 0; cidx != clusters.size(); cidx++) { + auto d = NGT::PrimitiveComparator::compareL2(reinterpret_cast(obj.data()), + clusters[cidx].centroid.data(), obj.size()); + if (d < min) { + min = d; + minidx = cidx; + } + } + if (minidx < 0) { + std::cerr << "assign: Fatal error!" << std::endl; + abort(); + } +#pragma omp critical + { + clusters[minidx].members.push_back(NGT::Clustering::Entry(id, minidx, min)); + count++; + if (count % 1000000 == 0) { + std::cerr << "# of assigned objects=" << count << std::endl; + } + } + } + } static void assign(std::vector &clusters, size_t beginID, size_t endID, NGT::ObjectSpace &objectSpace, QBGObjectList &objectList) { @@ -851,8 +997,80 @@ namespace QBG { } + static float optimizeEpsilon(NGT::Index &index, size_t beginID, size_t endID, + size_t nOfObjects, + QBGObjectList &objectList, float expectedRecall, + NGT::ObjectSpace &objectSpace) { + std::cerr << "optimizeEpsilon: expectedRecall=" << expectedRecall << std::endl; + NGT::ObjectDistances groundTruth[endID - beginID]; +#pragma omp parallel for + for (size_t id = beginID; id < endID; id++) { + std::vector obj; +#ifdef MULTIPLE_OBJECT_LISTS + objectList.get(omp_get_thread_num(), id, obj, &objectSpace); +#else + objectList.get(id, obj, &objectSpace); +#endif + NGT::SearchQuery sc(obj); + sc.setResults(&groundTruth[id - beginID]); + sc.setSize(nOfObjects); + index.linearSearch(sc); + } + + float startEpsilon = 0.12; + float epsilon; + std::vector recall(endID - beginID, 0.0); + for (epsilon = startEpsilon; epsilon < 1.0; epsilon += 0.01) { + float totalRecall = 0.0; + NGT::ObjectDistances results[endID - beginID]; +#pragma omp parallel for + for (size_t id = beginID; id < endID; id++) { + if (recall[id - beginID] == 1.0) { + continue; + } + std::vector obj; +#ifdef MULTIPLE_OBJECT_LISTS + objectList.get(omp_get_thread_num(), id, obj, &objectSpace); +#else + objectList.get(id, obj, &objectSpace); +#endif + NGT::SearchQuery sc(obj); + sc.setResults(&results[id - beginID]); + sc.setSize(nOfObjects); + sc.setEpsilon(epsilon); + index.search(sc); + } + size_t notExactResultCount = 0; + for (size_t id = beginID; id < endID; id++) { + if (recall[id - beginID] == 1.0) { + totalRecall += 1.0; + continue; + } + notExactResultCount++; + size_t count = 0; + NGT::ObjectDistances > = groundTruth[id - beginID]; + for (auto &r : results[id - beginID]) { + if (std::find(gt.begin(), gt.end(), r) != gt.end()) { + count++; + } + } + recall[id - beginID] = static_cast(count) / static_cast(results[id - beginID].size()); + totalRecall += recall[id - beginID]; + } + totalRecall /= endID - beginID; + std::cerr << "Info: # of not exact results=" << notExactResultCount << " start epsilon=" << startEpsilon + << " current epsilon=" << epsilon << " total recall=" << totalRecall << std::endl; + if (totalRecall >= expectedRecall) { + break; + } + } + return epsilon; + } + static void assignWithNGT(std::vector &clusters, size_t beginID, size_t endID, - NGT::ObjectSpace &objectSpace, QBGObjectList &objectList) { + NGT::ObjectSpace &objectSpace, QBGObjectList &objectList, + size_t epsilonExplorationSize = 100, + float expectedRecall = 0.98) { if (beginID > endID) { std::cerr << "assignWithNGT::Warning. beginID:" << beginID << " > endID:" << endID << std::endl; return; @@ -866,19 +1084,58 @@ namespace QBG { prop.edgeSizeForSearch = 40; #ifdef NGT_SHARED_MEMORY_ALLOCATOR - NGT::Index index(prop, "dummy"); + NGT::Index anngIndex(prop, "dummy"); #else - NGT::Index index(prop); + NGT::Index anngIndex(prop); #endif for (size_t cidx = 0; cidx < clusters.size(); cidx++) { if (cidx % 100000 == 0) { std::cerr << "# of appended cluster objects=" << cidx << std::endl; } - index.append(clusters[cidx].centroid); + anngIndex.append(clusters[cidx].centroid); } std::cerr << "createIndex..." << std::endl; - index.createIndex(500); - + anngIndex.createIndex(500); +#ifdef NGTQ_USING_ONNG + std::string onng; + std::string tmpDir; + { + NGT::Timer timer; + timer.start(); + const char *ngtDirString = "/tmp/ngt-XXXXXX"; + char ngtDir[strlen(ngtDirString) + 1]; + strcpy(ngtDir, ngtDirString); + tmpDir = mkdtemp(ngtDir); + std::string anng = tmpDir + "/anng"; + onng = tmpDir + "/onng"; +#ifndef NGT_SHARED_MEMORY_ALLOCATOR + anngIndex.save(anng); +#endif + auto unlog = false; + NGT::GraphOptimizer graphOptimizer(unlog); + graphOptimizer.searchParameterOptimization = false; + graphOptimizer.prefetchParameterOptimization = false; + graphOptimizer.accuracyTableGeneration = false; + int numOfOutgoingEdges = 10; + int numOfIncomingEdges = 120; + int numOfQueries = 200; + int numOfResultantObjects = 20; + graphOptimizer.set(numOfOutgoingEdges, numOfIncomingEdges, numOfQueries, numOfResultantObjects); + graphOptimizer.execute(anng, onng); + } +#ifdef NGT_SHARED_MEMORY_ALLOCATOR + NGT::Index onngIndex(onng, "dummy"); +#else + NGT::Index onngIndex(onng); +#endif + NGT::Index &index = onngIndex; + const string com = "rm -rf " + tmpDir; + if (system(com.c_str()) == -1) { + std::cerr << "Warning. remove is failed. " << com << std::endl; + } +#else + NGT::Index &index = anngIndex; +#endif std::cerr << "assign with NGT..." << std::endl; endID++; #ifdef MULTIPLE_OBJECT_LISTS @@ -887,7 +1144,18 @@ namespace QBG { abort(); } #endif - std::vector> clusterIDs(endID - beginID); + std::vector> clusterIDs(endID - beginID); + std::vector> distances(omp_get_max_threads(), std::make_pair(0, 0.0)); + size_t endOfEval = beginID + epsilonExplorationSize; + endOfEval = endOfEval > endID ? endID : endOfEval; + size_t nOfObjects = 20; + NGT::Timer timer; + timer.start(); + auto epsilon = optimizeEpsilon(index, beginID, endOfEval, nOfObjects, + objectList, expectedRecall, objectSpace); + timer.stop(); + std::cerr << "assignWithNGT: exploring epsilon. time=" << timer << " epsilon=" << epsilon << std::endl; + timer.start(); #pragma omp parallel for for (size_t id = beginID; id < endID; id++) { std::vector obj; @@ -896,371 +1164,120 @@ namespace QBG { #else objectList.get(id, obj, &objectSpace); #endif - NGT::SearchQuery sc(obj); + NGT::SearchQuery sc(obj); NGT::ObjectDistances objects; sc.setResults(&objects); - sc.setSize(10); - sc.setEpsilon(0.12); + sc.setSize(nOfObjects); + sc.setEpsilon(epsilon); index.search(sc); - //index.linearSearch(sc); clusterIDs[id - beginID] = make_pair(objects[0].id - 1, objects[0].distance); - } + auto threadID = omp_get_thread_num(); + distances[threadID].first++; + distances[threadID].second += objects[0].distance; + { + size_t cnt = 0; + for (auto d : distances) { + cnt += d.first; + } + if (cnt % ((endID - beginID) / 100) == 0) { + timer.stop(); + std::cerr << "assignWithNGT: " << cnt << " objects (" + << (cnt * 100 / (endID - beginID)) << "%) have been assigned. time=" << timer << std::endl; + timer.restart(); + } + } + } std::cerr << "pushing..." << std::endl; for (size_t id = beginID; id < endID; id++) { auto cid = clusterIDs[id - beginID].first; auto cdistance = clusterIDs[id - beginID].second; clusters[cid].members.push_back(NGT::Clustering::Entry(id - 1, cid, cdistance)); } - } - -#ifdef NGTQ_QBG - void treeBasedTopdownClustering(std::string prefix, QBG::Index &index, uint32_t rootID, std::vector &object, std::vector &nodes, NGT::Clustering &clustering) { - auto &quantizer = static_cast&>(index.getQuantizer()); - auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); - QBGObjectList &objectList = quantizer.objectList; - NGT::Timer timer; - timer.start(); - std::vector batch; - std::vector> exceededLeaves; - size_t nleaves = 1; - size_t nOfThreads = 32; - for (size_t id = 1; id <= numOfObjects; id++) { - if (id % (numOfObjects / 100) == 0) { - timer.stop(); - std::cerr << "# of processed objects=" << id << " " << id * 100 / numOfObjects << "% " << timer << " # of leaves=" << nleaves << std::endl; - timer.start(); + { + size_t n = 0; + float err = 0; + for (auto t : distances) { + n += t.first; + err += t.second; } - batch.push_back(id); - if (batch.size() > 100000) { - size_t kmeansBatchSize = nleaves < nOfThreads ? nleaves : nOfThreads; - hierarchicalKmeansBatch(batch, exceededLeaves, rootID, object, objectList, objectSpace, nodes, - clustering, maxSize, nleaves, kmeansBatchSize); - + if (n != 0) { + err /= static_cast(n); + std::cerr << "assign. quantization error=" << err << "/" << n << std::endl; + } else { + std::cerr << "assign. no assigned vectors." << std::endl; } } - hierarchicalKmeansBatch(batch, exceededLeaves, rootID, object, objectList, objectSpace, nodes, - clustering, maxSize, nleaves, 0); - - if (numOfTotalClusters != 0) { - NGT::Timer timer; - timer.start(); - size_t numOfLeaves = 0; - for (auto node : nodes) { - if (node->leaf) { - numOfLeaves++; - } - } - std::cerr << "# of nodes=" << nodes.size() << std::endl; - std::cerr << "# of leaves=" << numOfLeaves << std::endl; - std::cerr << "clustering for quantization." << std::endl; - hierarchicalKmeansWithNumberOfClustersInParallel(numOfTotalClusters, numOfObjects, numOfLeaves, - objectList, objectSpace, nodes, initMode); - if (numOfTotalBlobs != 0) { - NGT::Timer timer; - timer.start(); - size_t numOfLeaves = 0; - for (auto node : nodes) { - if (node->leaf) { - numOfLeaves++; - } - } - std::cerr << "# of leaves=" << numOfLeaves << ":" << numOfTotalClusters << std::endl; - if (numOfLeaves != numOfTotalClusters) { - std::cerr << "# of leaves is invalid " << numOfLeaves << ":" << numOfTotalClusters << std::endl; - abort(); - } - { - std::ofstream of(prefix + QBG::Index::getSecondCentroidSuffix()); - extractCentroids(of, nodes); - } - std::vector qNodeIDs; - for (uint32_t nid = 0; nid < nodes.size(); nid++) { - if (nodes[nid]->leaf) { - qNodeIDs.push_back(nid); - } - } - std::cerr << "clustering to make blobs." << std::endl; - hierarchicalKmeansWithNumberOfClustersInParallel(numOfTotalBlobs, numOfObjects, numOfTotalClusters, - objectList, objectSpace, nodes, initMode); - { - std::ofstream of(prefix + QBG::Index::get3rdTo2ndSuffix()); - extractBtoQIndex(of, nodes, qNodeIDs); - } - } - } - } - void multilayerClustering(std::string prefix, QBG::Index &index) { - auto &quantizer = static_cast&>(index.getQuantizer()); - auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); - { - std::cerr << "Three layer clustering..." << std::endl; - std::cerr << "HiearchicalKmeans::clustering: # of clusters=" << numOfThirdClusters << ":" << index.getQuantizer().property.globalCentroidLimit << std::endl; - if (index.getQuantizer().objectList.size() <= 1) { - NGTThrowException("optimize: No objects"); - } - if (numOfThirdClusters == 0) { - if (index.getQuantizer().property.globalCentroidLimit == 0) { - numOfThirdClusters = index.getQuantizer().objectList.size() / 1000; - numOfThirdClusters = numOfThirdClusters == 0 ? 1 : numOfThirdClusters; - numOfThirdClusters = numOfThirdClusters > 1000000 ? 1000000 : numOfThirdClusters; - } else { - numOfThirdClusters = index.getQuantizer().property.globalCentroidLimit; - } - } - if (numOfThirdClusters != 0 && index.getQuantizer().property.globalCentroidLimit != 0 && - numOfThirdClusters != index.getQuantizer().property.globalCentroidLimit) { - } - auto &quantizer = static_cast&>(index.getQuantizer()); - QBGObjectList &objectList = quantizer.objectList; - if (numOfObjects == 0) { - numOfObjects = objectList.size() - 1; - } - - std::cerr << "The first layer. " << numOfFirstClusters << ":" << numOfFirstObjects << std::endl; - if (numOfThirdClusters == 0 || numOfObjects == 0) { - NGTThrowException("numOfThirdClusters or numOfObjects are zero"); - } - numOfSecondClusters = numOfSecondClusters == 0 ? numOfThirdClusters : numOfSecondClusters; - numOfFirstClusters = numOfFirstClusters == 0 ? static_cast(sqrt(numOfSecondClusters)) : numOfFirstClusters; - numOfSecondObjects = numOfSecondClusters * 100; - numOfSecondObjects = numOfSecondObjects > numOfObjects ? numOfObjects : numOfSecondObjects; - numOfFirstObjects = numOfFirstClusters * 2000; - numOfFirstObjects = numOfFirstObjects > numOfSecondObjects ? numOfSecondObjects : numOfFirstObjects; - if (numOfFirstObjects < numOfFirstClusters) { - std::stringstream msg; - msg << "# of objects for the first should be larger than # of the first clusters. " << numOfFirstObjects << ":" << numOfFirstClusters; - NGTThrowException(msg); - } - if (numOfFirstClusters > numOfSecondClusters) { - std::stringstream msg; - msg << "# of the first clusters should be larger than or equal to # of the second clusters. " << numOfFirstClusters << ":" << numOfSecondClusters; - NGTThrowException(msg); - } - if (numOfSecondClusters > numOfThirdClusters) { - std::stringstream msg; - msg << "# of the third clusters should be larger than or equal to # of the second clusters. " << numOfSecondClusters << ":" << numOfThirdClusters; - NGTThrowException(msg); - } - if (numOfFirstClusters > numOfSecondClusters) { - std::stringstream msg; - msg << "# of the second clusters should be larger than # of the first clusters. " << numOfFirstClusters << ":" << numOfSecondClusters; - NGTThrowException(msg); - } +#ifdef NGTQ_QBG + void treeBasedTopdownClustering(std::string prefix, QBG::Index &index, uint32_t rootID, std::vector &object, std::vector &nodes, NGT::Clustering &clustering); + + static void twoLayerClustering(std::vector> vectors, + size_t numOfClusters, + std::vector &clusters, + size_t numOfObjectsForEachFirstCluster = 0, + size_t numOfObjectsForEachSecondCluster = 0, + size_t maximumIteration = 300, + NGT::Clustering::InitializationMode initMode = NGT::Clustering::InitializationModeKmeansPlusPlus) { + std::cerr << "clustering with two layers... # of clusters=" << numOfClusters << std::endl; + if (numOfClusters == 0) { + NGTThrowException("HierachicalKmeans:: # of clusters is zero."); + } + size_t numOfFirstClusters = sqrt(numOfClusters); + size_t numOfFirstObjects = sqrt(vectors.size()); + numOfFirstObjects = numOfObjectsForEachFirstCluster == 0 ? + numOfFirstObjects : numOfFirstClusters * numOfObjectsForEachFirstCluster; + size_t numOfSecondClusters = numOfClusters; + size_t numOfSecondObjects = vectors.size(); + numOfSecondObjects = numOfObjectsForEachSecondCluster == 0 ? + numOfSecondObjects : numOfSecondClusters * numOfObjectsForEachSecondCluster; + //auto &quantizer = static_cast&>(index.getQuantizer()); + //auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + size_t numOfObjects = vectors.size(); + if (numOfSecondObjects > numOfObjects) { + numOfSecondObjects = numOfObjects; + } - std::cerr << "Three layer clustering:" << numOfFirstClusters << ":" << numOfFirstObjects << "," << numOfSecondClusters << ":" << numOfSecondObjects << "," << numOfThirdClusters << ":" << numOfObjects << std::endl; - NGT::Clustering firstClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, 300); - float clusterSizeConstraint = 5.0; - firstClustering.setClusterSizeConstraintCoefficient(clusterSizeConstraint); - std::cerr << "size constraint=" << clusterSizeConstraint << std::endl; - std::vector> vectors; - vectors.reserve(numOfFirstObjects); - std::vector obj; - for (size_t id = 1; id <= numOfFirstObjects; id++) { - if (id % 1000000 == 0) { - std::cerr << "# of prcessed objects is " << id << std::endl; - } - if (!objectList.get(id, obj, &objectSpace)) { - std::stringstream msg; - msg << "qbg: Cannot get object. ID=" << id; - NGTThrowException(msg); - } - vectors.push_back(obj); + NGT::Clustering firstClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithoutNGT, maximumIteration); + //NGT::Clustering firstClustering(initMode, NGT::Clustering::ClusteringTypeKmeansWithNGT, maximumIteration); + firstClustering.setClusterSizeConstraintCoefficient(false); + std::vector firstClusters; + NGT::Timer timer; + { + std::vector> firstVectors; + firstVectors.reserve(numOfFirstObjects); + for (size_t c = 0; c < numOfFirstObjects; c++) { + firstVectors.push_back(vectors[c]); } - std::cerr << "Kmeans... " << vectors.size() << " to " << numOfFirstClusters << std::endl; - std::vector firstClusters; - NGT::Timer timer; - + std::cerr << "clustering for the first...." << std::endl; timer.start(); - firstClustering.kmeans(vectors, numOfFirstClusters, firstClusters); + firstClustering.kmeans(firstVectors, numOfFirstClusters, firstClusters); timer.stop(); - std::cerr << "# of clusters=" << firstClusters.size() << " time=" << timer << std::endl; + std::cerr << "clustering for the first. # of clusters=" << firstClusters.size() << " time=" << timer << std::endl; + } - std::vector> otherVectors; - timer.start(); - std::cerr << "Assign for the second. (" << numOfFirstObjects << "-" << numOfSecondObjects << ")..." << std::endl; - assign(firstClusters, numOfFirstObjects + 1, numOfSecondObjects, objectSpace, objectList); - timer.stop(); - std::cerr << "Assign(1) time=" << timer << std::endl; + timer.start(); + std::cerr << "assign for the second. (" << numOfFirstObjects << "-" << numOfSecondObjects << ")..." << std::endl; + assign(firstClusters, numOfFirstObjects, numOfSecondObjects - 1, vectors); + timer.stop(); + std::cerr << "assign for the second. time=" << timer << std::endl; - std::cerr << "subclustering for the second." << std::endl; - std::vector secondClusters; - timer.start(); - subclustering(firstClusters, numOfSecondClusters, numOfSecondObjects, objectSpace, objectList, initMode, secondClusters); - timer.stop(); - std::cerr << "subclustering(1) time=" << timer << std::endl; - std::cerr << "save quantization centroid" << std::endl; - NGT::Clustering::saveClusters(prefix + QBG::Index::getSecondCentroidSuffix(), secondClusters); - timer.start(); - std::cerr << "Assign for the third. (" << numOfSecondObjects << "-" << numOfObjects << ")..." << std::endl; - assignWithNGT(secondClusters, numOfSecondObjects + 1, numOfObjects, objectSpace, objectList); - timer.stop(); - std::cerr << "Assign(2) time=" << timer << std::endl; - std::cerr << "subclustering for the third." << std::endl; - std::vector> thirdClusters; - timer.start(); - subclustering(secondClusters, numOfThirdClusters, numOfObjects, objectSpace, objectList, initMode, thirdClusters); - timer.stop(); - std::cerr << "subclustering(2) time=" << timer << std::endl; - { - std::vector bqindex; - for (size_t idx1 = 0; idx1 < thirdClusters.size(); idx1++) { - for (size_t idx2 = 0; idx2 < thirdClusters[idx1].size(); idx2++) { - bqindex.push_back(idx1); - } - } - std::cerr << "save bqindex..." << std::endl; - NGT::Clustering::saveVector(prefix + QBG::Index::get3rdTo2ndSuffix(), bqindex); - } - - std::vector thirdFlatClusters; - flattenClusters(secondClusters, thirdClusters, numOfThirdClusters, thirdFlatClusters); + std::cerr << "subclustering for the second..." << std::endl; + std::vector &secondClusters = clusters; + timer.start(); + subclustering(firstClusters, numOfSecondClusters, numOfSecondObjects, initMode, secondClusters, vectors, maximumIteration); + timer.stop(); + std::cerr << "subclustering for the second. time=" << timer << std::endl; + } - std::cerr << "save centroid..." << std::endl; - NGT::Clustering::saveClusters(prefix + QBG::Index::getThirdCentroidSuffix(), thirdFlatClusters); + void threeLayerClustering(std::string prefix, QBG::Index &index); - { - std::vector cindex(numOfObjects); - for (size_t cidx = 0; cidx < thirdFlatClusters.size(); cidx++) { - for (auto mit = thirdFlatClusters[cidx].members.begin(); mit != thirdFlatClusters[cidx].members.end(); ++mit) { - size_t vid = (*mit).vectorID; - cindex[vid] = cidx; - } - } - std::cerr << "save index... " << cindex.size() << std::endl; - NGT::Clustering::saveVector(prefix + QBG::Index::getObjTo3rdSuffix(), cindex); - } - std::cerr << "end of clustering" << std::endl; - return; - } + void twoPlusLayerClustering(std::string prefix, QBG::Index &index); - } + void multiLayerClustering(QBG::Index &index, std::string prefix, std::string objectIDsFile); - void clustering(std::string indexPath, std::string prefix = "", std::string objectIDsFile = "") { - NGT::StdOstreamRedirector redirector(silence); - redirector.begin(); - - bool readOnly = false; - QBG::Index index(indexPath, readOnly); - index.getQuantizer().objectList.size(); - std::cerr << "clustering... " << std::endl; - if (threeLayerClustering) { - try { - if (numOfObjects == 0) { - numOfObjects = index.getQuantizer().objectList.size() - 1; - } - if (numOfObjects != index.getQuantizer().objectList.size() - 1) { - std::cerr << "HierarchicalKmeans::clustering: Warning! # of objects is invalid." << std::endl; - std::cerr << " " << index.getQuantizer().objectList.size() - 1 << " is set to # of object instead of " << numOfObjects << std::endl; - numOfObjects = index.getQuantizer().objectList.size() - 1; - } - if (prefix.empty()) { - std::cerr << "Prefix is not specified." << std::endl; - prefix = indexPath + "/" + QBG::Index::getWorkspaceName(); - try { - NGT::Index::mkdir(prefix); - } catch(...) {} - prefix +="/" + QBG::Index::getHierarchicalClusteringPrefix(); - std::cerr << prefix << " is used" << std::endl; - } - auto &quantizer = static_cast&>(index.getQuantizer()); - auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); - size_t paddedDimension = objectSpace.getPaddedDimension(); - size_t dimension = objectSpace.getDimension(); - if (paddedDimension != dimension) { - std::cerr << "HierarachicalKmeans: Warning! Dimensions are inconsistent. Dimension=" << paddedDimension << ":" << dimension << std::endl; - } - multilayerClustering(prefix, index); - } catch(NGT::Exception &err) { - redirector.end(); - throw err; - } - redirector.end(); - return; - } - try { - NGT::Clustering::ClusteringType clusteringType = NGT::Clustering::ClusteringTypeKmeansWithoutNGT; - - uint32_t rootID = 0; - std::vector nodes; - nodes.push_back(new HKLeafNode); - - std::vector object; - size_t iteration = 1000; - NGT::Clustering clustering(initMode, clusteringType, iteration, numOfClusters); - auto &quantizer = static_cast&>(index.getQuantizer()); - QBGObjectList &objectList = quantizer.objectList; - if (objectIDsFile.empty()) { - treeBasedTopdownClustering(prefix, index, rootID, object, nodes, clustering); - } else { - std::cerr << "Cluster ID=" << clusterID << std::endl; - if (clusterID < 0) { - std::stringstream msg; - msg << "Any target cluster ID is not specified."; - NGTThrowException(msg); - } - std::ifstream objectIDs(objectIDsFile); - if (!objectIDs) { - std::stringstream msg; - msg << "Cannot open the object id file. " << objectIDsFile; - NGTThrowException(msg); - } - auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); - uint32_t id = 1; - int32_t cid; - size_t ccount = 0; - while (objectIDs >> cid) { - std::cerr << cid << std::endl; - if (id % 100000 == 0) { - std::cerr << "# of processed objects=" << id << std::endl; - } - if (cid == -1) { - continue; - } - if (cid == clusterID) { - ccount++; - hierarchicalKmeans(id, rootID, object, objectList, objectSpace, nodes, clustering, maxSize); - } - id++; - } - } - size_t objectCount = 0; - if (prefix.empty()) { - objectCount = extractCentroids(std::cout, nodes); - } else { - { - std::ofstream of(prefix + QBG::Index::getThirdCentroidSuffix()); - objectCount = extractCentroids(of, nodes); - } - { - std::ofstream of(prefix + QBG::Index::getObjTo3rdSuffix()); - extractIndex(of, nodes, numOfObjects); - } - if (numOfFirstObjects > 0) { - std::ofstream btoqof(prefix + QBG::Index::get3rdTo2ndSuffix()); - std::ofstream qcof(prefix + QBG::Index::getSecondCentroidSuffix()); - extractBtoQAndQCentroid(btoqof, qcof, nodes, numOfThirdClusters); - } - if (numOfRandomObjects > 0) { - std::ofstream of(prefix + "_random_object.tsv"); - if (extractCentroid) { - extractRandomObjectsFromEachBlob(of, nodes, numOfObjects, numOfRandomObjects - 1, quantizer, extractCentroid); - } else { - extractRandomObjectsFromEachBlob(of, nodes, numOfObjects, numOfRandomObjects, quantizer, extractCentroid); - } - } - } - if (objectCount != numOfObjects) { - std::cerr << "# of objects is invalid. " << objectCount << ":" << numOfObjects << std::endl; - } - } catch(NGT::Exception &err) { - redirector.end(); - throw err; - } - redirector.end(); - } + void clustering(std::string indexPath, std::string prefix = "", std::string objectIDsFile = ""); #endif size_t maxSize; @@ -1278,10 +1295,12 @@ namespace QBG { size_t numOfFirstClusters; size_t numOfSecondObjects; size_t numOfSecondClusters; + size_t numOfThirdObjects; size_t numOfThirdClusters; bool extractCentroid; - - bool threeLayerClustering; - bool silence; + ClusteringType clusteringType; + size_t epsilonExplorationSize; + float expectedRecall; + bool verbose; }; } diff --git a/lib/NGT/NGTQ/Optimizer.cpp b/lib/NGT/NGTQ/Optimizer.cpp index d8be480..fd29e73 100644 --- a/lib/NGT/NGTQ/Optimizer.cpp +++ b/lib/NGT/NGTQ/Optimizer.cpp @@ -17,33 +17,49 @@ #include "QuantizedBlobGraph.h" #include "Optimizer.h" + QBG::Optimizer::Optimizer(QBG::BuildParameters ¶m) { #ifdef NGTQ_QBG - clusteringType = param.optimization.clusteringType; - initMode = param.optimization.initMode; - - timelimit = param.optimization.timelimit; - iteration = param.optimization.iteration; - clusterIteration = param.optimization.clusterIteration; - clusterSizeConstraint = param.optimization.clusterSizeConstraint; - clusterSizeConstraintCoefficient = param.optimization.clusterSizeConstraintCoefficient; - convergenceLimitTimes = param.optimization.convergenceLimitTimes; - numberOfObjects = param.optimization.numberOfObjects; - numberOfClusters = param.optimization.numberOfClusters; - numberOfSubvectors = param.optimization.numberOfSubvectors; - nOfMatrices = param.optimization.nOfMatrices; - seedStartObjectSizeRate = param.optimization.seedStartObjectSizeRate; - seedStep = param.optimization.seedStep; - reject = param.optimization.reject; - repositioning = param.optimization.repositioning; - rotation = param.optimization.rotation; - globalType = param.optimization.globalType; - randomizedObjectExtraction = param.optimization.randomizedObjectExtraction; - showClusterInfo = param.optimization.showClusterInfo; - silence = param.silence; + setParameters(param.optimization); #endif } +QBG::Optimizer::Optimizer(QBG::OptimizationParameters ¶m) { + setParameters(param); +} +void QBG::Optimizer::initialize() { +#ifdef NGTQ_QBG + OptimizationParameters params; + setParameters(params); +#endif +} + +void QBG::Optimizer::setParameters(QBG::OptimizationParameters ¶m) { +#ifdef NGTQ_QBG + clusteringType = param.clusteringType; + initMode = param.initMode; + + timelimit = param.timelimit; + iteration = param.iteration; + clusterIteration = param.clusterIteration; + clusterSizeConstraint = param.clusterSizeConstraint; + clusterSizeConstraintCoefficient = param.clusterSizeConstraintCoefficient; + convergenceLimitTimes = param.convergenceLimitTimes; + numberOfObjects = param.numOfObjects; + numberOfClusters = param.numOfClusters; + numberOfSubvectors = param.numOfSubvectors; + numberOfMatrices = param.numOfMatrices; + seedNumberOfSteps = param.seedNumberOfSteps; + seedStep = param.seedStep; + reject = param.reject; + repositioning = param.repositioning; + rotation = param.rotation; + globalType = param.globalType; + randomizedObjectExtraction = param.randomizedObjectExtraction; + showClusterInfo = param.showClusterInfo; + verbose = param.verbose; +#endif +} #ifdef NGTQ_QBG void QBG::Optimizer::evaluate(string global, vector> &vectors, char clusteringType, string &ofile, size_t &numberOfSubvectors, size_t &subvectorSize) @@ -224,8 +240,11 @@ void QBG::Optimizer::evaluate(vector> &vectors, string &ofile, siz #ifdef NGTQ_QBG void QBG::Optimizer::optimize(const std::string indexPath, size_t threadSize) { - NGT::StdOstreamRedirector redirector(silence); + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); + if (threadSize == 0) { + threadSize = omp_get_max_threads(); + } try { QBG::Index index(indexPath); if (index.getQuantizer().objectList.size() <= 1) { @@ -245,7 +264,10 @@ void QBG::Optimizer::optimize(const std::string indexPath, size_t threadSize) { } if (index.getQuantizer().property.localCentroidLimit != 0 && numberOfClusters != 0 && index.getQuantizer().property.localCentroidLimit != numberOfClusters) { - std::cerr << "optimize: warning! # of clusters is already specified. " << index.getQuantizer().property.localCentroidLimit << ":" << numberOfClusters << std::endl; + std::stringstream msg; + msg << "optimize: # of clusters is already specified. " << index.getQuantizer().property.localCentroidLimit << ":" << numberOfClusters; + NGTThrowException(msg); + } if (numberOfClusters == 0) { numberOfClusters = index.getQuantizer().property.localCentroidLimit; @@ -277,6 +299,27 @@ void QBG::Optimizer::optimize(const std::string indexPath, size_t threadSize) { std::vector> global(1); global[0].resize(index.getQuantizer().property.dimension, 0.0); NGT::Clustering::saveVectors(QBG::Index::getQuantizerCodebookFile(indexPath), global); + + ifstream ifs(QBG::Index::getCodebookIndexFile(indexPath)); + if (!ifs) { + std::stringstream msg; + msg << "Cannot open the file. " << QBG::Index::getCodebookIndexFile(indexPath); + NGTThrowException(msg); + } + size_t id; + size_t count = 0; + while (ifs >> id) { + count++; + } + ofstream ofs(QBG::Index::getCodebookIndexFile(indexPath)); + if (!ofs) { + std::stringstream msg; + msg << "Cannot open the file. " << QBG::Index::getCodebookIndexFile(indexPath); + NGTThrowException(msg); + } + for (size_t i = 0; i < count; i++) { + ofs << "0" << std::endl; + } } else if (globalType == GlobalTypeMean) { std::vector> vectors; std::string objects = QBG::Index::getTrainObjectFile(indexPath); @@ -286,7 +329,7 @@ void QBG::Optimizer::optimize(const std::string indexPath, size_t threadSize) { loadVectors(objects, vectors); #endif if (vectors.size() == 0 || vectors[0].size() == 0) { - NGTThrowException("Optimizer::optimize: invalid input vectors"); + NGTThrowException("optimize::optimize: invalid input vectors"); } std::vector> global(1); global[0].resize(index.getQuantizer().property.dimension, 0); @@ -334,27 +377,24 @@ void QBG::Optimizer::optimizeWithinIndex(std::string indexPath) { #ifdef NGTQ_QBG -void QBG::Optimizer::optimize(std::string invector, std::string ofile, std::string global) { +void QBG::Optimizer::optimize(vector> &vectors, vector> &globalCentroid, Matrix &r, vector> &localClusters, vector &errors) { #if defined(NGT_SHARED_MEMORY_ALLOCATOR) std::cerr << "optimize: Not implemented." << std::endl; abort(); #else - vector> vectors; - - -#ifdef NGT_CLUSTERING - NGT::Clustering::loadVectors(invector, vectors); -#else - loadVectors(invector, vectors); -#endif if (vectors.size() == 0) { std::stringstream msg; - msg << "Optimizer: error! the specified vetor file is empty. " << invector << ". the size=" << vectors.size(); + msg << "optimize: error! the specified vetor is empty. the size=" << vectors.size(); NGTThrowException(msg); } - dim = vectors[0].size(); + auto dim = vectors[0].size(); + if (numberOfSubvectors == 0) { + std::stringstream msg; + msg << "# of subspaces (m) is zero."; + NGTThrowException(msg); + } subvectorSize = dim / numberOfSubvectors; if (dim % numberOfSubvectors != 0) { std::stringstream msg; @@ -362,6 +402,8 @@ void QBG::Optimizer::optimize(std::string invector, std::string ofile, std::stri NGTThrowException(msg); } + generateResidualObjects(globalCentroid, vectors); + timelimitTimer.start(); @@ -378,111 +420,164 @@ void QBG::Optimizer::optimize(std::string invector, std::string ofile, std::stri dstidx++; } } - std::cerr << "Optimizer: Each axis was repositioned." << std::endl; + std::cerr << "optimize: Each axis was repositioned." << std::endl; } - vector>> localClusters; - vector errors; + vector>> localClustersSet; - bool useEye = false; - nOfMatrices = nOfMatrices == 0 ? 1 : nOfMatrices; + numberOfMatrices = numberOfMatrices == 0 ? 1 : numberOfMatrices; if (!rotation) { iteration = 1; - seedStartObjectSizeRate = 1.0; + seedNumberOfSteps = 1; seedStep = 2; + numberOfMatrices = 1; + } + + seedNumberOfSteps = numberOfMatrices == 1 ? 1 : seedNumberOfSteps; + seedNumberOfSteps = seedNumberOfSteps < 1 ? 1 : seedNumberOfSteps; + if (numberOfMatrices > 100) { + std::stringstream msg; + msg << "# of matrices is too large. Should be less than 10. " << numberOfMatrices << " was specified."; + NGTThrowException(msg); + } + if (seedStep > 100 || seedStep == 0) { + std::stringstream msg; + msg << "the seed step is illegal. Should be less than 100. " << seedStep << " was specified."; + NGTThrowException(msg); } - useEye = !rotation; - vector> rs(nOfMatrices); + vector> rs(numberOfMatrices); for (auto &r: rs) { - if (useEye) { + if (!rotation) { r.eye(dim); } else { r.randomRotation(dim); } } - for (size_t vsize = static_cast(vectors.size()) * seedStartObjectSizeRate; ; vsize *= seedStep) { + NGT::Timer timer; + timer.start(); + for (int step = seedNumberOfSteps - 1; ; step--) { + size_t vsize = vectors.size() / pow(seedStep, step); + std::cerr << "optimize: # of vectors=" << vsize << "/" << vectors.size() + << ", # of matrices=" << numberOfMatrices << std::endl; + if (vsize <= 1) { + std::stringstream msg; + msg << "# of partial vectors is too small, because # of vectors is too small or seedStep is too large." + << " # of partial vectors=" << vsize << " # of vectors=" << vectors.size() << " seedStep=" << seedStep << std::endl; + NGTThrowException(msg); + } auto partialVectors = vectors; if (vsize < vectors.size()) { partialVectors.resize(vsize); } optimize(partialVectors, - global, - ofile, reposition, rs, - localClusters, + localClustersSet, errors); if (rs.size() > 1) { - nOfMatrices = static_cast(nOfMatrices) * (1.0 - reject); - nOfMatrices = nOfMatrices == 0 ? 1 : nOfMatrices; + numberOfMatrices = static_cast(numberOfMatrices) * (1.0 - reject); + numberOfMatrices = numberOfMatrices == 0 ? 1 : numberOfMatrices; vector*, vector>*>>> sortedErrors; for (size_t idx = 0; idx < errors.size(); idx++) { - sortedErrors.emplace_back(make_pair(errors[idx], make_pair(&rs[idx], &localClusters[idx]))); + sortedErrors.emplace_back(make_pair(errors[idx], make_pair(&rs[idx], &localClustersSet[idx]))); } sort(sortedErrors.begin(), sortedErrors.end()); vector> tmpMatrix; vector>> tmpLocalClusters; - for (size_t idx = 0; idx < nOfMatrices; idx++) { + for (size_t idx = 0; idx < numberOfMatrices; idx++) { tmpMatrix.emplace_back(*sortedErrors[idx].second.first); tmpLocalClusters.emplace_back(*sortedErrors[idx].second.second); } - if (tmpMatrix.size() != nOfMatrices) { - std::cerr << "something strange. " << tmpMatrix.size() << ":" << nOfMatrices << std::endl; + if (tmpMatrix.size() != numberOfMatrices) { + std::cerr << "something strange. " << tmpMatrix.size() << ":" << numberOfMatrices << std::endl; } rs = std::move(tmpMatrix); - localClusters = std::move(tmpLocalClusters); + localClustersSet = std::move(tmpLocalClusters); } + timer.stop(); + std::cerr << "optimize: time=" << timer << std::endl; + timer.restart(); if (vsize >= vectors.size()) { break; } } if (rs.size() != 1) { - std::cerr << "Optimizer: Warning. rs.size=" << rs.size() << std::endl; + std::cerr << "optimize: Warning. rs.size=" << rs.size() << std::endl; } - auto minR = std::move(rs[0]); - auto minLocalClusters = std::move(localClusters[0]); - //-/size_t pos = std::distance(std::find(ofile.rbegin(), ofile.rend(), '.'), ofile.rend()) - 1; + + localClusters = std::move(localClustersSet[0]); if (repositioning) { Matrix repositionedR(reposition); - repositionedR.mul(minR); - Matrix::save(ofile + QBG::Index::getRotationFile(), repositionedR); + repositionedR.mul(rs[0]); + r = std::move(repositionedR); } else { - Matrix::save(ofile + QBG::Index::getRotationFile(), minR); + r = std::move(rs[0]); } + + //-/size_t pos = std::distance(std::find(ofile.rbegin(), ofile.rend(), '.'), ofile.rend()) - 1; + +#endif +} + +void QBG::Optimizer::optimize(std::string invector, std::string ofile, std::string global) { +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + std::cerr << "optimize: Not implemented." << std::endl; + abort(); +#else + + + vector> vectors; +#ifdef NGT_CLUSTERING + NGT::Clustering::loadVectors(invector, vectors); +#else + loadVectors(invector, vectors); +#endif + + vector> globalCentroid; + NGT::Clustering::loadVectors(global, globalCentroid); + + Matrix r; + vector> localClusters; + vector errors; + + optimize(vectors, globalCentroid, r, localClusters, errors); + + Matrix::save(ofile + QBG::Index::getRotationFile(), r); + if (showClusterInfo) { - if (minLocalClusters.size() != numberOfSubvectors) { + if (localClusters.size() != numberOfSubvectors) { std::stringstream msg; - msg << "Fatal error. minLocalClusters.size() != numberOfSubvectors " << - minLocalClusters.size() << ":" << numberOfSubvectors; + msg << "Fatal error. localClusters.size() != numberOfSubvectors " + << localClusters.size() << ":" << numberOfSubvectors; NGTThrowException(msg); } float totalRate = 0.0; - for (size_t m = 0; m < minLocalClusters.size(); m++) { + for (size_t m = 0; m < localClusters.size(); m++) { size_t min = std::numeric_limits::max(); size_t max = 0; size_t nOfVectors = 0; - for (size_t i = 0; i < minLocalClusters[m].size(); i++) { - nOfVectors += minLocalClusters[m][i].members.size(); - if (minLocalClusters[m][i].members.size() < min) { - min = minLocalClusters[m][i].members.size(); + for (size_t i = 0; i < localClusters[m].size(); i++) { + nOfVectors += localClusters[m][i].members.size(); + if (localClusters[m][i].members.size() < min) { + min = localClusters[m][i].members.size(); } - if (minLocalClusters[m][i].members.size() > max) { - max = minLocalClusters[m][i].members.size(); + if (localClusters[m][i].members.size() > max) { + max = localClusters[m][i].members.size(); } } float rate = static_cast(max - min) / static_cast(nOfVectors); totalRate += rate; std::cout << "cluster " << m << " " << rate << "," << max - min << "," << min << "," << max << " : "; - for (size_t i = 0; i < minLocalClusters[m].size(); i++) { - std::cout << minLocalClusters[m][i].members.size() << " "; + for (size_t i = 0; i < localClusters[m].size(); i++) { + std::cout << localClusters[m][i].members.size() << " "; } std::cout << std::endl; } - totalRate /= minLocalClusters.size(); + totalRate /= localClusters.size(); std::cout << "Range rate=" << totalRate << std::endl; std::cout << "Error=" << errors[0] << std::endl; } @@ -490,9 +585,9 @@ void QBG::Optimizer::optimize(std::string invector, std::string ofile, std::stri stringstream str; str << ofile << QBG::Index::getSubvectorPrefix() << "-" << m; #ifdef NGT_CLUSTERING - NGT::Clustering::saveClusters(str.str(), minLocalClusters[m]); + NGT::Clustering::saveClusters(str.str(), localClusters[m]); #else - saveClusters(str.str(), minLocalClusters[m]); + saveClusters(str.str(), localClusters[m]); #endif } #endif diff --git a/lib/NGT/NGTQ/Optimizer.h b/lib/NGT/NGTQ/Optimizer.h index cfd3cc2..e5edcb4 100644 --- a/lib/NGT/NGTQ/Optimizer.h +++ b/lib/NGT/NGTQ/Optimizer.h @@ -32,6 +32,7 @@ namespace QBG { class BuildParameters; + class OptimizationParameters; class Optimizer { public: @@ -41,28 +42,14 @@ namespace QBG { GlobalTypeMean = 2 }; - Optimizer() { - clusteringType = NGT::Clustering::ClusteringTypeKmeansWithNGT; - initMode = NGT::Clustering::InitializationModeRandom; + Optimizer() { initialize(); } - iteration = 100; - clusterIteration = 100; - clusterSizeConstraint = false; - clusterSizeConstraintCoefficient = 10.0; - convergenceLimitTimes = 5; - - numberOfClusters = 0; - numberOfSubvectors = 0; + Optimizer(QBG::BuildParameters ¶m); + Optimizer(QBG::OptimizationParameters ¶m); - repositioning = false; - rotation = true; - globalType = GlobalTypeNone; - randomizedObjectExtraction = true; - silence = true; - showClusterInfo = false; - } + void setParameters(QBG::OptimizationParameters ¶m); - Optimizer(QBG::BuildParameters ¶m); + void initialize(); static void extractSubvector(vector> &vectors, vector> &subvectors, size_t start , size_t size) @@ -137,24 +124,13 @@ namespace QBG { #endif void - generateResidualObjects(string global, vector> &vectors) + generateResidualObjects(vector> &globalCentroid, vector> &vectors) { #if defined(NGT_SHARED_MEMORY_ALLOCATOR) std::cerr << "generateResidualObjects: Not implemented." << std::endl; abort(); #else - if (global.empty()) { - NGTThrowException("A global codebook is not specified!"); - } vector> residualVectors; - vector> globalCentroid; - try { - NGT::Clustering::loadVectors(global, globalCentroid); - } catch (...) { - std::stringstream msg; - msg << "Optimizer::generateResidualObjects: Cannot load global vectors. " << global; - NGTThrowException(msg); - } NGT::Property property; property.objectType = NGT::Index::Property::ObjectType::Float; @@ -201,6 +177,23 @@ namespace QBG { #endif } + void + generateResidualObjects(string global, vector> &vectors) + { + if (global.empty()) { + NGTThrowException("A global codebook is not specified!"); + } + vector> globalCentroid; + try { + NGT::Clustering::loadVectors(global, globalCentroid); + } catch (...) { + std::stringstream msg; + msg << "Optimizer::generateResidualObjects: Cannot load global vectors. " << global; + NGTThrowException(msg); + } + generateResidualObjects(globalCentroid, vectors); + } + static void optimizeRotation( size_t iteration, vector> &vectors, @@ -222,8 +215,12 @@ namespace QBG { bool rotation ) { + if (numberOfClusters <= 1) { + std::stringstream msg; + msg << "Optimizer::optimize: # of clusters is zero or one. " << numberOfClusters; + NGTThrowException(msg); + } minDistortion = DBL_MAX; - int minIt = 0; for (size_t it = 0; it < iteration; it++) { vector> xp = vectors; @@ -313,9 +310,7 @@ namespace QBG { } } - void optimize(vector> &vectors, - string global, - string ofile, + void optimize(vector> &vectors, Matrix &reposition, vector> &rs, vector>> &localClusters, @@ -324,7 +319,6 @@ namespace QBG { if (vectors.size() == 0) { NGTThrowException("the vector is empty"); } - generateResidualObjects(global, vectors); if (!reposition.isEmpty()) { Matrix::mulSquare(vectors, reposition); } @@ -369,40 +363,33 @@ namespace QBG { #ifdef NGTQ_QBG void optimize(const std::string indexPath, size_t threadSize = 0); -#endif - -#ifdef NGTQ_QBG void optimizeWithinIndex(std::string indexPath); -#endif - -#ifdef NGTQ_QBG void optimize(std::string invector, std::string ofile, std::string global); -#endif + void optimize(vector> &vectors, vector> &globalCentroid, Matrix &r, vector> &localClusters, vector &errors); +#endif + NGT::Timer timelimitTimer; + size_t subvectorSize; NGT::Clustering::ClusteringType clusteringType; NGT::Clustering::InitializationMode initMode; - - NGT::Timer timelimitTimer; - float timelimit; size_t iteration; size_t clusterIteration; bool clusterSizeConstraint; float clusterSizeConstraintCoefficient; size_t convergenceLimitTimes; - size_t dim; size_t numberOfObjects; size_t numberOfClusters; size_t numberOfSubvectors; - size_t subvectorSize; - size_t nOfMatrices; - float seedStartObjectSizeRate; + size_t numberOfMatrices; + size_t seedNumberOfSteps; size_t seedStep; float reject; bool repositioning; bool rotation; GlobalType globalType; bool randomizedObjectExtraction; - bool silence; + bool verbose; bool showClusterInfo; + float timelimit; }; } // namespace QBG diff --git a/lib/NGT/NGTQ/QbgCli.cpp b/lib/NGT/NGTQ/QbgCli.cpp index 90e8f29..81d03a8 100644 --- a/lib/NGT/NGTQ/QbgCli.cpp +++ b/lib/NGT/NGTQ/QbgCli.cpp @@ -44,11 +44,11 @@ class QbgCliBuildParameters : public QBG::BuildParameters { creation.threadSize = args.getl("p", 24); creation.dimension = args.getl("d", 0); #ifdef NGTQ_QBG - creation.localCentroidLimit = args.getl("c", 16); + creation.numOfLocalClusters = args.getl("c", 16); #else - creation.localCentroidLimit = args.getl("c", 65000); + creation.numOfLocalClusters = args.getl("c", 65000); #endif - creation.localDivisionNo = args.getl("N", 0); + creation.numOfSubvectors = args.getl("N", 0); creation.batchSize = args.getl("b", 1000); creation.localClusteringSampleCoefficient = args.getl("s", 10); { @@ -136,13 +136,16 @@ class QbgCliBuildParameters : public QBG::BuildParameters { } } #endif - + { + char objectListOnMemory = args.getChar("R", 'f'); + creation.objectListOnMemory = (objectListOnMemory == 't' || objectListOnMemory == 'T'); + } } void getHierarchicalClustringParameters() { hierarchicalClustering.maxSize = args.getl("r", 1000); - hierarchicalClustering.numOfObjects = args.getl("O", 0); + hierarchicalClustering.numOfObjects = args.getl("O", 0); hierarchicalClustering.numOfClusters = args.getl("E", 2); try { hierarchicalClustering.numOfTotalClusters = args.getl("C", 0); @@ -151,7 +154,7 @@ class QbgCliBuildParameters : public QBG::BuildParameters { } hierarchicalClustering.numOfTotalBlobs = args.getl("b", 0); hierarchicalClustering.clusterID = args.getl("c", -1); - silence = !args.getBool("v"); + hierarchicalClustering.verbose = args.getBool("v"); char iMode = args.getChar("i", '-'); hierarchicalClustering.initMode = NGT::Clustering::InitializationModeKmeansPlusPlus; @@ -174,51 +177,69 @@ class QbgCliBuildParameters : public QBG::BuildParameters { hierarchicalClustering.extractCentroid = false; } + hierarchicalClustering.expectedRecall = args.getf("A", 0.98); + hierarchicalClustering.numOfFirstObjects = 0; hierarchicalClustering.numOfFirstClusters = 0; hierarchicalClustering.numOfSecondObjects = 0; hierarchicalClustering.numOfSecondClusters = 0; hierarchicalClustering.numOfThirdClusters = 0; + hierarchicalClustering.numOfThirdObjects = 0; - hierarchicalClustering.threeLayerClustering = true; + std::string blob = args.getString("B", "-"); - std::string blob = args.getString("B", ""); - if (blob == "-") { - hierarchicalClustering.threeLayerClustering = false; - } else { - hierarchicalClustering.threeLayerClustering = true; - } - if (hierarchicalClustering.threeLayerClustering) { + if (blob != "-") { std::vector tokens; NGT::Common::tokenize(blob, tokens, ","); - if (tokens.size() > 0) { + size_t idx = 0; + if (tokens.size() > 3) { + if (tokens[idx] == "3") { + hierarchicalClustering.clusteringType = QBG::HierarchicalKmeans::ClusteringTypeThreeLayer; + } else if (tokens[idx] == "21") { + hierarchicalClustering.clusteringType = QBG::HierarchicalKmeans::ClusteringTypeTwoPlusOneLayer; + } else if (tokens[idx] == "21N") { + hierarchicalClustering.clusteringType = QBG::HierarchicalKmeans::ClusteringTypeTwoPlusOneLayerWithNGT; + } else if (tokens[idx] == "22") { + hierarchicalClustering.clusteringType = QBG::HierarchicalKmeans::ClusteringTypeTwoPlusTwoLayer; + } else { + std::stringstream msg; + msg << "invalid clustering type. " << tokens[idx]; + NGTThrowException(msg); + } + idx++; + } else { + hierarchicalClustering.clusteringType = QBG::HierarchicalKmeans::ClusteringTypeThreeLayer; + } + if (tokens.size() > idx) { std::vector ftokens; - NGT::Common::tokenize(tokens[0], ftokens, ":"); + NGT::Common::tokenize(tokens[idx], ftokens, ":"); if (ftokens.size() >= 1) { hierarchicalClustering.numOfFirstObjects = NGT::Common::strtof(ftokens[0]); } if (ftokens.size() >= 2) { hierarchicalClustering.numOfFirstClusters = NGT::Common::strtof(ftokens[1]); } + idx++; } - if (tokens.size() > 1) { + if (tokens.size() > idx) { std::vector ftokens; - NGT::Common::tokenize(tokens[1], ftokens, ":"); + NGT::Common::tokenize(tokens[idx], ftokens, ":"); if (ftokens.size() >= 1) { hierarchicalClustering.numOfSecondObjects = NGT::Common::strtof(ftokens[0]); } if (ftokens.size() >= 2) { hierarchicalClustering.numOfSecondClusters = NGT::Common::strtof(ftokens[1]); } + idx++; } - if (tokens.size() > 2) { + if (tokens.size() > idx) { std::vector ftokens; - NGT::Common::tokenize(tokens[2], ftokens, ":"); + NGT::Common::tokenize(tokens[idx], ftokens, ":"); if (ftokens.size() >= 1) { if (ftokens[0] == "" || ftokens[0] == "-") { - hierarchicalClustering.numOfObjects = 0; + hierarchicalClustering.numOfThirdObjects = 0; } else { - hierarchicalClustering.numOfObjects = NGT::Common::strtof(ftokens[0]); + hierarchicalClustering.numOfThirdObjects = NGT::Common::strtof(ftokens[0]); } } if (ftokens.size() >= 2) { @@ -229,9 +250,9 @@ class QbgCliBuildParameters : public QBG::BuildParameters { } void getOptimizationParameters() { - optimization.numberOfObjects = args.getl("o", 1000); - optimization.numberOfClusters = args.getl("n", 0); - optimization.numberOfSubvectors = args.getl("m", 0); + optimization.numOfObjects = args.getl("o", 1000); + optimization.numOfClusters = args.getl("n", 0); + optimization.numOfSubvectors = args.getl("m", 0); optimization.randomizedObjectExtraction = true; @@ -292,14 +313,14 @@ class QbgCliBuildParameters : public QBG::BuildParameters { optimization.clusterSizeConstraintCoefficient = args.getf("s", 5.0); } - optimization.nOfMatrices = args.getl("M", 2); - optimization.seedStartObjectSizeRate = args.getf("S", 0.1); - optimization.seedStep = args.getl("X", 2); + optimization.numOfMatrices = args.getl("M", 2); + optimization.seedNumberOfSteps = args.getf("S", 2); + optimization.seedStep = args.getl("X", 10); optimization.reject = args.getf("R", 0.9); optimization.timelimit = args.getf("L", 24 * 1); optimization.timelimit *= 60.0 * 60.0; optimization.showClusterInfo = args.getBool("Z"); - silence = !args.getBool("v"); + optimization.verbose = args.getBool("v"); #ifdef NGTQG_NO_ROTATION char positionMode = args.getChar("P", 'n'); @@ -397,13 +418,13 @@ QBG::CLI::buildQG(NGT::Args &args) } if (phase == 0 || phase == 2) { std::cerr << "building the inverted index..." << std::endl; - bool silence = true; - QBG::Index::buildNGTQ(qgPath, silence); + bool verbose = false; + QBG::Index::buildNGTQ(qgPath, !verbose); } if (phase == 0 || phase == 3) { std::cerr << "building the quantized graph... " << std::endl; - bool silence = true; - NGTQG::Index::realign(indexPath, maxNumOfEdges, silence); + bool verbose = false; + NGTQG::Index::realign(indexPath, maxNumOfEdges, !verbose); } } @@ -604,6 +625,46 @@ QBG::CLI::appendQG(NGT::Args &args) } +void +QBG::CLI::info(NGT::Args &args) +{ + const string usage = "Usage: qbg index"; + + std::string indexPath; + try { + indexPath = args.get("#1"); + } catch (...) { + cerr << "Index is not specified" << endl; + cerr << usage << endl; + return; + } + + try { + bool readOnly = true; + try { + QBG::Index index(indexPath, readOnly); + } catch(NGT::Exception &err) { + readOnly = false; + } + QBG::Index index(indexPath, readOnly); + auto &quantizer = index.getQuantizer(); + std::cout << "The index type: QBG" << std::endl; + std::cout << "# of the dimensions: " << quantizer.globalCodebookIndex.getObjectSpace().getDimension() << std::endl; + std::cout << "# of the padded dimensions: " << quantizer.globalCodebookIndex.getObjectSpace().getPaddedDimension() << std::endl; + std::cout << "# of the stored objects: " << (quantizer.objectList.size() == 0 ? 0 : quantizer.objectList.size() - 1) << std::endl; + } catch(NGT::Exception &err) { + bool readOnly = true; + try { + NGTQG::Index index(indexPath, 128, readOnly); + std::cout << "The index type: QG" << std::endl; + } catch (...) { + cerr << "qbg: The specified index is neither QBG nor QG." << std::endl; + cerr << usage << endl; + } + } + +} + void QBG::CLI::create(NGT::Args &args) { @@ -687,13 +748,18 @@ QBG::CLI::load(NGT::Args &args) localCodebooks = args.get("#3"); } catch (...) {} + std::string quantizerCodebooks; + try { + quantizerCodebooks = args.get("#4"); + } catch (...) {} + std::string rotationPath; try { - rotationPath = args.get("#4"); + rotationPath = args.get("#5"); } catch (...) {} cerr << "rotation is " << rotationPath << "." << endl; - QBG::Index::load(indexPath, blobs, localCodebooks, rotationPath, threadSize); + QBG::Index::load(indexPath, blobs, localCodebooks, quantizerCodebooks, rotationPath, threadSize); } void @@ -707,7 +773,7 @@ QBG::CLI::search(NGT::Args &args) try { indexPath = args.get("#1"); } catch (...) { - cerr << "DB is not specified" << endl; + cerr << "Index is not specified" << endl; cerr << usage << endl; return; } @@ -725,18 +791,23 @@ QBG::CLI::search(NGT::Args &args) char outputMode = args.getChar("o", '-'); float epsilon = 0.1; - char searchMode = args.getChar("M", 'g'); + char searchMode = args.getChar("M", 'n'); if (args.getString("e", "none") == "-") { // linear search epsilon = FLT_MAX; } else { epsilon = args.getf("e", 0.1); } - float blobEpsilon = args.getf("B", 0.0); + float blobEpsilon = args.getf("B", 0.05); size_t edgeSize = args.getl("E", 0); float cutback = args.getf("C", 0.0); size_t explorationSize = args.getf("N", 256); size_t nOfProbes = args.getl("P", 10); + size_t nOfTrials = args.getl("T", 1); + if (nOfTrials != 1) { + std::cerr << "# of trials=" << nOfTrials << std::endl; + } + std::vector queryTimes; float beginOfResultExpansion, endOfResultExpansion, stepOfResultExpansion; bool mulStep = false; @@ -768,107 +839,107 @@ QBG::CLI::search(NGT::Args &args) QBG::Index index(indexPath, true); std::cerr << "qbg::The index is open." << std::endl; - std::cerr << " vmsize==" << NGT::Common::getProcessVmSizeStr() << std::endl; - std::cerr << " peak vmsize==" << NGT::Common::getProcessVmPeakStr() << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; auto dimension = index.getQuantizer().globalCodebookIndex.getObjectSpace().getDimension(); try { - ifstream is(query); - if (!is) { - cerr << "Cannot open the specified file. " << query << endl; - return; - } - if (outputMode == 's') { cout << "# Beginning of Evaluation" << endl; } - string line; - double totalTime = 0; - int queryCount = 0; - while(getline(is, line)) { - vector queryVector; - stringstream linestream(line); - while (!linestream.eof()) { - float value; - linestream >> value; - queryVector.push_back(value); + for (size_t trial = 0; trial < nOfTrials; trial++) { + ifstream is(query); + if (!is) { + cerr << "Cannot open the specified file. " << query << endl; + return; } - queryVector.resize(dimension); - queryCount++; - for (auto resultExpansion = beginOfResultExpansion; - resultExpansion <= endOfResultExpansion; - resultExpansion = mulStep ? resultExpansion * stepOfResultExpansion : - resultExpansion + stepOfResultExpansion) { - NGT::ObjectDistances objects; - NGT::Timer timer; - timer.start(); - QBG::SearchContainer searchContainer; - auto query = queryVector; - searchContainer.setObjectVector(query); - searchContainer.setResults(&objects); - if (resultExpansion >= 1.0) { - searchContainer.setSize(static_cast(size) * resultExpansion); - searchContainer.setExactResultSize(size); - } else { - searchContainer.setSize(size); - searchContainer.setExactResultSize(0); - } - searchContainer.setEpsilon(epsilon); - searchContainer.setBlobEpsilon(blobEpsilon); - searchContainer.setEdgeSize(edgeSize); - searchContainer.setCutback(cutback); - searchContainer.setGraphExplorationSize(explorationSize); - searchContainer.setNumOfProbes(nOfProbes); - switch (searchMode) { - case 'b': - index.searchBlobGraphNaively(searchContainer); - break; - case 'n': - index.searchBlobNaively(searchContainer); - break; - case 'g': - default: - index.searchBlobGraph(searchContainer); - break; - } - if (objects.size() > size) { - objects.resize(size); - } - timer.stop(); - totalTime += timer.time; - if (outputMode == 'e') { - cout << "# Query No.=" << queryCount << endl; - cout << "# Query=" << line.substr(0, 20) + " ..." << endl; - cout << "# Index Type=" << "----" << endl; - cout << "# Size=" << size << endl; - cout << "# Epsilon=" << epsilon << endl; - cout << "# Result expansion=" << resultExpansion << endl; - cout << "# Distance Computation=" << index.getQuantizer().distanceComputationCount << endl; - cout << "# Query Time (msec)=" << timer.time * 1000.0 << endl; - } else { - cout << "Query No." << queryCount << endl; - cout << "Rank\tIN-ID\tID\tDistance" << endl; + if (outputMode == 's') { cout << "# Beginning of Evaluation" << endl; } + string line; + double totalTime = 0; + int queryCount = 0; + while(getline(is, line)) { + vector queryVector; + stringstream linestream(line); + while (!linestream.eof()) { + float value; + linestream >> value; + queryVector.push_back(value); } + queryVector.resize(dimension); + queryCount++; + for (auto resultExpansion = beginOfResultExpansion; + resultExpansion <= endOfResultExpansion; + resultExpansion = mulStep ? resultExpansion * stepOfResultExpansion : + resultExpansion + stepOfResultExpansion) { + NGT::ObjectDistances objects; + QBG::SearchContainer searchContainer; + auto query = queryVector; + searchContainer.setObjectVector(query); + searchContainer.setResults(&objects); + if (resultExpansion >= 1.0) { + searchContainer.setSize(static_cast(size) * resultExpansion); + searchContainer.setExactResultSize(size); + } else { + searchContainer.setSize(size); + searchContainer.setExactResultSize(0); + } + searchContainer.setEpsilon(epsilon); + searchContainer.setBlobEpsilon(blobEpsilon); + searchContainer.setEdgeSize(edgeSize); + searchContainer.setCutback(cutback); + searchContainer.setGraphExplorationSize(explorationSize); + searchContainer.setNumOfProbes(nOfProbes); + NGT::Timer timer; + timer.start(); + switch (searchMode) { + case 'n': + index.searchInTwoSteps(searchContainer); + break; + case 'g': + default: + index.searchInOneStep(searchContainer); + break; + } + if (objects.size() > size) { + objects.resize(size); + } + timer.stop(); + totalTime += timer.time; + if (outputMode == 'e') { + cout << "# Query No.=" << queryCount << endl; + cout << "# Query=" << line.substr(0, 20) + " ..." << endl; + cout << "# Index Type=" << "----" << endl; + cout << "# Size=" << size << endl; + cout << "# Epsilon=" << epsilon << endl; + cout << "# Result expansion=" << resultExpansion << endl; + cout << "# Distance Computation=" << index.getQuantizer().distanceComputationCount << endl; + cout << "# Query Time (msec)=" << timer.time * 1000.0 << endl; + } else { + cout << "Query No." << queryCount << endl; + cout << "Rank\tIN-ID\tID\tDistance" << endl; + } - for (size_t i = 0; i < objects.size(); i++) { - cout << i + 1 << "\t" << objects[i].id << "\t"; - cout << objects[i].distance << endl; - } + for (size_t i = 0; i < objects.size(); i++) { + cout << i + 1 << "\t" << objects[i].id << "\t"; + cout << objects[i].distance << endl; + } + if (outputMode == 'e') { + cout << "# End of Search" << endl; + } else { + cout << "Query Time= " << timer.time << " (sec), " << timer.time * 1000.0 << " (msec)" << endl; + } + } if (outputMode == 'e') { - cout << "# End of Search" << endl; - } else { - cout << "Query Time= " << timer.time << " (sec), " << timer.time * 1000.0 << " (msec)" << endl; + cout << "# End of Query" << endl; } } + queryTimes.push_back(totalTime * 1000.0 / static_cast(queryCount)); if (outputMode == 'e') { - cout << "# End of Query" << endl; + cout << "# Average Query Time (msec)=" << queryTimes.back() << endl; + cout << "# Number of queries=" << queryCount << endl; + cout << "# End of Evaluation" << endl; + } else { + cout << "Average Query Time= " << totalTime / (double)queryCount << " (sec), " + << totalTime * 1000.0 / (double)queryCount << " (msec), (" + << totalTime << "/" << queryCount << ")" << endl; } - } - if (outputMode == 'e') { - cout << "# Average Query Time (msec)=" << totalTime * 1000.0 / (double)queryCount << endl; - cout << "# Number of queries=" << queryCount << endl; - cout << "# End of Evaluation" << endl; - } else { - cout << "Average Query Time= " << totalTime / (double)queryCount << " (sec), " - << totalTime * 1000.0 / (double)queryCount << " (msec), (" - << totalTime << "/" << queryCount << ")" << endl; } } catch (NGT::Exception &err) { cerr << "Error " << err.what() << endl; @@ -877,9 +948,15 @@ QBG::CLI::search(NGT::Args &args) cerr << "Error" << endl; cerr << usage << endl; } - std::cerr << "qbg::The end of search" << std::endl; - std::cerr << " vmsize==" << NGT::Common::getProcessVmSizeStr() << std::endl; - std::cerr << " peak vmsize==" << NGT::Common::getProcessVmPeakStr() << std::endl; + if (outputMode == 'e') { + if (nOfTrials >= 1) { + std::cout << "# Total minimum query time (msec)=" << *std::min_element(queryTimes.begin(), queryTimes.end()) + << "/" << nOfTrials << " (msec)" << std::endl; + } + std::cout << "# qbg: the end of search" << std::endl; + std::cout << "# vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cout << "# peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + } index.close(); } @@ -887,12 +964,13 @@ QBG::CLI::search(NGT::Args &args) void QBG::CLI::append(NGT::Args &args) { - const string usage = "Usage: qbg append [-n data-size] index(output) data.tsv(input)"; - string index; + const string usage = "Usage: qbg append [-n data-size] [-m b|e] [-v] index(output) data.tsv(input)"; + args.parse("v"); + string indexPath; try { - index = args.get("#1"); + indexPath = args.get("#1"); } catch (...) { - cerr << "DB is not specified." << endl; + cerr << "Index is not specified." << endl; cerr << usage << endl; return; } @@ -900,17 +978,37 @@ QBG::CLI::append(NGT::Args &args) try { data = args.get("#2"); } catch (...) { + cerr << usage << endl; cerr << "Data is not specified." << endl; } size_t dataSize = args.getl("n", 0); - char mode = args.getChar("m", '-'); + std::string mode = args.getString("m", ""); + bool verbose = args.getBool("v"); + + if (mode.find_first_of('e') != std::string::npos) { + QBG::Index index(indexPath, false); + std::cerr << "size=" << index.getQuantizer().objectList.size() << std::endl; + if (index.getQuantizer().objectList.size() > 1) { + if (verbose) { + std::cerr << "QBG: Error. The index is not empty." << std::endl; + cerr << usage << endl; + } + return; + } + } - if (mode == 'b') { - QBG::Index::appendBinary(index, data, dataSize); + std::cerr << "qbg: appending..." << std::endl; + NGT::Timer timer; + timer.start(); + if (mode.find_first_of('b') != std::string::npos) { + QBG::Index::appendBinary(indexPath, data, dataSize, !verbose); } else { - QBG::Index::append(index, data, dataSize); + QBG::Index::append(indexPath, data, dataSize, !verbose); } + timer.stop(); + std::cerr << "qbg: appending time=" << timer << std::endl; + } @@ -1070,24 +1168,78 @@ QBG::CLI::build(NGT::Args &args) return; } - size_t phase = args.getl("p", 0); + + std::string phaseString = args.getString("p", "1-3"); + bool phase[3]; + if (phaseString.empty()) { + phase[0] = phase[1] = phase[2] = true; + } else { + vector tokens; + NGT::Common::tokenize(phaseString, tokens, "-"); + int beginOfPhase, endOfPhase; + if (tokens.size() >= 1) { + if (tokens[0].empty()) { + beginOfPhase = endOfPhase = 0; + } else { + beginOfPhase = endOfPhase = NGT::Common::strtod(tokens[0]) - 1; + } + } + if (tokens.size() >= 2) { endOfPhase = NGT::Common::strtod(tokens[1]) - 1;} + if (tokens.size() >= 3 || tokens.size() == 0) { + cerr << "The specified phases are invalid! " << phaseString << endl; + cerr << usage << endl; + return; + } + phase[0] = phase[1] = phase[2] = false; + for (int p = beginOfPhase; p <= endOfPhase; p++) { + phase[p] = true; + } + } HierarchicalKmeans hierarchicalKmeans(buildParameters); - if (phase == 0 || phase == 1) { + if (phase[0]) { + std::cerr << "qbg: hierarchical clustering..." << std::endl; + NGT::Timer timer; + timer.start(); hierarchicalKmeans.clustering(indexPath); + timer.stop(); + if (buildParameters.verbose) { + std::cerr << "qbg: hierarchical clustering successfully completed." << std::endl;; + std::cerr << " ph0 time=" << timer << std::endl; + std::cerr << " ph0 vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " ph0 peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + } } QBG::Optimizer optimizer(buildParameters); - if (phase == 0 || phase == 2) { - std::cerr << "optimizing..." << std::endl; + if (phase[1]) { + std::cerr << "qbg: optimizing..." << std::endl; + NGT::Timer timer; + timer.start(); optimizer.optimize(indexPath); + timer.stop(); + if (buildParameters.verbose) { + std::cerr << "qbg: optimization successfully completed." << std::endl;; + std::cerr << " ph1 time=" << timer << std::endl; + std::cerr << " ph1 vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " ph1 peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + } } - if (phase == 0 || phase == 3) { - std::cerr << "building..." << std::endl; - QBG::Index::build(indexPath, optimizer.silence); + if (phase[2]) { + std::cerr << "qbg: building..." << std::endl; + NGT::Timer timer; + timer.start(); + QBG::Index::build(indexPath, optimizer.verbose); + timer.stop(); + if (buildParameters.verbose) { + std::cerr << "qbg: index build successfully completed." << std::endl;; + std::cerr << " ph2 time=" << timer << std::endl; + std::cerr << " ph2 vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " ph2 peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + } } } @@ -1129,6 +1281,13 @@ QBG::CLI::hierarchicalKmeans(NGT::Args &args) hierarchicalKmeans.clustering(indexPath, prefix, objectIDsFile); + + if (buildParameters.verbose) { + std::cerr << "qbg: the end of clustering" << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + } + } void @@ -1408,7 +1567,6 @@ QBG::CLI::gtRange(NGT::Args &args) for (size_t qidx = 0; qidx < numQueries; qidx++) { uint32_t v; stream.read(reinterpret_cast(&v), sizeof(v)); - //std::cerr << qidx << ":" << v << std::endl; numResultsPerQuery[qidx] = v; } { diff --git a/lib/NGT/NGTQ/QbgCli.h b/lib/NGT/NGTQ/QbgCli.h index aa33c6c..c50eefd 100644 --- a/lib/NGT/NGTQ/QbgCli.h +++ b/lib/NGT/NGTQ/QbgCli.h @@ -27,23 +27,23 @@ namespace QBG { int debugLevel; #if !defined(NGTQ_QBG) || defined(NGTQ_SHARED_INVERTED_INDEX) - void create(NGT::Args &args) {}; - void load(NGT::Args &args) {}; - void append(NGT::Args &args) {}; - void buildIndex(NGT::Args &args) {}; - void hierarchicalKmeans(NGT::Args &args) {}; - void search(NGT::Args &args) {}; - void assign(NGT::Args &args) {}; - void extract(NGT::Args &args) {}; - void gt(NGT::Args &args) {}; - void gtRange(NGT::Args &args) {}; - void optimize(NGT::Args &args) {}; - void build(NGT::Args &args) {}; - void createQG(NGT::Args &args) {}; - void buildQG(NGT::Args &args) {}; - void appendQG(NGT::Args &args) {}; - void searchQG(NGT::Args &args) {}; - void info(NGT::Args &args) {}; + void create(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void load(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void append(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void buildIndex(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void hierarchicalKmeans(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void search(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void assign(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void extract(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void gt(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void gtRange(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void optimize(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void build(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void createQG(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void buildQG(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void appendQG(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void searchQG(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; + void info(NGT::Args &args) { std::cerr << "not implemented." << std::endl; }; #else void create(NGT::Args &args); void load(NGT::Args &args); @@ -119,6 +119,8 @@ namespace QBG { appendQG(args); } else if (command == "search-qg") { searchQG(args); + } else if (command == "info") { + info(args); } else { cerr << "Illegal command. " << command << endl; } diff --git a/lib/NGT/NGTQ/QuantizedBlobGraph.h b/lib/NGT/NGTQ/QuantizedBlobGraph.h index 07c7005..b5dd01d 100644 --- a/lib/NGT/NGTQ/QuantizedBlobGraph.h +++ b/lib/NGT/NGTQ/QuantizedBlobGraph.h @@ -22,193 +22,257 @@ #ifdef NGTQ_QBG #include "NGT/NGTQ/QuantizedGraph.h" #include "NGT/NGTQ/Optimizer.h" +#include "NGT/NGTQ/HierarchicalKmeans.h" #include namespace QBG { - class BuildParameters { + class CreationParameters { public: - BuildParameters(){ setDefault(); } - + CreationParameters() { setDefault(); } void setDefault() { - creation.numOfObjects = 0; - creation.threadSize = 24; - creation.localCentroidLimit = 16; - creation.dimension = 0; + numOfObjects = 0; + threadSize = 24; + numOfLocalClusters = 16; + dimension = 0; #ifdef NGTQ_QBG - creation.genuineDimension = 0; - creation.dimensionOfSubvector = 1; - creation.genuineDataType = ObjectFile::DataTypeFloat; + genuineDimension = 0; + dimensionOfSubvector = 1; + genuineDataType = ObjectFile::DataTypeFloat; #endif - creation.dataType = NGTQ::DataTypeFloat; - creation.distanceType = NGTQ::DistanceType::DistanceTypeL2; - creation.singleLocalCodebook = false; - creation.localDivisionNo = 0; - creation.batchSize = 1000; - creation.centroidCreationMode = NGTQ::CentroidCreationModeStaticLayer; - creation.localCentroidCreationMode = NGTQ::CentroidCreationModeStatic; - creation.localIDByteSize = 1; - creation.localClusteringSampleCoefficient = 10; - creation.globalEdgeSizeForCreation = 10; - creation.globalEdgeSizeForSearch = 40; - creation.globalIndexType = NGT::Property::GraphAndTree; - creation.globalInsertionRadiusCoefficient = 1.1; - creation.globalGraphType = NGT::NeighborhoodGraph::GraphTypeANNG; - creation.localIndexType = NGT::Property::GraphAndTree; - creation.localInsertionRadiusCoefficient = 1.1; - creation.localGraphType = NGT::NeighborhoodGraph::GraphTypeANNG; - - hierarchicalClustering.maxSize = 1000; - hierarchicalClustering.numOfObjects = 0; - hierarchicalClustering.numOfClusters = 2; - hierarchicalClustering.numOfTotalClusters = 0; - hierarchicalClustering.numOfTotalBlobs = 0; - hierarchicalClustering.clusterID = -1; - hierarchicalClustering.initMode = NGT::Clustering::InitializationModeKmeansPlusPlus; - hierarchicalClustering.numOfRandomObjects = 0; - hierarchicalClustering.numOfFirstObjects = 0; - hierarchicalClustering.numOfFirstClusters = 0; - hierarchicalClustering.numOfSecondObjects = 0; - hierarchicalClustering.numOfSecondClusters = 0; - hierarchicalClustering.numOfThirdClusters = 0; - hierarchicalClustering.extractCentroid = false; - hierarchicalClustering.threeLayerClustering = true; - - optimization.clusteringType = NGT::Clustering::ClusteringTypeKmeansWithoutNGT; - optimization.initMode = NGT::Clustering::InitializationModeHead; - optimization.timelimit = 24 * 1 * 60.0 * 60.0; - optimization.iteration = 100; - optimization.clusterIteration = 100; - optimization.clusterSizeConstraint = false; - optimization.clusterSizeConstraintCoefficient = 5.0; - optimization.convergenceLimitTimes = 5; - optimization.numberOfObjects = 1000; - optimization.numberOfClusters = 0; - optimization.numberOfSubvectors = 0; - optimization.nOfMatrices = 2; - optimization.seedStartObjectSizeRate = 0.1; - optimization.seedStep = 2; - optimization.reject = 0.9; - optimization.repositioning = false; - optimization.rotation = true; - optimization.globalType = QBG::Optimizer::GlobalTypeNone; - optimization.randomizedObjectExtraction = true; - optimization.showClusterInfo = false; + dataType = NGTQ::DataTypeFloat; + distanceType = NGTQ::DistanceType::DistanceTypeL2; + singleLocalCodebook = false; + numOfSubvectors = 0; + batchSize = 1000; + centroidCreationMode = NGTQ::CentroidCreationModeStaticLayer; + localCentroidCreationMode = NGTQ::CentroidCreationModeStatic; + localIDByteSize = 1; + localClusteringSampleCoefficient = 10; + objectListOnMemory = false; + + globalEdgeSizeForCreation = 10; + globalEdgeSizeForSearch = 40; + globalIndexType = NGT::Property::GraphAndTree; + globalInsertionRadiusCoefficient = 1.1; + globalGraphType = NGT::NeighborhoodGraph::GraphTypeANNG; + + localIndexType = NGT::Property::GraphAndTree; + localInsertionRadiusCoefficient = 1.1; + localGraphType = NGT::NeighborhoodGraph::GraphTypeANNG; + + verbose = false; } - void setProperties(NGTQ::Property &property, NGT::Property &globalProperty, - NGT::Property &localProperty) { + static void setProperties(CreationParameters &creation, NGTQ::Property &property, NGT::Property &globalProperty, + NGT::Property &localProperty) { property.threadSize = creation.threadSize; property.globalCentroidLimit = 0; - property.localCentroidLimit = creation.localCentroidLimit; + property.localCentroidLimit = creation.numOfLocalClusters; property.dimension = creation.dimension; property.globalRange = 0; property.localRange = 0; - property.localCentroidLimit = creation.localCentroidLimit; + property.localCentroidLimit = creation.numOfLocalClusters; #ifdef NGTQ_QBG - property.genuineDimension = creation.genuineDimension; + property.genuineDimension = creation.genuineDimension; //-/property.dimensionOfSubvector = creation.dimensionOfSubvector; - property.genuineDataType = creation.genuineDataType; + property.genuineDataType = creation.genuineDataType; #endif - property.dataType = creation.dataType; - property.distanceType = creation.distanceType; - property.singleLocalCodebook = false; - property.localDivisionNo = creation.localDivisionNo; - property.batchSize = creation.batchSize; - property.centroidCreationMode = creation.centroidCreationMode; + property.dataType = creation.dataType; + property.distanceType = creation.distanceType; + property.singleLocalCodebook = false; + property.localDivisionNo = creation.numOfSubvectors; + property.batchSize = creation.batchSize; + property.centroidCreationMode = creation.centroidCreationMode; property.localCentroidCreationMode = creation.localCentroidCreationMode; - property.localIDByteSize = creation.localIDByteSize; + property.localIDByteSize = creation.localIDByteSize; property.localClusteringSampleCoefficient = creation.localClusteringSampleCoefficient; + property.objectListOnMemory = creation.objectListOnMemory; globalProperty.edgeSizeForCreation = creation.globalEdgeSizeForCreation; - globalProperty.edgeSizeForSearch = creation.globalEdgeSizeForSearch; - globalProperty.indexType = creation.globalIndexType; + globalProperty.edgeSizeForSearch = creation.globalEdgeSizeForSearch; + globalProperty.indexType = creation.globalIndexType; globalProperty.insertionRadiusCoefficient = creation.globalInsertionRadiusCoefficient; - globalProperty.graphType = creation.globalGraphType; - localProperty.indexType = creation.localIndexType; + globalProperty.graphType = creation.globalGraphType; + localProperty.indexType = creation.localIndexType; localProperty.insertionRadiusCoefficient = creation.localInsertionRadiusCoefficient; - localProperty.graphType = creation.localGraphType; + localProperty.graphType = creation.localGraphType; + if (property.localCentroidLimit >= 0xFF) { + if (property.localIDByteSize < 2) { + property.localIDByteSize = 2; + } + } else if (property.localCentroidLimit >= 0xFFFF) { + property.localIDByteSize = 4; + } + property.dimension = property.dimension == 0 ? property.genuineDimension : property.dimension; + property.localDivisionNo = property.localDivisionNo == 0 ? property.dimension : property.localDivisionNo; } - struct { - size_t numOfObjects; - size_t threadSize; - size_t localCentroidLimit; - size_t dimension; + + size_t numOfObjects; + size_t threadSize; + size_t numOfLocalClusters; + size_t dimension; #ifdef NGTQ_QBG - size_t genuineDimension; - size_t dimensionOfSubvector; - ObjectFile::DataType genuineDataType; + size_t genuineDimension; + size_t dimensionOfSubvector; + ObjectFile::DataType genuineDataType; #endif - NGTQ::DataType dataType; - NGTQ::DistanceType distanceType; - bool singleLocalCodebook; - size_t localDivisionNo; - size_t batchSize; - NGTQ::CentroidCreationMode centroidCreationMode; - NGTQ::CentroidCreationMode localCentroidCreationMode; - size_t localIDByteSize; - size_t localClusteringSampleCoefficient; - - size_t globalEdgeSizeForCreation; - size_t globalEdgeSizeForSearch; - NGT::Property::IndexType globalIndexType; - float globalInsertionRadiusCoefficient; - NGT::Property::GraphType globalGraphType; - - NGT::Property::IndexType localIndexType; - float localInsertionRadiusCoefficient; - NGT::Property::GraphType localGraphType; - } creation; - - struct { - size_t maxSize; - size_t numOfObjects; - size_t numOfClusters; - size_t numOfTotalClusters; - size_t numOfTotalBlobs; - int32_t clusterID; - - NGT::Clustering::InitializationMode initMode; - - size_t numOfRandomObjects; - - size_t numOfFirstObjects; - size_t numOfFirstClusters; - size_t numOfSecondObjects; - size_t numOfSecondClusters; - size_t numOfThirdClusters; - bool extractCentroid; - - bool threeLayerClustering; - } hierarchicalClustering; - - struct { - NGT::Clustering::ClusteringType clusteringType; - NGT::Clustering::InitializationMode initMode; - - float timelimit; - size_t iteration; - size_t clusterIteration; - bool clusterSizeConstraint; - float clusterSizeConstraintCoefficient; - size_t convergenceLimitTimes; - size_t numberOfObjects; - size_t numberOfClusters; - size_t numberOfSubvectors; - size_t nOfMatrices; - float seedStartObjectSizeRate; - size_t seedStep; - float reject; - bool repositioning; - bool rotation; - QBG::Optimizer::GlobalType globalType; - bool randomizedObjectExtraction; - bool showClusterInfo; - - } optimization; - - bool silence; + NGTQ::DataType dataType; + NGTQ::DistanceType distanceType; + bool singleLocalCodebook; + size_t numOfSubvectors; + size_t batchSize; + NGTQ::CentroidCreationMode centroidCreationMode; + NGTQ::CentroidCreationMode localCentroidCreationMode; + size_t localIDByteSize; + size_t localClusteringSampleCoefficient; + bool objectListOnMemory; + + size_t globalEdgeSizeForCreation; + size_t globalEdgeSizeForSearch; + NGT::Property::IndexType globalIndexType; + float globalInsertionRadiusCoefficient; + NGT::Property::GraphType globalGraphType; + + NGT::Property::IndexType localIndexType; + float localInsertionRadiusCoefficient; + NGT::Property::GraphType localGraphType; + + bool verbose; + }; + + class HierarchicalClusteringParameters { + public: + HierarchicalClusteringParameters() { setDefault(); } + void setDefault() { + maxSize = 1000; + numOfObjects = 0; + numOfClusters = 2; + numOfTotalClusters = 0; + numOfTotalBlobs = 0; + clusterID = -1; + initMode = NGT::Clustering::InitializationModeKmeansPlusPlus; + numOfRandomObjects = 0; + numOfFirstObjects = 0; + numOfFirstClusters = 0; + numOfSecondObjects = 0; + numOfSecondClusters = 0; + numOfThirdObjects = 0; + numOfThirdClusters = 0; + extractCentroid = false; + clusteringType = QBG::HierarchicalKmeans::ClusteringTypeThreeLayer; + epsilonExplorationSize = 1000; + expectedRecall = 0.98; + + verbose = false; + } + + size_t maxSize; + size_t numOfObjects; + size_t numOfClusters; + size_t numOfTotalClusters; + size_t numOfTotalBlobs; + int32_t clusterID; + + NGT::Clustering::InitializationMode initMode; + + size_t numOfRandomObjects; + + size_t numOfFirstObjects; + size_t numOfFirstClusters; + size_t numOfSecondObjects; + size_t numOfSecondClusters; + size_t numOfThirdObjects; + size_t numOfThirdClusters; + bool extractCentroid; + + QBG::HierarchicalKmeans::ClusteringType clusteringType; + size_t epsilonExplorationSize; + float expectedRecall; + + bool verbose; + }; + + class OptimizationParameters { + public: + OptimizationParameters() { setDefault(); } + void setDefault() { + clusteringType = NGT::Clustering::ClusteringTypeKmeansWithoutNGT; + initMode = NGT::Clustering::InitializationModeHead; + timelimit = 24 * 1 * 60.0 * 60.0; + iteration = 1000; + clusterIteration = 400; + clusterSizeConstraint = false; + clusterSizeConstraintCoefficient = 10.0; + convergenceLimitTimes = 5; + numOfObjects = 1000; + numOfClusters = 0; + numOfSubvectors = 0; + numOfMatrices = 1; + seedNumberOfSteps = 2; + seedStep = 10; + reject = 0.9; + repositioning = false; + rotation = true; + globalType = QBG::Optimizer::GlobalTypeNone; + randomizedObjectExtraction = true; + showClusterInfo = false; + + verbose = false; + } + NGT::Clustering::ClusteringType clusteringType; + NGT::Clustering::InitializationMode initMode; + + float timelimit; + size_t iteration; + size_t clusterIteration; + bool clusterSizeConstraint; + float clusterSizeConstraintCoefficient; + size_t convergenceLimitTimes; + size_t numOfObjects; + size_t numOfClusters; + size_t numOfSubvectors; + size_t numOfMatrices; + size_t seedNumberOfSteps; + size_t seedStep; + float reject; + bool repositioning; + bool rotation; + QBG::Optimizer::GlobalType globalType; + bool randomizedObjectExtraction; + bool showClusterInfo; + + bool verbose; + }; + + class BuildParameters { + public: + BuildParameters(){ setDefault(); } + + void setDefault() { + creation.setDefault(); + hierarchicalClustering.setDefault(); + optimization.setDefault(); + } + + void setProperties(NGTQ::Property &property, NGT::Property &globalProperty, + NGT::Property &localProperty) { + CreationParameters::setProperties(creation, property, globalProperty, localProperty); + } + + void setVerbose(bool s) { + creation.verbose = s; + hierarchicalClustering.verbose = s; + optimization.verbose = s; + verbose = s; + } + + CreationParameters creation; + HierarchicalClusteringParameters hierarchicalClustering; + OptimizationParameters optimization; + + bool verbose; }; @@ -270,11 +334,17 @@ namespace QBG { NGTQ::InvertedIndexEntry invertedIndexObjects(numOfSubspaces); quantizedIndex.getQuantizer().extractInvertedIndexObject(invertedIndexObjects, gid); quantizedIndex.getQuantizer().eraseInvertedIndexObject(gid); - NGTQ::QuantizedObjectProcessingStream quantizedStream(quantizedIndex.getQuantizer(), invertedIndexObjects.size()); - (*this)[gid].ids.reserve(invertedIndexObjects.size()); + NGTQ::QuantizedObjectProcessingStream quantizedStream(quantizedIndex.getQuantizer().divisionNo, invertedIndexObjects.size()); + rearrange(invertedIndexObjects, (*this)[gid], quantizedStream); + } +#endif + } + + static void rearrange(NGTQ::InvertedIndexEntry &invertedIndexObjects, NGTQG::QuantizedNode &rearrangedObjects, NGTQ::QuantizedObjectProcessingStream &quantizedStream) { + rearrangedObjects.ids.reserve(invertedIndexObjects.size()); for (size_t oidx = 0; oidx < invertedIndexObjects.size(); oidx++) { - (*this)[gid].ids.push_back(invertedIndexObjects[oidx].id); - for (size_t idx = 0; idx < numOfSubspaces; idx++) { + rearrangedObjects.ids.push_back(invertedIndexObjects[oidx].id); + for (size_t idx = 0; idx < invertedIndexObjects.numOfSubvectors; idx++) { #ifdef NGTQ_UINT8_LUT #ifdef NGTQ_SIMD_BLOCK_SIZE size_t dataNo = oidx; @@ -290,22 +360,48 @@ namespace QBG { objectData[idx * noobjs + dataNo] = invertedIndexObjects[oidx].localID[idx]; #endif } - } + } - (*this)[gid].subspaceID = invertedIndexObjects.subspaceID; - (*this)[gid].objects = quantizedStream.compressIntoUint4(); - } + rearrangedObjects.subspaceID = invertedIndexObjects.subspaceID; + rearrangedObjects.objects = quantizedStream.compressIntoUint4(); + } + + static void rearrange(NGTQ::InvertedIndexEntry &invertedIndexObjects, NGTQG::QuantizedNode &rearrangedObjects) { +#if defined(NGT_SHARED_MEMORY_ALLOCATOR) + std::cerr << "construct: Not implemented" << std::endl; + abort(); +#else + if (invertedIndexObjects.numOfSubvectors == 0) { + NGTThrowException("# of subvectors is zero."); + } + + //(*this).resize(quantizedIndex.getInvertedIndexSize()); + NGT::Timer timer; + timer.start(); + { + //NGTQ::InvertedIndexEntry invertedIndexObjects(numOfSubspaces); + //quantizedIndex.getQuantizer().extractInvertedIndexObject(invertedIndexObjects, gid); + //quantizedIndex.getQuantizer().eraseInvertedIndexObject(gid); + NGTQ::QuantizedObjectProcessingStream quantizedStream(invertedIndexObjects.numOfSubvectors, invertedIndexObjects.size()); + + rearrange(invertedIndexObjects, rearrangedObjects, quantizedStream); + } #endif } + static void rearrange(NGTQ::QuantizedObjectSet &quantizedObjects, NGTQG::QuantizedNode &rearrangedObjects) { + NGTQ::InvertedIndexEntry iie; + iie.set(quantizedObjects); + rearrange(iie, rearrangedObjects); + } }; class Index : public NGTQ::Index { public: - Index(const std::string &indexPath, bool readOnly = false, bool silence = true) : + Index(const std::string &indexPath, bool readOnly = false, bool verbose = false) : NGTQ::Index(indexPath, readOnly), path(indexPath), quantizedBlobGraph(*this) { searchable = false; - NGT::StdOstreamRedirector redirector(silence); + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); try { load(); @@ -323,27 +419,31 @@ namespace QBG { ~Index() {} - bool &getSilence() { return silence; } - - NGT::Object *allocateObject(std::vector &objectVector) { - auto &globalIndex = getQuantizer().globalCodebookIndex; - auto dim = getQuantizer().property.dimension; - objectVector.resize(dim, 0); - return globalIndex.allocateObject(objectVector); - } + bool &getVerbose() { return verbose; } #ifdef NGTQ_QBG static void create(const std::string &index, BuildParameters &buildParameters, - std::vector *rotation,const std::string &objectFile) { + std::vector *rotation = 0,const std::string objectFile = "") { + create(index, buildParameters.creation, rotation, objectFile); + } + static void create(const std::string &index, + CreationParameters &creation, + std::vector *rotation = 0,const std::string objectFile = "") { NGTQ::Property property; NGT::Property globalProperty; NGT::Property localProperty; - buildParameters.setProperties(property, globalProperty, localProperty); + CreationParameters::setProperties(creation, property, globalProperty, localProperty); property.quantizerType = NGTQ::QuantizerTypeQBG; NGTQ::Index::create(index, property, globalProperty, localProperty, rotation, objectFile); } #endif +#ifdef NGTQ_QBG + static void initialize(NGTQ::Property &property, NGT::Property &globalProperty,NGT::Property &localProperty) { + QBG::CreationParameters params; + QBG::CreationParameters::setProperties(params, property, globalProperty, localProperty); + } +#endif static void create(const std::string &index, NGTQ::Property &property, NGT::Property &globalProperty, @@ -362,9 +462,9 @@ namespace QBG { #endif } - static void load(const std::string &indexPath, const std::vector &rotation) { + static void load(const std::string &indexPath, const std::vector> &quantizerCodebook, const std::vector &rotation) { NGTQ::Index index(indexPath); - index.getQuantizer().saveRotation(rotation); + index.getQuantizer().loadQuantizationCodebookAndRotation(quantizerCodebook, rotation); } void insert(const size_t id, std::vector &object) { @@ -387,11 +487,12 @@ namespace QBG { static void append(const std::string &indexName, // index file const std::string &data, // data file size_t dataSize = 0, // data size - bool silence = true + bool verbose = false ) { - NGT::StdOstreamRedirector redirector(silence); + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); QBG::Index index(indexName); + auto &quantizer = index.getQuantizer(); istream *is; if (data == "-") { is = &cin; @@ -406,9 +507,11 @@ namespace QBG { } string line; vector > objects; + size_t idx = quantizer.objectList.size() == 0 ? 0 : quantizer.objectList.size() - 1; size_t count = 0; // extract objects from the file and insert them to the object list. while(getline(*is, line)) { + idx++; count++; std::vector object; NGT::Common::extractVector(line, " ,\t", object); @@ -416,11 +519,14 @@ namespace QBG { cerr << "An empty line or invalid value: " << line << endl; continue; } - index.insert(count, object); + index.insert(idx, object); if (count % 100000 == 0) { - cerr << "Processed " << count; - cerr << endl; + std::cerr << "appended " << static_cast(count) / 1000000.0 << "M objects."; + if (count != idx) { + std::cerr << " # of the total objects=" << static_cast(idx) / 1000000.0 << "M"; + } + cerr << " virtual memory(kbyte)=" << NGT::Common::getProcessVmSize() << std::endl; } } if (data != "-") { @@ -435,11 +541,12 @@ namespace QBG { static void appendBinary(const std::string &indexName, // index file const std::string &data, // data file size_t dataSize = 0, // data size - bool silence = true + bool verbose = false ) { - NGT::StdOstreamRedirector redirector(silence); + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); QBG::Index index(indexName); + auto &quantizer = index.getQuantizer(); std::vector tokens; NGT::Common::tokenize(data, tokens, "."); if (tokens.size() < 2) { @@ -448,16 +555,22 @@ namespace QBG { NGTThrowException(msg); } StaticObjectFileLoader loader(data, tokens[tokens.size() - 1]); - size_t idx = 0; + size_t idx = quantizer.objectList.size() == 0 ? 0 : quantizer.objectList.size() - 1; + size_t count = 0; while (!loader.isEmpty()) { idx++; + count++; if (dataSize > 0 && idx > dataSize) { break; } auto object = loader.getObject(); index.insert(idx, object); - if (idx % 100000 == 0) { - std::cerr << "loaded " << static_cast(idx) / 1000000.0 << "M objects." << std::endl; + if (count % 1000000 == 0) { + std::cerr << "appended " << static_cast(count) / 1000000.0 << "M objects."; + if (count != idx) { + std::cerr << " # of the total objects=" << static_cast(idx) / 1000000.0 << "M"; + } + cerr << " virtual memory(kbyte)=" << NGT::Common::getProcessVmSize() << std::endl; } } index.save(); @@ -465,10 +578,77 @@ namespace QBG { redirector.end(); } + float getApproximateDistances(std::vector &query, NGTQG::RearrangedQuantizedObjectSet &quantizedObjects, + size_t subspaceID, std::vector &distances) { + if (query.empty()) { + NGTThrowException("The specified query is empty."); + } + auto &quantizer = this->getQuantizer(); + if (quantizer.getNumOfLocalClusters() != 16) { + std::stringstream msg; + msg << "# of the local clusters is not 16. " << quantizer.getNumOfLocalClusters() << std::endl; + NGTThrowException(msg); + } + distances.clear(); + auto noOfObjects = quantizedObjects.ids.size(); + if (noOfObjects == 0) { + return 0.0; + } + auto rotatedQuery = query; + auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); + quantizedObjectDistance.rotation->mul(rotatedQuery.data()); + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 lookupTable; + quantizedObjectDistance.initialize(lookupTable); + quantizedObjectDistance.createDistanceLookup(rotatedQuery.data(), subspaceID, lookupTable); + distances.resize(NGTQ::QuantizedObjectProcessingStream::getNumOfAlignedObjects(noOfObjects)); + auto minDistance = quantizedObjectDistance(quantizedObjects.objects, distances.data(), noOfObjects, lookupTable); + distances.resize(noOfObjects); + return minDistance; + } + + void getApproximateDistances(std::vector &query, NGTQ::QuantizedObjectSet &quantizedObjects, + size_t subspaceID, std::vector &distances) { + if (query.empty()) { + NGTThrowException("The specified query is empty."); + } + auto &quantizer = this->getQuantizer(); + distances.clear(); + auto noOfObjects = quantizedObjects.size(); + if (noOfObjects == 0) { + return; + } + auto rotatedQuery = query; + auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); + quantizedObjectDistance.rotation->mul(rotatedQuery.data()); + NGTQ::QuantizedObjectDistance::DistanceLookupTable lookupTable; + quantizedObjectDistance.initialize(lookupTable); + quantizedObjectDistance.createDistanceLookup(rotatedQuery.data(), subspaceID, lookupTable); + distances.resize(noOfObjects); + if (quantizer.localIDByteSize == 1) { + NGTQ::InvertedIndexEntry iie; + iie.set(quantizedObjects); + for (size_t idx = 0; idx < iie.size(); idx++) { + distances[idx] = quantizedObjectDistance(&iie[idx].localID[0], lookupTable); + } + } else if (quantizer.localIDByteSize == 2) { + NGTQ::InvertedIndexEntry iie; + iie.set(quantizedObjects); + for (size_t idx = 0; idx < iie.size(); idx++) { + distances[idx] = quantizedObjectDistance(&iie[idx].localID[0], lookupTable); + } + } else if (quantizer.localIDByteSize == 4) { + NGTQ::InvertedIndexEntry iie; + iie.set(quantizedObjects); + for (size_t idx = 0; idx < iie.size(); idx++) { + distances[idx] = quantizedObjectDistance(&iie[idx].localID[0], lookupTable); + } + } + } + static void appendFromObjectRepository(const std::string &ngtIndex, // QG const std::string &qgIndex, // NGT - bool silence = true) { - NGT::StdOstreamRedirector redirector(silence); + bool verbose = false) { + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); NGT::Index ngt(ngtIndex); @@ -658,367 +838,425 @@ namespace QBG { } + static void refineDistances(QBG::SearchContainer &searchContainer, NGTQ::Quantizer &quantizer, + NGT::NeighborhoodGraph::ResultSet &result, + NGT::ObjectDistances &qresults) { + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + NGT::ResultPriorityQueue qres; + if (objectSpace.getObjectType() == typeid(float)) { + refineDistances(searchContainer, quantizer, result, qres); + } else if (objectSpace.getObjectType() == typeid(uint8_t)) { + refineDistances(searchContainer, quantizer, result, qres); + } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { + refineDistances(searchContainer, quantizer, result, qres); + } else { + std::stringstream msg; + msg << "refineDistances: Fatal error! Invalid datatype. " << objectSpace.getObjectType().name() << std::endl; + NGTThrowException(msg); + } + qresults.resize(qres.size()); + for (int i = qresults.size() - 1; i >= 0; i--) { + qresults[i] = qres.top(); + qres.pop(); + } + } + + static void refineDistances(QBG::SearchContainer &searchContainer, NGTQ::Quantizer &quantizer, + NGT::NeighborhoodGraph::ResultSet &result, + NGT::ResultPriorityQueue &qresults) { + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + if (objectSpace.getObjectType() == typeid(float)) { + refineDistances(searchContainer, quantizer, result, qresults); + } else if (objectSpace.getObjectType() == typeid(uint8_t)) { + refineDistances(searchContainer, quantizer, result, qresults); + } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { + refineDistances(searchContainer, quantizer, result, qresults); + } else { + std::stringstream msg; + msg << "refineDistances: Fatal error! Invalid datatype. " << objectSpace.getObjectType().name() << std::endl; + NGTThrowException(msg); + } + } + + template + static void refineDistances(QBG::SearchContainer &searchContainer, NGTQ::Quantizer &quantizer, + NGT::NeighborhoodGraph::ResultSet &result, + NGT::ResultPriorityQueue &qresults) { + qresults = NGT::ResultPriorityQueue(); + NGT::Object &query = searchContainer.object; + auto &objectSpace = quantizer.globalCodebookIndex.getObjectSpace(); + auto paddedDimension = objectSpace.getPaddedDimension(); + const size_t prefetchSize = objectSpace.getPrefetchSize(); +#ifdef NGTQ_OBJECT_IN_MEMORY + if (quantizer.objectListOnMemory.size() != 0) { + while (!result.empty()) { + auto r = result.top(); + result.pop(); + NGT::Object &object = *quantizer.objectListOnMemory.get(r.id); + if (!result.empty()) { + uint8_t *ptr = static_cast(quantizer.objectListOnMemory.get(result.top().id)->getPointer()); + NGT::MemoryCache::prefetch(ptr, prefetchSize); + } + r.distance = objectSpace.getComparator()(query, object); + qresults.push(r); + } + } else { +#endif + auto threadid = omp_get_thread_num(); + while (!result.empty()) { + auto r = result.top(); + result.pop(); + std::vector object; +#ifdef MULTIPLE_OBJECT_LISTS + quantizer.objectList.get(threadid, r.id, object); +#else + quantizer.objectList.get(r.id, object); +#endif + r.distance = NGT::PrimitiveComparator::compareL2(static_cast(query.getPointer()), + static_cast(object.data()), paddedDimension); + - void searchBlobNaively(QBG::SearchContainer &searchContainer) { + qresults.push(r); + } +#ifdef NGTQ_OBJECT_IN_MEMORY + } +#endif + while (qresults.size() > searchContainer.exactResultSize) { + qresults.pop(); + } + + } + + void searchInTwoSteps(QBG::SearchContainer &searchContainer) { + if (searchContainer.isEmptyObject()) { + NGT::Object query(searchContainer.objectVector, getQuantizer().globalCodebookIndex.getObjectSpace()); + SearchContainer sc(searchContainer, query); + searchInTwoSteps(sc); + searchContainer.workingResult = std::move(sc.workingResult); + return; + } NGT::ObjectDistances blobs; NGT::SearchContainer sc(searchContainer); sc.setResults(&blobs); + sc.setEpsilon(searchContainer.blobExplorationCoefficient - 1.0); sc.setSize(searchContainer.numOfProbes); auto &quantizer = getQuantizer(); auto &globalIndex = quantizer.globalCodebookIndex; - auto &objectSpace = globalIndex.getObjectSpace(); - globalIndex.search(sc); - + auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); + if (searchContainer.objectVector.size() == 0) { + NGTThrowException("search: object is null."); + } + std::vector rotatedQuery = searchContainer.objectVector; + { + NGT::Object *query = allocateObject(searchContainer.objectVector); + NGT::SearchContainer tsc(sc, *query); + tsc.setResults(&sc.getResult()); + globalIndex.search(tsc); + globalIndex.deleteObject(query); + } if (blobs.empty()) { - std::cerr << "something wrong." << std::endl; + std::cerr << "Warning: No blobs can be searched." << std::endl; + std::cerr << " global index size=" << globalIndex.getObjectRepositorySize() << std::endl; + std::cerr << " size=" << sc.size << std::endl; return; } - auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); - NGT::Object rotatedQuery(&objectSpace); - objectSpace.copy(rotatedQuery, searchContainer.object); - #if defined(NGTQG_ROTATION) - quantizedObjectDistance.rotation->mul(static_cast(rotatedQuery.getPointer())); + if (quantizedObjectDistance.rotation != 0) { + quantizedObjectDistance.rotation->mul(rotatedQuery.data()); + } #endif std::unordered_map luts; size_t foundCount = 0; - size_t k = searchContainer.size; NGT::Distance radius = FLT_MAX; - NGT::Distance distance; NGT::NeighborhoodGraph::ResultSet result; - +#ifdef NGTQBG_COARSE_BLOB + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 lookupTable; + quantizedObjectDistance.initialize(lookupTable); +#endif for (size_t idx = 0; idx < blobs.size(); idx++) { - auto blobID = blobs[idx].id; - auto subspaceID = quantizedBlobGraph[blobID].subspaceID; - auto luti = luts.find(subspaceID); - if (luti == luts.end()) { - luts.insert(std::make_pair(subspaceID, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8())); - luti = luts.find(subspaceID); - quantizedObjectDistance.initialize((*luti).second); - quantizedObjectDistance.createDistanceLookup(rotatedQuery, subspaceID, (*luti).second); - } - std::tie(distance, radius) = judge(quantizedBlobGraph[blobID], k, radius, (*luti).second, result, foundCount); +#ifdef NGTQBG_COARSE_BLOB + NGT::Distance blobDistance = std::numeric_limits::max(); + auto graphNodeID = blobs[idx].id; + auto &graphNodeToInvertedIndexEntries = quantizer.getGraphNodeToInvertedIndexEntries(); + auto beginIvtID = graphNodeToInvertedIndexEntries[graphNodeID - 1] + 1; + auto endIvtID = graphNodeToInvertedIndexEntries[graphNodeID] + 1; + for (auto blobID = beginIvtID; blobID < endIvtID; blobID++) { + auto subspaceID = quantizedBlobGraph[blobID].subspaceID; + quantizedObjectDistance.createDistanceLookup(rotatedQuery, subspaceID, lookupTable); + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 &lut = lookupTable; +#else + { + auto blobID = blobs[idx].id; + auto subspaceID = quantizedBlobGraph[blobID].subspaceID; + auto luti = luts.find(subspaceID); + if (luti == luts.end()) { + luts.insert(std::make_pair(subspaceID, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8())); + luti = luts.find(subspaceID); + quantizedObjectDistance.initialize((*luti).second); + quantizedObjectDistance.createDistanceLookup(rotatedQuery.data(), subspaceID, (*luti).second); + } + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 &lut = (*luti).second; +#endif + NGT::Distance bd; + std::tie(bd, radius) = judge(quantizedBlobGraph[blobID], k, radius, lut, result, foundCount); +#ifdef NGTQBG_COARSE_BLOB + if (bd < blobDistance) { + blobDistance = bd; + } +#else +#endif + } +#ifdef NGTQBG_MIN +#endif } - if (searchContainer.resultIsAvailable()) { - searchContainer.getResult().clear(); - searchContainer.getResult().moveFrom(result); + if (searchContainer.exactResultSize > 0) { + NGT::ObjectDistances &qresults = searchContainer.getResult(); + refineDistances(searchContainer, quantizer, result, qresults); + } else { + searchContainer.getResult().moveFrom(result); + } } else { - searchContainer.workingResult = result; + if (searchContainer.exactResultSize > 0) { + refineDistances(searchContainer, quantizer, result, searchContainer.workingResult); + } else { + searchContainer.workingResult = std::move(result); + } } + } - + void searchInOneStep(QBG::SearchContainer &searchContainer) { + auto &globalIndex = getQuantizer().globalCodebookIndex; + auto &globalGraph = static_cast(globalIndex.getIndex()); + NGT::ObjectDistances seeds; + const size_t dimension = globalIndex.getObjectSpace().getPaddedDimension(); + if (dimension > searchContainer.objectVector.size()) { + searchContainer.objectVector.resize(dimension); + } + NGT::Object query(searchContainer.objectVector, globalIndex.getObjectSpace()); + SearchContainer sc(searchContainer, query); + globalGraph.getSeedsFromTree(sc, seeds); + if (seeds.empty()) { + globalGraph.getRandomSeeds(globalGraph.repository, seeds, 20); + } + searchInOneStep(sc, seeds); + searchContainer.workingResult = std::move(sc.workingResult); } - void searchBlobGraph(QBG::SearchContainer &searchContainer) { - auto &globalIndex = getQuantizer().globalCodebookIndex; - auto &globalGraph = static_cast(globalIndex.getIndex()); - NGT::ObjectDistances seeds; - NGT::Object *query = allocateObject(searchContainer.objectVector); - SearchContainer sc(searchContainer, *query); - globalGraph.getSeedsFromTree(sc, seeds); - if (seeds.empty()) { - globalGraph.getRandomSeeds(globalGraph.repository, seeds, 20); - } - searchBlobGraph(sc, seeds); - globalIndex.deleteObject(query); - searchContainer.workingResult = std::move(sc.workingResult); - } - - void searchBlobGraph(QBG::SearchContainer &searchContainer, NGT::ObjectDistances &seeds) { + void searchInOneStep(QBG::SearchContainer &searchContainer, NGT::ObjectDistances &seeds) { #if defined(NGT_SHARED_MEMORY_ALLOCATOR) - std::cerr << "searchBlobGraph: Not implemented. " << std::endl; - abort(); + std::cerr << "searchBlobGraph: Not implemented. " << std::endl; + abort(); #else - if (!searchable) { - std::stringstream msg; - msg << "The specified index is not now searchable. "; - NGTThrowException(msg); - } + if (!searchable) { + std::stringstream msg; + msg << "The specified index is not now searchable. "; + NGTThrowException(msg); + } - auto &quantizer = getQuantizer(); - auto &globalIndex = quantizer.globalCodebookIndex; - auto &globalGraph = static_cast(globalIndex.getIndex()); - auto &objectSpace = globalIndex.getObjectSpace(); + auto &quantizer = getQuantizer(); + auto &globalIndex = quantizer.globalCodebookIndex; + auto &globalGraph = static_cast(globalIndex.getIndex()); + auto &objectSpace = globalIndex.getObjectSpace(); - if (searchContainer.explorationCoefficient == 0.0) { - searchContainer.explorationCoefficient = NGT_EXPLORATION_COEFFICIENT; - } + if (globalGraph.searchRepository.empty()) { + NGTThrowException("QBG:Index: graph repository is empty."); + } + if (searchContainer.explorationCoefficient == 0.0) { + searchContainer.explorationCoefficient = NGT_EXPLORATION_COEFFICIENT; + } - const auto requestedSize = searchContainer.size; - searchContainer.size = std::numeric_limits::max(); + const auto requestedSize = searchContainer.size; + searchContainer.size = std::numeric_limits::max(); - // setup edgeSize - size_t edgeSize = globalGraph.getEdgeSize(searchContainer); + // setup edgeSize + size_t edgeSize = globalGraph.getEdgeSize(searchContainer); - NGT::NeighborhoodGraph::UncheckedSet untracedNodes; + NGT::NeighborhoodGraph::UncheckedSet untracedNodes; - NGT::NeighborhoodGraph::DistanceCheckedSet distanceChecked(globalGraph.searchRepository.size()); - NGT::NeighborhoodGraph::ResultSet results; + NGT::NeighborhoodGraph::DistanceCheckedSet distanceChecked(globalGraph.searchRepository.size()); + NGT::NeighborhoodGraph::ResultSet results; - if (objectSpace.getObjectType() == typeid(float)) { - globalGraph.setupDistances(searchContainer, seeds, NGT::PrimitiveComparator::L2Float::compare); + if (objectSpace.getObjectType() == typeid(float)) { + globalGraph.setupDistances(searchContainer, seeds, NGT::PrimitiveComparator::L2Float::compare); + } else if (objectSpace.getObjectType() == typeid(uint8_t)) { + globalGraph.setupDistances(searchContainer, seeds, NGT::PrimitiveComparator::L2Uint8::compare); #ifdef NGT_HALF_FLOAT - } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { - globalGraph.setupDistances(searchContainer, seeds, NGT::PrimitiveComparator::L2Float16::compare); - } + } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { + globalGraph.setupDistances(searchContainer, seeds, NGT::PrimitiveComparator::L2Float16::compare); + } #endif - std::sort(seeds.begin(), seeds.end()); - NGT::ObjectDistance currentNearestBlob = seeds.front(); - NGT::Distance explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; - std::priority_queue, std::greater> discardedObjects; - untracedNodes.push(seeds.front()); - distanceChecked.insert(seeds.front().id); - for (size_t i = 1; i < seeds.size(); i++) { - untracedNodes.push(seeds[i]); - distanceChecked.insert(seeds[i].id); - discardedObjects.push(seeds[i]); - } - size_t explorationSize = 1; - auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); - std::unordered_map luts; - std::vector &rotatedQuery = searchContainer.objectVector; - if (objectSpace.getObjectType() == typeid(float)) { - memcpy(rotatedQuery.data(), searchContainer.object.getPointer(), rotatedQuery.size() * sizeof(float)); - } else if (objectSpace.getObjectType() == typeid(uint8_t)) { - auto *ptr = static_cast(searchContainer.object.getPointer()); - for (size_t i = 0; i < rotatedQuery.size(); i++) { - rotatedQuery[i] = ptr[i]; + std::sort(seeds.begin(), seeds.end()); + NGT::ObjectDistance currentNearestBlob = seeds.front(); + NGT::Distance explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; + std::priority_queue, std::greater> discardedObjects; + untracedNodes.push(seeds.front()); + distanceChecked.insert(seeds.front().id); + for (size_t i = 1; i < seeds.size(); i++) { + untracedNodes.push(seeds[i]); + distanceChecked.insert(seeds[i].id); + discardedObjects.push(seeds[i]); } -#ifdef NGT_HALF_FLOAT - } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { - auto *ptr = static_cast(searchContainer.object.getPointer()); - for (size_t i = 0; i < rotatedQuery.size(); i++) { - rotatedQuery[i] = ptr[i]; + size_t explorationSize = 1; + auto &quantizedObjectDistance = quantizer.getQuantizedObjectDistance(); + std::unordered_map luts; + std::vector rotatedQuery = searchContainer.objectVector; + quantizedObjectDistance.rotation->mul(rotatedQuery.data()); + NGT::Distance radius = searchContainer.radius; + if (requestedSize >= std::numeric_limits::max()) { + radius *= searchContainer.explorationCoefficient; } + NGT::ReadOnlyGraphNode *nodes = globalGraph.searchRepository.data(); + NGT::ReadOnlyGraphNode *neighbors = 0; + NGT::ObjectDistance target; + const size_t prefetchSize = objectSpace.getPrefetchSize(); + const size_t prefetchOffset = objectSpace.getPrefetchOffset(); + pair *neighborptr; + pair *neighborendptr; +#ifdef NGTQBG_COARSE_BLOB + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 lookupTable; + quantizedObjectDistance.initialize(lookupTable); #endif - } else { - std::cerr << "Fatal inner error! Invalid object type." << std::endl; - } - quantizedObjectDistance.rotation->mul(rotatedQuery.data()); - NGT::Distance radius = searchContainer.radius; - if (requestedSize >= std::numeric_limits::max()) { - radius *= searchContainer.explorationCoefficient; - } - const size_t dimension = objectSpace.getPaddedDimension(); - if (globalGraph.searchRepository.empty()) { - NGTThrowException("QBG:Index: searchRepository is empty."); - } - NGT::ReadOnlyGraphNode *nodes = globalGraph.searchRepository.data(); - NGT::ReadOnlyGraphNode *neighbors = 0; - NGT::ObjectDistance target; - const size_t prefetchSize = objectSpace.getPrefetchSize(); - const size_t prefetchOffset = objectSpace.getPrefetchOffset(); - pair *neighborptr; - pair *neighborendptr; - for (;;) { - if (untracedNodes.empty() || untracedNodes.top().distance > explorationRadius) { - explorationSize++; - auto blobID = currentNearestBlob.id; - auto subspaceID = quantizedBlobGraph[blobID].subspaceID; - auto luti = luts.find(subspaceID); - if (luti == luts.end()) { - luts.insert(std::make_pair(subspaceID, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8())); - luti = luts.find(subspaceID); - quantizedObjectDistance.initialize((*luti).second); - quantizedObjectDistance.createDistanceLookup(rotatedQuery.data(), subspaceID, (*luti).second); - } - NGT::Distance blobDistance; - size_t foundCount; - std::tie(blobDistance, radius) = judge(quantizedBlobGraph[blobID], requestedSize, - radius, (*luti).second, results, foundCount); + for (;;) { + if (untracedNodes.empty() || untracedNodes.top().distance > explorationRadius) { + explorationSize++; + NGT::Distance blobDistance = std::numeric_limits::max(); +#ifdef NGTQBG_COARSE_BLOB + auto graphNodeID = currentNearestBlob.id; + auto &graphNodeToInvertedIndexEntries = quantizer.getGraphNodeToInvertedIndexEntries(); + auto beginIvtID = graphNodeToInvertedIndexEntries[graphNodeID - 1] + 1; + auto endIvtID = graphNodeToInvertedIndexEntries[graphNodeID] + 1; + for (auto blobID = beginIvtID; blobID < endIvtID; blobID++) { + auto subspaceID = quantizedBlobGraph[blobID].subspaceID; + quantizedObjectDistance.createDistanceLookup(rotatedQuery.data(), subspaceID, lookupTable); + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 &lut = lookupTable; +#else + { + auto blobID = currentNearestBlob.id; + auto subspaceID = quantizedBlobGraph[blobID].subspaceID; + auto luti = luts.find(subspaceID); + if (luti == luts.end()) { + luts.insert(std::make_pair(subspaceID, NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8())); + luti = luts.find(subspaceID); + quantizedObjectDistance.initialize((*luti).second); + quantizedObjectDistance.createDistanceLookup(rotatedQuery.data(), subspaceID, (*luti).second); + } + NGTQ::QuantizedObjectDistance::DistanceLookupTableUint8 &lut = (*luti).second; +#endif + size_t foundCount; + NGT::Distance bd; + std::tie(bd, radius) = judge(quantizedBlobGraph[blobID], requestedSize, + radius, lut, results, foundCount); +#ifdef NGTQBG_COARSE_BLOB + if (bd < blobDistance) { + blobDistance = bd; + } +#else + blobDistance = bd; +#endif + } + #ifdef NGTQBG_MIN - if (blobDistance > radius * searchContainer.explorationCoefficient) { - break; - } + if (blobDistance > radius * searchContainer.explorationCoefficient) { + break; + } #endif - if (explorationSize > searchContainer.graphExplorationSize) { - break; - } - if (discardedObjects.empty()) { - break; + if (explorationSize > searchContainer.graphExplorationSize) { + break; + } + if (discardedObjects.empty()) { + break; + } + currentNearestBlob = discardedObjects.top(); + discardedObjects.pop(); + explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; + continue; } - currentNearestBlob = discardedObjects.top(); - discardedObjects.pop(); - explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; - continue; - } - target = untracedNodes.top(); - untracedNodes.pop(); + target = untracedNodes.top(); + untracedNodes.pop(); - neighbors = &nodes[target.id]; - neighborptr = &(*neighbors)[0]; - size_t neighborSize = neighbors->size() < edgeSize ? neighbors->size() : edgeSize; - neighborendptr = neighborptr + neighborSize; + neighbors = &nodes[target.id]; + neighborptr = &(*neighbors)[0]; + size_t neighborSize = neighbors->size() < edgeSize ? neighbors->size() : edgeSize; + neighborendptr = neighborptr + neighborSize; - pair* nsPtrs[neighborSize]; - size_t nsPtrsSize = 0; + pair* nsPtrs[neighborSize]; + size_t nsPtrsSize = 0; #ifndef PREFETCH_DISABLE - for (; neighborptr < neighborendptr; ++neighborptr) { + for (; neighborptr < neighborendptr; ++neighborptr) { #ifdef NGT_VISIT_COUNT - searchContainer.visitCount++; + searchContainer.visitCount++; #endif - if (!distanceChecked[(*(neighborptr)).first]) { - distanceChecked.insert((*(neighborptr)).first); - nsPtrs[nsPtrsSize] = neighborptr; - if (nsPtrsSize < prefetchOffset) { - unsigned char *ptr = reinterpret_cast((*(neighborptr)).second); - NGT::MemoryCache::prefetch(ptr, prefetchSize); - } - nsPtrsSize++; - } - } + if (!distanceChecked[(*(neighborptr)).first]) { + distanceChecked.insert((*(neighborptr)).first); + nsPtrs[nsPtrsSize] = neighborptr; + if (nsPtrsSize < prefetchOffset) { + unsigned char *ptr = reinterpret_cast((*(neighborptr)).second); + NGT::MemoryCache::prefetch(ptr, prefetchSize); + } + nsPtrsSize++; + } + } #endif #ifdef PREFETCH_DISABLE - for (; neighborptr < neighborendptr; ++neighborptr) { + for (; neighborptr < neighborendptr; ++neighborptr) { #else - for (size_t idx = 0; idx < nsPtrsSize; idx++) { + for (size_t idx = 0; idx < nsPtrsSize; idx++) { #endif #ifdef PREFETCH_DISABLE - if (distanceChecked[(*(neighborptr)).first]) { - continue; - } - distanceChecked.insert((*(neighborptr)).first); + if (distanceChecked[(*(neighborptr)).first]) { + continue; + } + distanceChecked.insert((*(neighborptr)).first); #else - neighborptr = nsPtrs[idx]; - if (idx + prefetchOffset < nsPtrsSize) { - unsigned char *ptr = reinterpret_cast((*(nsPtrs[idx + prefetchOffset])).second); - NGT::MemoryCache::prefetch(ptr, prefetchSize); - } + neighborptr = nsPtrs[idx]; + if (idx + prefetchOffset < nsPtrsSize) { + unsigned char *ptr = reinterpret_cast((*(nsPtrs[idx + prefetchOffset])).second); + NGT::MemoryCache::prefetch(ptr, prefetchSize); + } #endif #ifdef NGT_DISTANCE_COMPUTATION_COUNT - searchContainer.distanceComputationCount++; -#endif - NGT::Distance distance = 0.0; - if (objectSpace.getObjectType() == typeid(float)) { - distance = NGT::PrimitiveComparator::L2Float::compare(searchContainer.object.getPointer(), - neighborptr->second->getPointer(), dimension); - } else if (objectSpace.getObjectType() == typeid(uint8_t)) { - distance = NGT::PrimitiveComparator::L2Uint8::compare(searchContainer.object.getPointer(), - neighborptr->second->getPointer(), dimension); -#ifdef NGT_HALF_FLOAT - } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { - distance = NGT::PrimitiveComparator::L2Float16::compare(searchContainer.object.getPointer(), - neighborptr->second->getPointer(), dimension); + searchContainer.distanceComputationCount++; #endif - } else { - assert(false); - } - NGT::ObjectDistance r; - r.set(neighborptr->first, distance); - untracedNodes.push(r); - if (distance < currentNearestBlob.distance) { - discardedObjects.push(currentNearestBlob); - currentNearestBlob = r; - explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; - } else { - discardedObjects.push(r); - } + NGT::Distance distance = objectSpace.getComparator()(searchContainer.object, *neighborptr->second); + NGT::ObjectDistance r; + r.set(neighborptr->first, distance); + untracedNodes.push(r); + if (distance < currentNearestBlob.distance) { + discardedObjects.push(currentNearestBlob); + currentNearestBlob = r; + explorationRadius = searchContainer.blobExplorationCoefficient * currentNearestBlob.distance; + } else { + discardedObjects.push(r); + } + } } - } - if (searchContainer.resultIsAvailable()) { - if (searchContainer.exactResultSize > 0) { - NGT::ObjectDistances &qresults = searchContainer.getResult(); - auto threadid = omp_get_thread_num(); - auto paddedDimension = getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); - NGT::ResultPriorityQueue rs; - qresults.resize(results.size()); - size_t idx = results.size(); - while (!results.empty()) { - auto r = results.top(); - results.pop(); - if (objectSpace.getObjectType() == typeid(float)) { - std::vector object; -#ifdef MULTIPLE_OBJECT_LISTS - quantizer.objectList.get(threadid, r.id, object); -#else - quantizer.objectList.get(r.id, object); -#endif - r.distance = NGT::PrimitiveComparator::compareL2(static_cast(searchContainer.object.getPointer()), - static_cast(object.data()), paddedDimension); - } else if (objectSpace.getObjectType() == typeid(uint8_t)) { - std::vector object; -#ifdef MULTIPLE_OBJECT_LISTS - quantizer.objectList.get(threadid, r.id, object); -#else - quantizer.objectList.get(r.id, object); -#endif - r.distance = NGT::PrimitiveComparator::compareL2(static_cast(searchContainer.object.getPointer()), - static_cast(object.data()), paddedDimension); -#ifdef NGT_HALF_FLOAT - } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { - std::vector object; -#ifdef MULTIPLE_OBJECT_LISTS - quantizer.objectList.get(threadid, r.id, object); -#else - quantizer.objectList.get(r.id, object); -#endif - r.distance = NGT::PrimitiveComparator::compareL2(static_cast(searchContainer.object.getPointer()), - static_cast(object.data()), paddedDimension); -#endif - } - qresults[--idx] = r; + if (searchContainer.resultIsAvailable()) { + if (searchContainer.exactResultSize > 0) { + NGT::ObjectDistances &qresults = searchContainer.getResult(); + refineDistances(searchContainer, quantizer, results, qresults); + } else { + searchContainer.getResult().moveFrom(results); } - std::sort(qresults.begin(), qresults.end()); - qresults.resize(searchContainer.exactResultSize); } else { - NGT::ObjectDistances &qresults = searchContainer.getResult(); - qresults.moveFrom(results); - } - } else { - if (searchContainer.exactResultSize > 0) { - auto threadid = omp_get_thread_num(); - auto paddedDimension = getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); - NGT::ResultPriorityQueue rs; - while (!results.empty()) { - auto r = results.top(); - results.pop(); - if (objectSpace.getObjectType() == typeid(float)) { - std::vector object; -#ifdef MULTIPLE_OBJECT_LISTS - quantizer.objectList.get(threadid, r.id, object); -#else - quantizer.objectList.get(r.id, object); -#endif - r.distance = NGT::PrimitiveComparator::compareL2(static_cast(searchContainer.object.getPointer()), - static_cast(object.data()), paddedDimension); - } else if (objectSpace.getObjectType() == typeid(uint8_t)) { - std::vector object; -#ifdef MULTIPLE_OBJECT_LISTS - quantizer.objectList.get(threadid, r.id, object); -#else - quantizer.objectList.get(r.id, object); -#endif - r.distance = NGT::PrimitiveComparator::compareL2(reinterpret_cast(searchContainer.object.getPointer()), - reinterpret_cast(object.data()), paddedDimension); -#ifdef NGT_HALF_FLOAT - } else if (objectSpace.getObjectType() == typeid(NGT::float16)) { - std::vector object; -#ifdef MULTIPLE_OBJECT_LISTS - quantizer.objectList.get(threadid, r.id, object); -#else - quantizer.objectList.get(r.id, object); -#endif - r.distance = NGT::PrimitiveComparator::compareL2(reinterpret_cast(searchContainer.object.getPointer()), - reinterpret_cast(object.data()), paddedDimension); -#endif - } - rs.push(r); + if (searchContainer.exactResultSize > 0) { + refineDistances(searchContainer, quantizer, results, searchContainer.workingResult); + } else { + searchContainer.workingResult = std::move(results); } - results = std::move(rs); - } else { - searchContainer.workingResult = std::move(results); } - } #endif - } + } + void search(QBG::SearchContainer &searchContainer) { + searchInOneStep(searchContainer); + } void save() { quantizedBlobGraph.save(path); } @@ -1031,30 +1269,34 @@ namespace QBG { } } - static void buildNGTQ(const std::string &indexPath, bool silence = true) { - load(indexPath, QBG::Index::getQuantizerCodebookFile(indexPath), "", ""); - buildNGTQ(indexPath, "", "-", "-", 1, 0, silence); - std::cerr << "NGTQ and NGTQBG indices are completed." << std::endl; - std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; - std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + static void buildNGTQ(const std::string &indexPath, bool verbose = false) { + load(indexPath, QBG::Index::getQuantizerCodebookFile(indexPath), "", ""); + buildNGTQ(indexPath, "", "-", "-", 1, 0, verbose); + if (verbose) { + std::cerr << "NGTQ and NGTQBG indices are completed." << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + } } - static void build(const std::string &indexPath, bool silence = true) { - load(indexPath, "", "", ""); - buildNGTQ(indexPath, "", "", "", 1, 0, silence); - buildQBG(indexPath, silence); - std::cerr << "NGTQ and NGTQBG indices are completed." << std::endl; - std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; - std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + static void build(const std::string &indexPath, bool verbose = false) { + load(indexPath, "", "", ""); + buildNGTQ(indexPath, "", "", "", 1, 0, verbose); + buildQBG(indexPath, verbose); + if (verbose) { + std::cerr << "NGTQ and NGTQBG indices are completed." << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + } } static void build(const std::string &indexPath, std::string quantizerCodebookFile = "", std::string codebookIndexFile = "", std::string objectIndexFile = "", - size_t beginID = 1, size_t endID = 0, bool silence = true) { - buildNGTQ(indexPath, quantizerCodebookFile, codebookIndexFile, objectIndexFile, beginID, endID, silence); - buildQBG(indexPath, silence); + size_t beginID = 1, size_t endID = 0, bool verbose = false) { + buildNGTQ(indexPath, quantizerCodebookFile, codebookIndexFile, objectIndexFile, beginID, endID, verbose); + buildQBG(indexPath, verbose); std::cerr << "NGTQ and NGTQBG indices are completed." << std::endl; std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; @@ -1076,7 +1318,7 @@ namespace QBG { std::string quantizerCodebookFile = "", std::string codebookIndexFile = "", std::string objectIndexFile = "", - size_t beginID = 1, size_t endID = 0, bool silence = true) { + size_t beginID = 1, size_t endID = 0, bool verbose = false) { std::vector> quantizerCodebook; std::vector codebookIndex; std::vector objectIndex; @@ -1085,28 +1327,30 @@ namespace QBG { if (codebookPath.empty()) { codebookPath = QBG::Index::getQuantizerCodebookFile(indexPath); } - std::ifstream stream(codebookPath); - if (!stream) { - std::stringstream msg; - msg << "Cannot open the codebook. " << codebookPath; - NGTThrowException(msg); - } - std::string line; - while (getline(stream, line)) { - std::vector tokens; - NGT::Common::tokenize(line, tokens, " \t"); - std::vector object; - for (auto &token : tokens) { - object.push_back(NGT::Common::strtof(token)); - } - if (!quantizerCodebook.empty() && quantizerCodebook[0].size() != object.size()) { + if (codebookPath != "-") { + std::ifstream stream(codebookPath); + if (!stream) { std::stringstream msg; - msg << "The specified quantizer codebook is invalid. " << quantizerCodebook[0].size() - << ":" << object.size() << ":" << quantizerCodebook.size() << ":" << line; + msg << "Cannot open the codebook. " << codebookPath; NGTThrowException(msg); } - if (!object.empty()) { - quantizerCodebook.push_back(object); + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + std::vector object; + for (auto &token : tokens) { + object.push_back(NGT::Common::strtof(token)); + } + if (!quantizerCodebook.empty() && quantizerCodebook[0].size() != object.size()) { + std::stringstream msg; + msg << "The specified quantizer codebook is invalid. " << quantizerCodebook[0].size() + << ":" << object.size() << ":" << quantizerCodebook.size() << ":" << line; + NGTThrowException(msg); + } + if (!object.empty()) { + quantizerCodebook.push_back(object); + } } } } @@ -1116,7 +1360,6 @@ namespace QBG { codebookIndexPath = QBG::Index::getCodebookIndexFile(indexPath); } if (codebookIndexPath != "-") { - cerr << "buildNGTQ: codebook index is " << codebookIndexPath << "." << endl; std::ifstream stream(codebookIndexPath); if (!stream) { std::stringstream msg; @@ -1156,48 +1399,59 @@ namespace QBG { std::vector object; if (tokens.size() != 1) { std::stringstream msg; - msg << "The specified codebook index is invalid. " << line; + msg << "The specified object index is invalid. " << line; NGTThrowException(msg); } objectIndex.push_back(NGT::Common::strtol(tokens[0])); } } } - buildNGTQ(indexPath, quantizerCodebook, codebookIndex, objectIndex, beginID, endID, silence); + buildNGTQ(indexPath, quantizerCodebook, codebookIndex, objectIndex, beginID, endID, verbose); } static void buildNGTQ(const std::string &indexPath, std::vector> &quantizerCodebook, std::vector &codebookIndex, std::vector &objectIndex, - size_t beginID = 1, size_t endID = 0, bool silence = true) { - NGT::StdOstreamRedirector redirector(silence); + size_t beginID = 1, size_t endID = 0, bool verbose = false) { + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); NGT::Timer timer; timer.start(); NGTQ::Index index(indexPath); - if (quantizerCodebook.size() == 0) { + if ((quantizerCodebook.size() == 0) && (codebookIndex.size() == 0) && (objectIndex.size() == 0)) { index.createIndex(beginID, endID); } else { if (codebookIndex.size() == 0) { codebookIndex.resize(quantizerCodebook.size()); } + if (codebookIndex.size() == 0) { + stringstream msg; + msg << "The specified codebook indexe invalild " << codebookIndex.size(); + NGTThrowException(msg); + } if (objectIndex.size() == 0) { size_t size = index.getQuantizer().objectList.size(); size = size == 0 ? 0 : size - 1; objectIndex.resize(size); } - if ((quantizerCodebook.size() == 0) || (codebookIndex.size() == 0)) { - stringstream msg; - msg << "The specified codebooks or indexes are invalild " << quantizerCodebook.size() << ":" << codebookIndex.size(); - NGTThrowException(msg); - } index.createIndex(quantizerCodebook, codebookIndex, objectIndex, beginID, endID); } - const string com = "rm -rf " + indexPath + "/" + getWorkspaceName(); - if (system(com.c_str()) == -1) { - std::cerr << "Warning. cannot remove the workspace directory. " << std::endl; + { + char *s = getenv("NGT_NOT_REMOVE_WORKSPACE"); + if (s == 0) { + const string comrmdir = "rm -rf " + indexPath + "/" + getWorkspaceName(); + if (system(comrmdir.c_str()) == -1) { + std::cerr << "Warning. cannot remove the workspace directory. " + << comrmdir << std::endl; + } + } + const string comrm = "rm -f " + indexPath + "/" + NGTQ::Quantizer::getInvertedIndexFile(); + if (system(comrm.c_str()) == -1) { + std::cerr << "Warning. cannot remove the indeverted index. " + << comrm << std::endl; + } } timer.stop(); @@ -1210,12 +1464,11 @@ namespace QBG { redirector.end(); } - static void buildQBG(const std::string &indexPath, bool silence = true) { - std::cerr << "build QBG" << std::endl; + static void buildQBG(const std::string &indexPath, bool verbose = false) { NGT::Timer timer; timer.start(); auto readOnly = false; - QBG::Index index(indexPath, readOnly, silence); + QBG::Index index(indexPath, readOnly, verbose); try { index.load(); stringstream msg; @@ -1225,11 +1478,13 @@ namespace QBG { index.quantizedBlobGraph.construct(index); timer.stop(); - std::cerr << "QBG index is completed." << std::endl; - std::cerr << " time=" << timer << std::endl; - std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; - std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; - std::cerr << "saving..." << std::endl; + if (verbose) { + std::cerr << "QBG index is completed." << std::endl; + std::cerr << " time=" << timer << std::endl; + std::cerr << " vmsize=" << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << " peak vmsize=" << NGT::Common::getProcessVmPeakStr() << std::endl; + std::cerr << "saving..." << std::endl; + } index.save(); } @@ -1338,7 +1593,7 @@ namespace QBG { static void - load(std::string indexPath, std::string blobs = "", std::string localCodebooks = "", std::string rotationPath = "", int threadSize = 0) + load(std::string indexPath, std::string blobs = "", std::string localCodebooks = "", std::string quantizerCodebook = "", std::string rotationPath = "", int threadSize = 0) { if (blobs.empty()) { blobs = QBG::Index::getBlobFile(indexPath); @@ -1346,6 +1601,9 @@ namespace QBG { if (localCodebooks.empty()) { localCodebooks = QBG::Index::getPQFile(indexPath) + "/" + QBG::Index::getSubvectorPrefix() + "-@"; } + if (quantizerCodebook.empty()) { + quantizerCodebook = QBG::Index::getQuantizerCodebookFile(indexPath); + } if (rotationPath.empty()) { rotationPath = QBG::Index::getRotationFile(indexPath); } @@ -1354,8 +1612,37 @@ namespace QBG { assert(threadSize != 0); size_t dataSize = 0; - NGT::Index::append(indexPath + "/global", blobs, threadSize, dataSize); + { + const char *ngtDirString = "/tmp/ngt-XXXXXX"; + char ngtDir[strlen(ngtDirString) + 1]; + strcpy(ngtDir, ngtDirString); + std::string tmpDir = mkdtemp(ngtDir); + const std::string mvcom = "mv " + indexPath + "/" + NGTQ::Quantizer::getGlobalFile() + + " " + tmpDir + "/"; + if (system(mvcom.c_str()) == -1) { + std::stringstream msg; + msg << "Error! moving is failed. " << mvcom; + NGTThrowException(msg); + } + NGT::Index::append(tmpDir + "/" + NGTQ::Quantizer::getGlobalFile(), blobs, threadSize, dataSize); + + auto unlog = false; + NGT::GraphOptimizer graphOptimizer(unlog); + graphOptimizer.searchParameterOptimization = false; + graphOptimizer.prefetchParameterOptimization = false; + graphOptimizer.accuracyTableGeneration = false; + int numOfOutgoingEdges = 10; + int numOfIncomingEdges = 120; + int numOfQueries = 200; + int numOfResultantObjects = 20; + graphOptimizer.set(numOfOutgoingEdges, numOfIncomingEdges, numOfQueries, numOfResultantObjects); + graphOptimizer.execute(tmpDir + "/" + NGTQ::Quantizer::getGlobalFile(), indexPath + "/global"); + const std::string rmcom = "rm -rf " + tmpDir; + if (system(rmcom.c_str()) == -1) { + std::cerr << "Warning. remove is failed. " << rmcom << std::endl; + } + } NGTQ::Property property; property.load(indexPath); @@ -1368,31 +1655,59 @@ namespace QBG { std::stringstream data; data << tokens[0] << no << tokens[1]; std::stringstream localCodebook; - localCodebook << indexPath << "/local-" << no; + localCodebook << indexPath << "/" + NGTQ::Quantizer::getLocalPrefix() << no; std::cerr << data.str() << "->" << localCodebook.str() << std::endl; NGT::Index::append(localCodebook.str(), data.str(), threadSize, dataSize); } - cerr << "qbg: loading the rotation..." << endl; #ifdef NGTQ_QBG - std::vector rotation; - - std::ifstream stream(rotationPath); - if (!stream) { - std::stringstream msg; - msg << "Cannot open the rotation. " << rotationPath; - NGTThrowException(msg); + std::vector> qCodebook; + { + std::ifstream stream(quantizerCodebook); + if (!stream) { + std::stringstream msg; + msg << "Cannot open the codebook. " << quantizerCodebook; + NGTThrowException(msg); + } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + std::vector object; + for (auto &token : tokens) { + object.push_back(NGT::Common::strtof(token)); + } + if (!qCodebook.empty() && qCodebook[0].size() != object.size()) { + std::stringstream msg; + msg << "The specified quantizer codebook is invalid. " << qCodebook[0].size() + << ":" << object.size() << ":" << qCodebook.size() << ":" << line; + NGTThrowException(msg); + } + if (!object.empty()) { + qCodebook.push_back(object); + } + } } - std::string line; - while (getline(stream, line)) { - std::vector tokens; - NGT::Common::tokenize(line, tokens, " \t"); - for (auto &token : tokens) { - rotation.push_back(NGT::Common::strtof(token)); + { + cerr << "qbg: loading the rotation..." << endl; + std::vector rotation; + + std::ifstream stream(rotationPath); + if (!stream) { + std::stringstream msg; + msg << "Cannot open the rotation. " << rotationPath; + NGTThrowException(msg); } + std::string line; + while (getline(stream, line)) { + std::vector tokens; + NGT::Common::tokenize(line, tokens, " \t"); + for (auto &token : tokens) { + rotation.push_back(NGT::Common::strtof(token)); + } + } + QBG::Index::load(indexPath, qCodebook, rotation); } - std::cerr << "rotation matrix size=" << rotation.size() << std::endl; - QBG::Index::load(indexPath, rotation); #endif } @@ -1410,8 +1725,13 @@ namespace QBG { static const std::string getTrainObjectFile(std::string indexPath) { return getWorkSpacePrefix(indexPath) + "/" + getObjectFile(); } static const std::string getPrefix(std::string indexPath) { return getWorkSpacePrefix(indexPath) + "/" + getHierarchicalClusteringPrefix(); } static const std::string getPQFile(std::string indexPath) { return getPrefix(indexPath) + "_opt"; } +#ifdef NGTQBG_COARSE_BLOB + static const std::string getBlobFile(std::string indexPath) { return getPrefix(indexPath) + getSecondCentroidSuffix(); } + static const std::string getQuantizerCodebookFile(std::string indexPath) { return getPrefix(indexPath) + getThirdCentroidSuffix(); } +#else static const std::string getBlobFile(std::string indexPath) { return getPrefix(indexPath) + getThirdCentroidSuffix(); } static const std::string getQuantizerCodebookFile(std::string indexPath) { return getPrefix(indexPath) + getSecondCentroidSuffix(); } +#endif static const std::string getCodebookIndexFile(std::string indexPath) { return getPrefix(indexPath) + get3rdTo2ndSuffix(); } static const std::string getObjectIndexFile(std::string indexPath) { return getPrefix(indexPath) + getObjTo3rdSuffix(); } static const std::string getRotationFile(std::string indexPath) { return getPQFile(indexPath) + "/" + getRotationFile diff --git a/lib/NGT/NGTQ/QuantizedGraph.cpp b/lib/NGT/NGTQ/QuantizedGraph.cpp index 183b314..189399a 100644 --- a/lib/NGT/NGTQ/QuantizedGraph.cpp +++ b/lib/NGT/NGTQ/QuantizedGraph.cpp @@ -19,7 +19,7 @@ #include "NGT/NGTQ/Optimizer.h" #ifdef NGTQ_QBG -void NGTQG::Index::quantize(const std::string indexPath, size_t dimensionOfSubvector, size_t maxNumOfEdges, bool silence) { +void NGTQG::Index::quantize(const std::string indexPath, size_t dimensionOfSubvector, size_t maxNumOfEdges, bool verbose) { { NGT::Index index(indexPath); const std::string quantizedIndexPath = indexPath + "/qg"; @@ -29,7 +29,7 @@ void NGTQG::Index::quantize(const std::string indexPath, size_t dimensionOfSubve index.getProperty(ngtProperty); QBG::BuildParameters buildParameters; buildParameters.creation.dimensionOfSubvector = dimensionOfSubvector; - buildParameters.silence = silence; + buildParameters.setVerbose(verbose); NGTQG::Index::create(indexPath, buildParameters); @@ -51,9 +51,9 @@ void NGTQG::Index::quantize(const std::string indexPath, size_t dimensionOfSubve optimizer.optimize(quantizedIndexPath); - QBG::Index::buildNGTQ(quantizedIndexPath, silence); + QBG::Index::buildNGTQ(quantizedIndexPath, verbose); - NGTQG::Index::realign(indexPath, maxNumOfEdges, silence); + NGTQG::Index::realign(indexPath, maxNumOfEdges, verbose); } } @@ -62,19 +62,19 @@ void NGTQG::Index::quantize(const std::string indexPath, size_t dimensionOfSubve void NGTQG::Index::create(const std::string indexPath, QBG::BuildParameters &buildParameters) { auto dimensionOfSubvector = buildParameters.creation.dimensionOfSubvector; auto dimension = buildParameters.creation.dimension; - if (dimension != 0 && buildParameters.creation.localDivisionNo != 0) { - if (dimension % buildParameters.creation.localDivisionNo != 0) { + if (dimension != 0 && buildParameters.creation.numOfSubvectors != 0) { + if (dimension % buildParameters.creation.numOfSubvectors != 0) { std::stringstream msg; - msg << "NGTQBG:Index::create: Invalid dimension and local division No. " << dimension << ":" << buildParameters.creation.localDivisionNo; + msg << "NGTQBG:Index::create: Invalid dimension and local division No. " << dimension << ":" << buildParameters.creation.numOfSubvectors; NGTThrowException(msg); } - dimensionOfSubvector = dimension / buildParameters.creation.localDivisionNo; + dimensionOfSubvector = dimension / buildParameters.creation.numOfSubvectors; } create(indexPath, dimensionOfSubvector, dimension); } void NGTQG::Index::append(const std::string indexPath, QBG::BuildParameters &buildParameters) { - QBG::Index::appendFromObjectRepository(indexPath, indexPath + "/qg", buildParameters.silence); + QBG::Index::appendFromObjectRepository(indexPath, indexPath + "/qg", buildParameters.verbose); } #endif diff --git a/lib/NGT/NGTQ/QuantizedGraph.h b/lib/NGT/NGTQ/QuantizedGraph.h index 90cc907..5fa8a0c 100644 --- a/lib/NGT/NGTQ/QuantizedGraph.h +++ b/lib/NGT/NGTQ/QuantizedGraph.h @@ -55,6 +55,8 @@ namespace NGTQG { std::vector ids; void *objects; }; + + typedef QuantizedNode RearrangedQuantizedObjectSet; class QuantizedGraphRepository : public std::vector { typedef std::vector PARENT; @@ -91,7 +93,7 @@ namespace NGTQG { NGT::GraphNode &node = *graphRepository.VECTOR::get(id); size_t numOfEdges = node.size() < maxNoOfEdges ? node.size() : maxNoOfEdges; (*this)[id].ids.reserve(numOfEdges); - NGTQ::QuantizedObjectProcessingStream quantizedStream(quantizedIndex.getQuantizer(), numOfEdges); + NGTQ::QuantizedObjectProcessingStream quantizedStream(quantizedIndex.getQuantizer().divisionNo, numOfEdges); #ifdef NGT_SHARED_MEMORY_ALLOCATOR for (auto i = node.begin(graphRepository.allocator); i != node.end(graphRepository.allocator); ++i) { if (distance(node.begin(graphRepository.allocator), i) >= static_cast(numOfEdges)) { @@ -597,10 +599,10 @@ namespace NGTQG { static void append(const std::string indexPath, QBG::BuildParameters &buildParameters); #ifdef NGTQ_QBG - static void quantize(const std::string indexPath, size_t dimensionOfSubvector, size_t maxNumOfEdges, bool silence = true); + static void quantize(const std::string indexPath, size_t dimensionOfSubvector, size_t maxNumOfEdges, bool verbose = false); - static void realign(const std::string indexPath, size_t maxNumOfEdges, bool silence = true) { - NGT::StdOstreamRedirector redirector(silence); + static void realign(const std::string indexPath, size_t maxNumOfEdges, bool verbose = false) { + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); { std::string quantizedIndexPath = indexPath + "/qg"; @@ -618,8 +620,8 @@ namespace NGTQG { redirector.end(); } #else - static void quantize(const std::string indexPath, float dimensionOfSubvector, size_t maxNumOfEdges, bool silence = true) { - NGT::StdOstreamRedirector redirector(silence); + static void quantize(const std::string indexPath, float dimensionOfSubvector, size_t maxNumOfEdges, bool verbose = false) { + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); { NGT::Index index(indexPath); diff --git a/lib/NGT/NGTQ/Quantizer.h b/lib/NGT/NGTQ/Quantizer.h index ad09a4e..c30f6db 100644 --- a/lib/NGT/NGTQ/Quantizer.h +++ b/lib/NGT/NGTQ/Quantizer.h @@ -36,11 +36,14 @@ #endif #define NGTQBG_MIN +//#define NGTQBG_COARSE_BLOB +#define NGTQ_USING_ONNG #define MULTIPLE_OBJECT_LISTS #define NGTQG_ROTATION #define NGTQ_BLAS_FOR_ROTATION #define NGTQG_ROTATED_GLOBAL_CODEBOOKS +#define NGTQ_OBJECT_IN_MEMORY #define NGTQ_UINT8_LUT #define NGTQ_SIMD_BLOCK_SIZE 16 @@ -169,8 +172,25 @@ class Rotation : public std::vector { } + bool isIdentity() { + for (size_t i = 0; i < dim; i++) { + for (size_t j = 0; j < dim; j++) { + if (i == j) { + if ((*this)[i * dim + j] != 1.0) { + return false; + } + } else { + if ((*this)[i * dim + j] != 0.0) { + return false; + } + } + } + } + return true; + } + void show() { - std::cerr << "R=" << std::endl; + std::cerr << "R(" << dim << ")=" << std::endl; for (size_t i = 0; i < dim; i++) { for (size_t j = 0; j < dim; j++) { std::cerr << (*this)[i * dim + j] << " "; @@ -196,7 +216,6 @@ class QuantizationCodebook : public std::vector { NGTThrowException("NGTQ::QuantizationCodebook::operator=: paddedDimension is unset."); } dimension = qc[0].size(); - std::cerr << "dim=" << dimension << ":" << qc.size() << ":" << paddedDimension << std::endl; PARENT::resize(paddedDimension * qc.size()); for (size_t i = 0; i < qc.size(); i++) { if (qc[i].size() != dimension) { @@ -211,12 +230,17 @@ class QuantizationCodebook : public std::vector { } ~QuantizationCodebook() { delete index; } void setPaddedDimension(size_t pd) { paddedDimension = pd; } + size_t getDimension() { return paddedDimension; } T at(size_t id, size_t dim) { - return *(PARENT::data() + id * paddedDimension + dim); + return *(data(id) + dim); } T *data(size_t id) { return PARENT::data() + id * paddedDimension; } + std::vector getCentroid(size_t idx) { + std::vector centroid(data(idx), data(idx) + paddedDimension); + return centroid; + } size_t size() { return PARENT::size() == 0 ? 0 : PARENT::size() / paddedDimension; } void buildIndex() { if (PARENT::size() == 0) { @@ -247,18 +271,39 @@ class QuantizationCodebook : public std::vector { } index->createIndex(50); } - size_t search(NGT::Object &object) { + size_t search(std::vector &object, float epsilon = 0.1) { + auto obj = index->allocateObject(object); + size_t id = 0; + try { + if (epsilon >= 0.0) { + id = search(*obj, epsilon); + } else { + id = find(*obj); + } + } catch(NGT::Exception &err) { + index->deleteObject(obj); + } + index->deleteObject(obj); + return id; + } + + size_t search(NGT::Object &object, float epsilon = 0.1) { if (index == 0) { - std::cerr << "QuantizeationCodebook: Fatal Error! The index is not available." << std::endl; - abort(); + std::stringstream msg; + msg << "QuantizeationCodebook: Fatal Error! The index is not available."; + NGTThrowException(msg); } NGT::ObjectDistances result; NGT::SearchContainer sc(object); sc.setResults(&result); sc.setSize(10); sc.radius = FLT_MAX; - sc.setEpsilon(0.1); - index->search(sc); + sc.setEpsilon(epsilon); + if (epsilon < std::numeric_limits::max()) { + index->search(sc); + } else { + index->linearSearch(sc); + } if (result.size() == 0) { std::cerr << "QuantizeationCodebook: Fatal Error! Cannot search." << std::endl; abort(); @@ -266,7 +311,7 @@ class QuantizationCodebook : public std::vector { return result[0].id - 1; } } - size_t find(NGT::Object &object, NGT::ObjectSpace::Comparator &comparator) { + size_t find(NGT::Object &object) { size_t noOfCentroids = PARENT::size() / paddedDimension; auto mind = std::numeric_limits::max(); size_t minidx = 0; @@ -287,7 +332,7 @@ class QuantizationCodebook : public std::vector { } } - void serialize(std::ofstream &os) { + void serialize(std::ofstream &os) const { uint32_t v = PARENT::size(); NGT::Serializer::write(os, v); v = dimension; @@ -315,16 +360,19 @@ class QuantizationCodebook : public std::vector { NGT::Index *index; }; +#ifdef NGTQBG_COARSE_BLOB class GraphNodeToInvertedIndexEntries : public std::vector { typedef std::vector PARENT; public: GraphNodeToInvertedIndexEntries() {} ~GraphNodeToInvertedIndexEntries() {} + void serialize(std::ofstream &os) { uint32_t v = PARENT::size(); NGT::Serializer::write(os, v); os.write(reinterpret_cast(PARENT::data()), static_cast(v) * sizeof(uint32_t)); } + void deserialize(std::ifstream &is) { uint32_t v; NGT::Serializer::read(is, v); @@ -332,7 +380,64 @@ class GraphNodeToInvertedIndexEntries : public std::vector { is.read(reinterpret_cast(PARENT::data()), static_cast(v) * sizeof(uint32_t)); } + void setup(std::vector &codebookIndex) { + if (codebookIndex.size() == 0) { + NGTThrowException("Fatal Error! The codebook index is empty. "); + } + if (codebookIndex[0] != 0) { + stringstream msg; + msg << "Fatal Error! The first entry of the codebook index is non-zero. " << codebookIndex[0]; + NGTThrowException(msg); + } + PARENT::resize(codebookIndex.back() + 2); + PARENT::at(0) = 0; + uint32_t prev = 0; + uint32_t gidx = 1; + for (size_t idx = 1; idx < codebookIndex.size(); idx++) { + for (uint32_t sidx = prev; sidx < codebookIndex[idx]; sidx++) { + PARENT::at(gidx++) = idx; + } + prev = codebookIndex[idx]; + } + PARENT::at(gidx) = codebookIndex.size(); + } }; +#endif + +template +class BaseObject { + public: + BaseObject(): objectID(0), subspaceID(0) {} + BaseObject(std::vector obj, uint32_t oid = 0, uint32_t sid = 0): objectID(oid), subspaceID(sid), object(obj) {} + friend std::ostream& operator<<(std::ostream& os, BaseObject &obj) { + os << "object ID=" << obj.objectID << ", subspace ID=" << obj.subspaceID << ", # of subvectors=" << obj.object.size() << std::endl; + for (size_t idx = 0; idx < obj.object.size(); idx++) { + os << (int)obj.object[idx] << " "; + } + os << std::endl; + return os; + } + + uint32_t objectID; + uint32_t subspaceID; + std::vector object; +}; + +template +class BaseObjects : public std::vector> { + public: + friend std::ostream& operator<<(std::ostream& os, std::vector> &objs) { + for (size_t idx = 0; idx < objs.size(); idx++) { + os << idx << ": " << objs[idx]; + } + return os; + } +}; + +typedef BaseObject QuantizedObject; +typedef BaseObjects QuantizedObjectSet; +typedef BaseObject Object; +typedef BaseObjects ObjectSet; template class InvertedIndexObject { @@ -360,20 +465,20 @@ class InvertedIndexObject { uint32_t id; T localID[1]; }; - + template class InvertedIndexEntry : public NGT::DynamicLengthVector> { public: typedef NGT::DynamicLengthVector> PARENT; #ifdef NGTQ_SHARED_INVERTED_INDEX - InvertedIndexEntry(size_t n, NGT::ObjectSpace *os = 0):numOfLocalIDs(n) + InvertedIndexEntry(size_t n, NGT::ObjectSpace *os = 0):numOfSubvectors(n) #ifdef NGTQ_QBG , subspaceID(std::numeric_limits::max()) #endif { PARENT::elementSize = getSizeOfElement(); } - InvertedIndexEntry(size_t n, SharedMemoryAllocator &allocator, NGT::ObjectSpace *os = 0):numOfLocalIDs(n) + InvertedIndexEntry(size_t n, SharedMemoryAllocator &allocator, NGT::ObjectSpace *os = 0):numOfSubvectors(n) #ifdef NGTQ_QBG , subspaceID(std::numeric_limits::max()) #endif @@ -382,28 +487,32 @@ class InvertedIndexEntry : public NGT::DynamicLengthVector(), allocator); - PARENT::back(allocator).clear(numOfLocalIDs); + PARENT::back(allocator).clear(numOfSubvectors); } void pushBack(size_t id, SharedMemoryAllocator &allocator) { pushBack(allocator); PARENT::back(allocator).setID(id); } #else // NGTQ_SHARED_INVERTED_INDEX - InvertedIndexEntry(NGT::ObjectSpace *os = 0):numOfLocalIDs(0) + InvertedIndexEntry(NGT::ObjectSpace *os = 0):numOfSubvectors(0) #ifdef NGTQ_QBG , subspaceID(std::numeric_limits::max()) #endif {} - InvertedIndexEntry(size_t n, NGT::ObjectSpace *os = 0):numOfLocalIDs(n) + InvertedIndexEntry(size_t n, NGT::ObjectSpace *os = 0):numOfSubvectors(n) #ifdef NGTQ_QBG , subspaceID(std::numeric_limits::max()) #endif { PARENT::elementSize = getSizeOfElement(); } + void initialize(size_t n) { + numOfSubvectors = n; + PARENT::elementSize = getSizeOfElement(); + } void pushBack() { PARENT::push_back(InvertedIndexObject()); - PARENT::back().clear(numOfLocalIDs); + PARENT::back().clear(numOfSubvectors); } void pushBack(size_t id) { pushBack(); @@ -413,11 +522,12 @@ class InvertedIndexEntry : public NGT::DynamicLengthVector 0xFFFF) { - std::cerr << "num of local IDs is too large. " << numOfLocalIDs << std::endl; - abort(); + if (numOfSubvectors > 0xFFFF) { + std::stringstream msg; + msg << "# of subvectors is too large. " << numOfSubvectors; + NGTThrowException(msg); } - uint16_t nids = numOfLocalIDs; + uint16_t nids = numOfSubvectors; NGT::Serializer::write(os, nids); #ifdef NGTQ_QBG int32_t ssid = subspaceID; @@ -443,7 +553,7 @@ class InvertedIndexEntry : public NGT::DynamicLengthVector(PARENT::vector), sz * PARENT::elementSize); } + void get(size_t idx, QuantizedObject &qobj) { +#ifdef NGTQ_QBG + qobj.subspaceID = subspaceID; +#endif + qobj.objectID = PARENT::at(idx).id; + qobj.object.clear(); + qobj.object.resize(numOfSubvectors); + for (size_t i = 0; i < numOfSubvectors; i++) { + qobj.object[i] = PARENT::at(idx).localID[i]; + } + } + + void get(std::vector &qobjs) { + qobjs.clear(); + qobjs.resize(PARENT::size()); +#pragma omp parallel for + for (size_t idx = 0; idx < PARENT::size(); idx++) { + get(idx, qobjs[idx]); + } + } + + void set(size_t idx, QuantizedObject &qobj) { + if (PARENT::size() <= idx) { + std::stringstream msg; + msg << "The index is out of range. " << idx << ":" << PARENT::size(); + NGTThrowException(msg); + } + if (numOfSubvectors != qobj.object.size()) { + std::stringstream msg; + msg << "# of subectors are inconsitent. " << numOfSubvectors << ":" << qobj.object.size(); + NGTThrowException(msg); + } + PARENT::at(idx).id = qobj.objectID; + for (size_t i = 0; i < numOfSubvectors; i++) { + PARENT::at(idx).localID[i] = qobj.object[i]; + } + } + + void set(QuantizedObjectSet &qobjs) { + if (qobjs.empty()) { + std::stringstream msg; + msg << "Quantized objects is empty()"; + NGTThrowException(msg); + } + PARENT::clear(); + initialize(qobjs[0].object.size()); + PARENT::reserve(qobjs.size()); + PARENT::resize(qobjs.size()); +#ifdef NGTQ_QBG + subspaceID = qobjs[0].subspaceID; +#endif +#pragma omp parallel for + for (size_t idx = 0; idx < PARENT::size(); idx++) { +#ifdef NGTQ_QBG + if (subspaceID != qobjs[idx].subspaceID) { + std::stringstream msg; + msg << "Subspace IDs are inconsistent. " << subspaceID << ":" << qobjs[idx].subspaceID; + NGTThrowException(msg); + } +#endif + set(idx, qobjs[idx]); + } + } + #endif size_t getSizeOfElement() { InvertedIndexObject dummy; - size_t dsize = ((numOfLocalIDs * sizeof(T) - 1) / 4 + 1) * 4; + size_t dsize = ((numOfSubvectors * sizeof(T) - 1) / 4 + 1) * 4; return InvertedIndexObject::headerSize() + dsize; } - uint32_t numOfLocalIDs; + friend std::ostream& operator<<(std::ostream& os, InvertedIndexEntry &iie) { + os << "subspace ID=" << iie.subspaceID << std::endl; + os << "# of subvectors=" << iie.numOfSubvectors << std::endl; + for (size_t i = 0; i < iie.size(); i++) { + os << iie[i].id << ":"; + for (size_t j = 0; j < iie.numOfSubvectors; j++) { + os << (int)iie[i].localID[j] << " "; + } + os << std::endl; + } + return os; + } + + uint32_t numOfSubvectors; #ifdef NGTQ_QBG uint32_t subspaceID; #endif @@ -471,9 +658,9 @@ class LocalDatam { public: LocalDatam(){}; #ifdef NGTQ_QBG - LocalDatam(size_t iii, size_t iil, uint32_t ssID = 0) : iiIdx(iii), iiLocalIdx(iil), subspaceID(ssID) {} + LocalDatam(size_t iii, size_t iil, uint32_t ssID = 0) : iiIdx(iii), iiLocalIdx(iil), subspaceID(ssID) {} #else - LocalDatam(size_t iii, size_t iil) : iiIdx(iii), iiLocalIdx(iil) {} + LocalDatam(size_t iii, size_t iil) : iiIdx(iii), iiLocalIdx(iil) {} #endif size_t iiIdx; size_t iiLocalIdx; @@ -547,6 +734,9 @@ class SerializableObject : public NGT::Object { localCodebookState = false; // not completed localClusteringSampleCoefficient = 10; quantizerType = QuantizerTypeNone; +#ifdef NGTQ_OBJECT_IN_MEMORY + objectListOnMemory = false; +#endif #ifdef NGT_SHARED_MEMORY_ALLOCATOR invertedIndexSharedMemorySize = 512; // MB #endif @@ -576,6 +766,9 @@ class SerializableObject : public NGT::Object { prop.set("LocalCodebookState", (long)localCodebookState); prop.set("LocalSampleCoefficient", (long)localClusteringSampleCoefficient); prop.set("QuantizerType", (long)quantizerType); +#ifdef NGTQ_OBJECT_IN_MEMORY + prop.set("ObjectListOnMemory", (long)objectListOnMemory); +#endif #ifdef NGT_SHARED_MEMORY_ALLOCATOR prop.set("InvertedIndexSharedMemorySize", (long)invertedIndexSharedMemorySize); #endif @@ -631,6 +824,9 @@ class SerializableObject : public NGT::Object { localClusteringSampleCoefficient = prop.getl("LocalSampleCoefficient", localClusteringSampleCoefficient); setupLocalIDByteSize(); quantizerType = (QuantizerType)prop.getl("QuantizerType", quantizerType); +#ifdef NGTQ_OBJECT_IN_MEMORY + objectListOnMemory = prop.getl("ObjectListOnMemory", objectListOnMemory); +#endif #ifdef NGT_SHARED_MEMORY_ALLOCATOR invertedIndexSharedMemorySize = prop.getl("InvertedIndexSharedMemorySize", invertedIndexSharedMemorySize); @@ -699,6 +895,9 @@ class SerializableObject : public NGT::Object { bool localCodebookState; size_t localClusteringSampleCoefficient; QuantizerType quantizerType; +#ifdef NGTQ_OBJECT_IN_MEMORY + bool objectListOnMemory; +#endif #ifdef NGT_SHARED_MEMORY_ALLOCATOR size_t invertedIndexSharedMemorySize; #endif @@ -1047,11 +1246,22 @@ class QuantizedObjectDistance { } #else inline void createDistanceLookup(NGT::Object &object, size_t objectID, DistanceLookupTable &distanceLUT) { + void *objectPtr = &((NGT::Object&)object)[0]; + createDistanceLookup(objectPtr, objectID, distanceLUT); + } + inline void createDistanceLookup(void *objectPtr, size_t objectID, DistanceLookupTable &distanceLUT) { assert(globalCodebookIndex != 0); +#ifdef NGTQ_QBG + void *globalCentroid = quantizationCodebook->data(objectID); + size_t sizeOfObject = dimension * sizeOfType; +#else NGT::Object &gcentroid = (NGT::Object &)*globalCodebookIndex->getObjectSpace().getRepository().get(objectID); + void *globalCentroid = &gcentroid[0]; size_t sizeOfObject = globalCodebookIndex->getObjectSpace().getByteSizeOfObject(); +#endif - createFloatL2DistanceLookup(&((NGT::Object&)object)[0], sizeOfObject, &gcentroid[0], distanceLUT.localDistanceLookup); + + createFloatL2DistanceLookup(objectPtr, sizeOfObject, globalCentroid, distanceLUT.localDistanceLookup); } #endif @@ -1085,11 +1295,15 @@ class QuantizedObjectDistance { inline void createFloatL2DistanceLookup(void *object, size_t sizeOfObject, void *globalCentroid, DistanceLookupTableUint8 &distanceLUT) { - assert(localCodebookCentroidNoSIMD == 16); + if (localCodebookCentroidNoSIMD != 16) { + std::stringstream msg; + msg << "Invalid number of the local centroids for SIMD. " << localCodebookCentroidNoSIMD; + NGTThrowException(msg); + } size_t dim = sizeOfObject / sizeof(float); size_t localDim = dim / localDivisionNo; #if defined(NGT_AVX512) - __m512 flut[dim]; + __m512 flut[localCodebookNo]; __m512 *flutptr = &flut[0]; #ifdef NGTQ_TOTAL_SCALE_OFFSET_COMPRESSION __m512 mmin = _mm512_set1_ps(std::numeric_limits::max()); @@ -1176,7 +1390,7 @@ class QuantizedObjectDistance { #endif #elif defined(NGT_AVX2) - __m256 flut[dim * 2]; + __m256 flut[localCodebookNo * 2]; __m256 *flutptr = &flut[0]; __m256 mmin = _mm256_set1_ps(std::numeric_limits::max()); __m256 mmax = _mm256_set1_ps(-std::numeric_limits::max()); @@ -1233,8 +1447,10 @@ class QuantizedObjectDistance { float offset = min; float scale = (max - min) / 255.0; +#ifndef NGTQ_TOTAL_SCALE_OFFSET_COMPRESSION + std::cerr << "Individual scale offset compression is not implemented." << std::endl; +#endif distanceLUT.totalOffset = offset * static_cast(localCodebookNo); - //uint8_t blut[dim / 2 * localCodebookCentroidNoSIMD]; auto *blutptr = distanceLUT.localDistanceLookup; flutptr = &flut[0]; for (size_t li = 0; li < localCodebookNo; li++) { @@ -1264,41 +1480,46 @@ class QuantizedObjectDistance { distanceLUT.scales[li] = scale; } #else - float flut[dim / 2 * localCodebookCentroidNoSIMD]; + float flut[localCodebookNo * localCodebookCentroidNoSIMD]; auto *flutptr = &flut[0]; + memset(flutptr, 0, sizeof(float) * localCodebookNo * localCodebookCentroidNoSIMD); auto min = std::numeric_limits::max(); auto max = -std::numeric_limits::max(); auto *lcptr = static_cast(&localCentroidsForSIMD[0]); auto *optr = static_cast(object); + auto *optrend = optr + dim; #ifndef NGTQG_ZERO_GLOBAL auto *gcptr = static_cast(globalCentroid); #endif - float sub[localCodebookCentroidNoSIMD]; - for (size_t d = 0; d < dim; d++) { + while (optr < optrend) { + auto *optrllast = optr + localDim; + while (optr < optrllast) { #ifdef NGTQG_ZERO_GLOBAL - float rsv = optr[d]; + float rsv = *optr++; #else - float rsv = optr[d] - *gcptr++; + float rsv = *optr++ - *gcptr++; #endif - for (size_t k = 0; k < localCodebookCentroidNoSIMD; k++) { - auto v = rsv - *lcptr++; - v *= v; - if (d % 2 == 0) { - sub[k] = v; - } else { - v += sub[k]; - if (v < min) { - min = v; - } else if (v > max) { - max = v; - } - *flutptr++ = v; + for (size_t ci = 0; ci < localCodebookCentroidNoSIMD; ci++) { + auto v = rsv - *lcptr++; + v *= v; + flutptr[ci] += v; + } + } + for (size_t ci = 0; ci < localCodebookCentroidNoSIMD; ci++) { + auto v = flutptr[ci]; + if (v < min) { + min = v; + } else if (v > max) { + max = v; } } + flutptr += localCodebookCentroidNoSIMD; } float offset = min; float scale = (max - min) / 255.0; - +#ifndef NGTQ_TOTAL_SCALE_OFFSET_COMPRESSION + std::cerr << "Individual scale offset compression is not implemented." << std::endl; +#endif distanceLUT.totalOffset = offset * static_cast(localCodebookNo); auto *blutptr = distanceLUT.localDistanceLookup; flutptr = &flut[0]; @@ -1312,7 +1533,6 @@ class QuantizedObjectDistance { distanceLUT.offsets[li] = offset; distanceLUT.scales[li] = scale; } - #endif @@ -1623,7 +1843,7 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { data = _mm256_min_ps(data, (__m256)_mm256_permute4x64_epi64((__m256i)data, _MM_SHUFFLE(3, 2, 3, 2))); data = _mm256_min_ps(data, (__m256)_mm256_srli_si256((__m256i)data, 8)); data = _mm256_min_ps(data, (__m256)_mm256_srli_si256((__m256i)data, 4)); - //std::cerr << "ret=" << data[0] << std::endl; + return data[0]; } #endif @@ -1925,6 +2145,9 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { inline void operator()(void *inv, float *distances, size_t size, DistanceLookupTableUint8 &distanceLUT) { #endif uint8_t *localID = static_cast(inv); +#ifdef NGTQBG_MIN + float min = std::numeric_limits::max(); +#endif size_t numOfAlignedSubvectors = ((localDivisionNo - 1) / NGTQ_BATCH_SIZE + 1) * NGTQ_BATCH_SIZE; size_t alignedSize = ((size - 1) / 2 + 1) * 2; uint32_t d[NGTQ_SIMD_BLOCK_SIZE]; @@ -1949,9 +2172,17 @@ class QuantizedObjectDistanceFloat : public QuantizedObjectDistance { } for (size_t i = 0; i < NGTQ_SIMD_BLOCK_SIZE; i++) { distances[didx + i] = sqrt(static_cast(d[i]) * distanceLUT.scales[0] + distanceLUT.totalOffset); +#ifdef NGTQBG_MIN + if (min > distances[didx + i]) { + min = distances[didx + i]; + } +#endif } didx += NGTQ_SIMD_BLOCK_SIZE; } +#ifdef NGTQBG_MIN + return min; +#endif } #endif @@ -2036,9 +2267,14 @@ class Quantizer { #else NGT::Property &localPropertySet) = 0; #endif -#ifndef NGTQ_QBG - virtual void insert(vector > &objects) = 0; - virtual void insert(vector &object, vector > &objects, size_t id) = 0; +#ifdef NGTQ_QBG + virtual void encode(uint32_t subspaceID, ObjectSet &objects, QuantizedObjectSet &qobjs) = 0; + virtual void encode(uint32_t subspaceID, Object &object, QuantizedObject &qobj) = 0; + virtual void decode(QuantizedObjectSet &qobjs, ObjectSet &objects) = 0; + virtual void decode(QuantizedObject &qobj, Object &object) = 0; +#else + virtual void insert(vector> &objects) = 0; + virtual void insert(vector &object, vector> &objects, size_t id) = 0; #endif virtual void insertIntoObjectRepository(vector &object, size_t id) = 0; #ifdef NGTQ_QBG @@ -2051,7 +2287,7 @@ class Quantizer { virtual void rebuildIndex() = 0; #endif virtual void save() = 0; - virtual void saveRotation(const std::vector &rotation) = 0; + virtual void loadQuantizationCodebookAndRotation(const std::vector> &quantizationCodebook, const std::vector &rotation) = 0; virtual void open(const string &index, NGT::Property &globalProperty, bool readOnly) = 0; virtual void open(const string &index, bool readOnly) = 0; virtual void close() = 0; @@ -2115,6 +2351,8 @@ class Quantizer { void setDimension(size_t s) { property.dimension = s; } void setDistanceType(DistanceType t) { property.distanceType = t; } + size_t getNumOfLocalClusters() { return property.localCentroidLimit; } + NGT::Index &getLocalCodebook(size_t idx) { return localCodebookIndexes[idx]; } size_t getLocalCodebookSize(size_t size) { return localCodebookIndexes[size].getObjectRepositorySize(); } @@ -2127,9 +2365,19 @@ class Quantizer { virtual QuantizedObjectDistance &getQuantizedObjectDistance() = 0; +#ifdef NGTQBG_COARSE_BLOB + virtual GraphNodeToInvertedIndexEntries &getGraphNodeToInvertedIndexEntries() = 0; +#endif virtual size_t getInvertedIndexSize() = 0; + static const std::string getInvertedIndexFile() { return "ivt"; } + static const std::string getGlobalFile() { return "global"; } + static const std::string getLocalPrefix() { return "local-"; } + static const std::string getRotationFile() { return "qr"; } + static const std::string getGlobalToInvertedIndexFile() { return "g2i"; } + ObjectList objectList; + string rootDirectory; Property property; @@ -2148,12 +2396,15 @@ class Quantizer { std::vector objectToBlobIndex; Rotation rotation; +#ifdef NGTQ_OBJECT_IN_MEMORY + NGT::Repository objectListOnMemory; +#endif }; class QuantizedObjectProcessingStream { public: - QuantizedObjectProcessingStream(Quantizer &quantizer, size_t numOfObjects): stream(0) { - initialize(quantizer.divisionNo); + QuantizedObjectProcessingStream(size_t divisionNo, size_t numOfObjects): stream(0) { + initialize(divisionNo); setStreamSize(numOfObjects); stream = new uint8_t[streamSize](); } @@ -2252,10 +2503,10 @@ class GenerateResidualObject { virtual void operator()(NGT::Object &object, size_t centroidID, float *subspaceObject) = 0; #endif virtual void operator()(NGT::Object &object, size_t centroidID, - vector > > &localObjs) = 0; + vector>> &localObjs) = 0; #else virtual void operator()(size_t objectID, size_t centroidID, - vector > > &localObjs) = 0; + vector>> &localObjs) = 0; #endif void set(NGT::Index &gc, NGT::Index lc[], size_t dn, size_t lcn, Quantizer::ObjectList *ol, QuantizationCodebook *qc) { @@ -2290,10 +2541,10 @@ class GenerateResidualObjectUint8 : public GenerateResidualObject { void operator()(NGT::Object &object, size_t centroidID, float *subspaceObject) { abort(); } #endif void operator()(NGT::Object &xobject, size_t centroidID, - vector > > &localObjs) { abort(); } + vector>> &localObjs) { abort(); } #else void operator()(size_t objectID, size_t centroidID, - vector > > &localObjs) { + vector>> &localObjs) { NGT::PersistentObject &globalCentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(centroidID); NGT::Object object(&globalCodebookIndex->getObjectSpace()); objectList->get(objectID, object, &globalCodebookIndex->getObjectSpace()); @@ -2328,6 +2579,11 @@ class GenerateResidualObjectFloat : public GenerateResidualObject { void operator()(NGT::Object &object, size_t centroidID, float *subspaceObject) { #endif size_t dimension = globalCodebookIndex->getObjectSpace().getPaddedDimension(); + if (object.size() != dimension) { + std::stringstream msg; + msg << "The dimensionalities are inconsitent." << object.size() << ":" << dimension; + NGTThrowException(msg); + } #ifdef NGTQ_VECTOR_OBJECT auto *vector = object.data(); #else @@ -2377,7 +2633,7 @@ class GenerateResidualObjectFloat : public GenerateResidualObject { class GenerateResidualObjectFloat : public GenerateResidualObject { public: void operator()(size_t objectID, size_t centroidID, - vector > > &localObjs) { + vector>> &localObjs) { NGT::PersistentObject &globalCentroid = *globalCodebookIndex->getObjectSpace().getRepository().get(centroidID); NGT::Object object(&globalCodebookIndex->getObjectSpace()); objectList->get(objectID, object, &globalCodebookIndex->getObjectSpace()); @@ -2416,7 +2672,7 @@ class QuantizerInstance : public Quantizer { quantizedObjectDistance = 0; generateResidualObject = 0; localCodebooks = 0; - silence = true; + verbose = false; } virtual ~QuantizerInstance() { close(); } @@ -2433,7 +2689,7 @@ class QuantizerInstance : public Quantizer { { rootDirectory = index; NGT::Index::mkdir(rootDirectory); - string global = rootDirectory + "/global"; + string global = rootDirectory + "/" + getGlobalFile(); NGT::Index::mkdir(global); #ifdef NGT_SHARED_MEMORY_ALLOCATOR @@ -2450,7 +2706,7 @@ class QuantizerInstance : public Quantizer { size_t localCodebookNo = property.getLocalCodebookNo(); for (size_t i = 0; i < localCodebookNo; ++i) { stringstream local; - local << rootDirectory << "/local-" << i; + local << rootDirectory << "/" + getLocalPrefix() << i; NGT::Index::mkdir(local.str()); NGT::GraphAndTreeIndex localCodebook(local.str(), localProperty); localCodebook.saveIndex(local.str()); @@ -2460,16 +2716,16 @@ class QuantizerInstance : public Quantizer { size_t localCodebookNo = property.getLocalCodebookNo(); for (size_t i = 0; i < localCodebookNo; ++i) { stringstream local; - local << rootDirectory << "/local-" << i; + local << rootDirectory << "/" + getLocalPrefix() << i; NGT::Index::mkdir(local.str()); localCodebook.saveIndex(local.str()); } localCodebook.close(); #endif #ifdef NGTQ_SHARED_INVERTED_INDEX - invertedIndex.open(index + "/ivt", property.invertedIndexSharedMemorySize); + invertedIndex.open(index + "/" + getInvertedIndexFile(), property.invertedIndexSharedMemorySize); #else - ofstream of(rootDirectory + "/ivt"); + ofstream of(rootDirectory + "/" + getInvertedIndexFile()); invertedIndex.serialize(of); #endif string fname = rootDirectory + "/obj"; @@ -2508,21 +2764,46 @@ class QuantizerInstance : public Quantizer { void saveRotation(const std::vector &rotation) { Rotation r; r = rotation; - ofstream ofs(rootDirectory + "/qr"); + ofstream ofs(rootDirectory + "/" + getRotationFile()); r.serialize(ofs); } + void saveQuantizationCodebook(const QuantizationCodebook &qCodebook) { +#ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS + ofstream ofs(rootDirectory + "/rqcb"); +#else + ofstream ofs(rootDirectory + "/qcb"); +#endif + qCodebook.serialize(ofs); + } + + void loadQuantizationCodebookAndRotation(const std::vector> &qCodebook, const std::vector &rotation) { + QuantizationCodebook qc; + qc.setPaddedDimension(globalCodebookIndex.getObjectSpace().getPaddedDimension()); + qc = qCodebook; + Rotation r; + r = rotation; +#ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS + if (rotation.empty()) { + NGTThrowException("The rotation is empty."); + } + qc.rotate(r); +#endif + saveRotation(r); + saveQuantizationCodebook(qc); + } + void open(const string &index, NGT::Property &globalProperty, bool readOnly) { open(index, readOnly); globalCodebookIndex.setProperty(globalProperty); } void open(const string &index, bool readOnly) { - NGT::StdOstreamRedirector redirector(silence); + NGT::StdOstreamRedirector redirector(!verbose); redirector.begin(); rootDirectory = index; property.load(rootDirectory); - string globalIndex = index + "/global"; + string globalIndex = index + "/" + getGlobalFile(); globalCodebookIndex.open(globalIndex, readOnly); if ((globalCodebookIndex.getObjectRepositorySize() == 0) && readOnly) { std::cerr << "open: Warning. global codebook is empty." << std::endl; @@ -2532,9 +2813,10 @@ class QuantizerInstance : public Quantizer { localCodebookIndexes.resize(localCodebookNo); for (size_t i = 0; i < localCodebookNo; ++i) { stringstream localIndex; - localIndex << index << "/local-" << i; + localIndex << index << "/" + getLocalPrefix() << i; localCodebookIndexes[i].open(localIndex.str()); } + constructLocalCodebooks(); #ifdef NGTQ_QBG if (!readOnly) { #else @@ -2542,27 +2824,48 @@ class QuantizerInstance : public Quantizer { #endif #ifdef NGTQ_SHARED_INVERTED_INDEX - invertedIndex.open(index + "/ivt", 0); + invertedIndex.open(index + "/" + getInvertedIndexFile(), 0); #else - ifstream ifs(index + "/ivt"); + ifstream ifs(index + "/" + getInvertedIndexFile()); if (!ifs) { - cerr << "Cannot open " << index + "/ivt" << "." << endl; + cerr << "Cannot open " << index + "/" + getInvertedIndexFile() << "." << endl; return; } invertedIndex.deserialize(ifs); #endif } +#ifdef NGTQBG_COARSE_BLOB + { + ifstream ifs(index + "/" + getGlobalToInvertedIndexFile()); + if (!ifs) { + std::cerr << getGlobalToInvertedIndexFile() << " doesn't exist." << std::endl; + } else { + graphNodeToInvertedIndexEntries.deserialize(ifs); + } + } +#endif #ifdef NGTQ_QBG if (!objectList.open(index + "/obj", property.genuineDataType, property.distanceType, property.dimension)) { #else if (!objectList.open(index + "/obj", static_cast(property.dataType), property.distanceType, property.dimension)) { -#endif +#endif stringstream msg; msg << "NGTQ::Quantizer::open: cannot open the object file. " << index + "/obj" << std::endl; std::cerr << "Ignore. " << msg.str() << std::endl; } #ifdef MULTIPLE_OBJECT_LISTS objectList.openMultipleStreams(omp_get_max_threads()); +#endif +#ifdef NGTQ_OBJECT_IN_MEMORY + if (property.objectListOnMemory) { + objectListOnMemory.resize(objectList.size()); + for (size_t id = 1; id < objectList.size(); id++) { + std::vector object; + objectList.get(id, object, &globalCodebookIndex.getObjectSpace()); + NGT::Object *ngtObject = globalCodebookIndex.allocateObject(object); + objectListOnMemory.put(id, ngtObject); + } + } #endif NGT::Property globalProperty; globalCodebookIndex.getProperty(globalProperty); @@ -2613,22 +2916,25 @@ class QuantizerInstance : public Quantizer { abort(); } assert(quantizedObjectDistance != 0); - quantizedObjectDistance->set(&globalCodebookIndex, localCodebookIndexes.data(), &quantizationCodebook, - property.localDivisionNo, property.getLocalCodebookNo(), sizeoftype, - property.dimension, &rotation); - generateResidualObject->set(globalCodebookIndex, localCodebookIndexes.data(), property.localDivisionNo, property.getLocalCodebookNo(), &objectList, &quantizationCodebook); - localIDByteSize = property.localIDByteSize; - objectType = globalProperty.objectType; - divisionNo = property.localDivisionNo; #ifdef NGTQ_QBG { - std::string streamName(rootDirectory + "/qr"); + std::string streamName(rootDirectory + "/" + getRotationFile()); ifstream ifs(streamName); if (ifs) { - std::cerr << "loading rotation..." << std::endl; + std::cerr << "loading the rotation..." << std::endl; rotation.deserialize(ifs); } } +#endif + Rotation *r = (readOnly && rotation.isIdentity()) ? 0 : &rotation; + quantizedObjectDistance->set(&globalCodebookIndex, localCodebookIndexes.data(), &quantizationCodebook, + property.localDivisionNo, property.getLocalCodebookNo(), sizeoftype, + property.dimension, r); + generateResidualObject->set(globalCodebookIndex, localCodebookIndexes.data(), property.localDivisionNo, property.getLocalCodebookNo(), &objectList, &quantizationCodebook); + localIDByteSize = property.localIDByteSize; + objectType = globalProperty.objectType; + divisionNo = property.localDivisionNo; +#ifdef NGTQ_QBG { #ifdef NGTQG_ROTATION std::string rqcbName(rootDirectory + "/rqcb"); @@ -2638,13 +2944,10 @@ class QuantizerInstance : public Quantizer { ifstream ifs(qcbName); if (!ifs) { } else { - std::cerr << "loading the global codebooks..." << std::endl; quantizationCodebook.deserialize(ifs, readOnly); - std::cerr << "rotating the global codebooks... " << rotation.size() << std::endl; quantizationCodebook.rotate(rotation); } } else { - std::cerr << "loading the rotated global codebooks..." << std::endl; quantizationCodebook.deserialize(irfs, readOnly); } #else @@ -2659,12 +2962,12 @@ class QuantizerInstance : public Quantizer { void save() { #ifndef NGT_SHARED_MEMORY_ALLOCATOR - string global = rootDirectory + "/global"; + string global = rootDirectory + "/" + getGlobalFile(); globalCodebookIndex.saveIndex(global); size_t localCodebookNo = property.getLocalCodebookNo(); for (size_t i = 0; i < localCodebookNo; ++i) { stringstream local; - local << rootDirectory << "/local-" << i; + local << rootDirectory << "/" + getLocalPrefix() << i; try { NGT::Index::mkdir(local.str()); } catch (...) {} @@ -2673,19 +2976,19 @@ class QuantizerInstance : public Quantizer { #endif // NGT_SHARED_MEMORY_ALLOCATOR #ifndef NGTQ_SHARED_INVERTED_INDEX { - ofstream of(rootDirectory + "/ivt"); + ofstream of(rootDirectory + "/" + getInvertedIndexFile()); invertedIndex.serialize(of); } #endif -#ifdef NGTQ_QBG +#ifdef NGTQBG_COARSE_BLOB { -#ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS - ofstream ofs(rootDirectory + "/rqcb"); -#else - ofstream ofs(rootDirectory + "/qcb"); -#endif - quantizationCodebook.serialize(ofs); + ofstream of(rootDirectory + "/" + getGlobalToInvertedIndexFile()); + graphNodeToInvertedIndexEntries.serialize(of); } +#endif +#ifdef NGTQ_QBG + saveQuantizationCodebook(quantizationCodebook); + saveRotation(rotation); #endif property.save(rootDirectory); } @@ -2699,6 +3002,11 @@ class QuantizerInstance : public Quantizer { void close() { objectList.close(); +#ifdef NGTQ_OBJECT_IN_MEMORY + for (size_t i = 1; i < objectListOnMemory.size(); i++) { + globalCodebookIndex.deleteObject(objectListOnMemory.get(i)); + } +#endif closeCodebooks(); if (quantizedObjectDistance != 0) { delete quantizedObjectDistance; @@ -2777,7 +3085,7 @@ class QuantizerInstance : public Quantizer { void createIndex(NGT::GraphAndTreeIndex &codebook, size_t centroidLimit, - const vector > &objects, + const vector> &objects, vector &ids, float &range) { @@ -2796,7 +3104,7 @@ class QuantizerInstance : public Quantizer { end += s; } vector idstmp; - vector > objtmp; + vector> objtmp; std::copy(start, end, std::back_inserter(objtmp)); codebook.createIndex(objtmp, idstmp, range, property.threadSize); assert(idstmp.size() == objtmp.size()); @@ -2805,7 +3113,7 @@ class QuantizerInstance : public Quantizer { } while (start != objects.end() && centroidLimit - getNumberOfObjects(codebook) > 0); range = FLT_MAX; vector idstmp; - vector > objtmp; + vector> objtmp; std::copy(start, objects.end(), std::back_inserter(objtmp)); codebook.createIndex(objtmp, idstmp, range, property.threadSize); std::copy(idstmp.begin(), idstmp.end(), std::back_inserter(ids)); @@ -2882,7 +3190,7 @@ class QuantizerInstance : public Quantizer { } } - void setSingleLocalCodeToInvertedIndexEntry(vector &lcodebook, vector &localData, vector > > &localObjs) { + void setSingleLocalCodeToInvertedIndexEntry(vector &lcodebook, vector &localData, vector>> &localObjs) { float lr = property.localRange; size_t localCentroidLimit = property.localCentroidLimit; if (property.localCodebookState) { @@ -2913,7 +3221,7 @@ class QuantizerInstance : public Quantizer { } #ifndef NGTQ_QBG - bool setMultipleLocalCodeToInvertedIndexEntry(vector &lcodebook, vector &localData, vector > > &localObjs) { + bool setMultipleLocalCodeToInvertedIndexEntry(vector &lcodebook, vector &localData, vector>> &localObjs) { size_t localCodebookNo = property.getLocalCodebookNo(); bool localCodebookFull = true; #pragma omp parallel for @@ -2960,8 +3268,8 @@ class QuantizerInstance : public Quantizer { #ifdef NGTQ_QBG bool setMultipleLocalCodeToInvertedIndexEntry(vector &lcodebook, - vector &localData, - float *subspaceObjects) { + vector &localData, + float *subspaceObjects) { size_t paddedDimension = globalCodebookIndex.getObjectSpace().getPaddedDimension(); size_t localCodebookNo = property.getLocalCodebookNo(); bool localCodebookFull = true; @@ -3013,7 +3321,14 @@ class QuantizerInstance : public Quantizer { #endif void constructLocalCodebooks() { - delete localCodebooks; + delete[] localCodebooks; + localCodebooks = 0; + if (localCodebookIndexes.size() == 0) { + return; + } + if (localCodebookIndexes[0].getObjectSpace().getSize() == 0) { + return; + } size_t localCodebookNo = property.getLocalCodebookNo(); size_t codebookSize = localCodebookIndexes[0].getObjectSpace().getSize() - 1; size_t paddedDimension = globalCodebookIndex.getObjectSpace().getPaddedDimension(); @@ -3022,8 +3337,9 @@ class QuantizerInstance : public Quantizer { size_t oft = 0; for (size_t li = 0; li < localCodebookNo; li++) { if (localCodebookIndexes[li].getObjectSpace().getSize() - 1 != codebookSize) { - std::cerr << "Fatal Error!" << std::endl; - abort(); + std::stringstream msg; + msg << "Fatal Error! # of the local centroids is invalid. " << localCodebookIndexes[li].getObjectSpace().getSize() - 1 << ":" << codebookSize; + NGTThrowException(msg); } for (size_t cid = 1; cid <= codebookSize; cid++) { std::vector v; @@ -3035,7 +3351,7 @@ class QuantizerInstance : public Quantizer { oft += localDimension; } if (oft != globalCodebookIndex.getObjectSpace().getDimension()) { - std::cerr << "somethig wrong " << oft << ":" << globalCodebookIndex.getObjectSpace().getDimension() << std::endl; + std::cerr << "Fatal error. somethig wrong " << oft << ":" << globalCodebookIndex.getObjectSpace().getDimension() << std::endl; abort(); } } @@ -3053,8 +3369,9 @@ class QuantizerInstance : public Quantizer { size_t localCodebookNo = property.getLocalCodebookNo(); size_t codebookSize = property.localCentroidLimit; if (property.dimension % property.localDivisionNo != 0) { - std::cerr << "setMultipleLocalCodeToInvertedIndexEntry!!!" << std::endl; - abort(); + std::stringstream msg; + msg << "Invalid dimension or # of subspaces. " << property.dimension << ":" << property.localDivisionNo; + NGTThrowException(msg); } size_t localDimension = property.dimension / property.localDivisionNo; std::unique_ptr distance(new float[localData.size() * codebookSize * localCodebookNo]()); @@ -3169,7 +3486,7 @@ class QuantizerInstance : public Quantizer { #endif #ifndef NGTQ_QBG - void insert(vector > &objects) { + void insert(vector> &objects) { std::cerr << "insert() is not implemented." << std::endl; abort(); } @@ -3268,20 +3585,21 @@ class QuantizerInstance : public Quantizer { { ids.clear(); ids.resize(objects.size()); -#ifdef GET_BLOG_EVAL +#ifdef GET_BLOB_EVAL size_t identicalObjectCount = 0; #endif for (size_t idx = 0; idx < objects.size(); idx++) { if (objects[idx].second - 1 >= objectToBlobIndex.size()) { - std::cerr << "Quantizer::insert: Fatal Error! Object ID is invalid. " - << idx << ":" << objects[idx].second - 1 << ":" << objectToBlobIndex.size() - << ":" << objects.size()<< std::endl; - abort(); + std::stringstream msg; + msg << "Quantizer::insert: Fatal Error! Object ID is invalid. " + << idx << ":" << objects[idx].second - 1 << ":" << objectToBlobIndex.size() + << ":" << objects.size(); + NGTThrowException(msg); } ids[idx].id = objectToBlobIndex[objects[idx].second - 1] + 1; ids[idx].distance = 0.0; ids[idx].identical = true; -#ifdef GET_BLOG_EVAL +#ifdef GET_BLOB_EVAL { NGT::ObjectDistances result; NGT::SearchContainer sc(*objects[idx].first); @@ -3298,7 +3616,7 @@ class QuantizerInstance : public Quantizer { } #endif } -#ifdef GET_BLOG_EVAL +#ifdef GET_BLOB_EVAL std::cerr << identicalObjectCount << "/" << objects.size() << std::endl; #endif return; @@ -3321,26 +3639,169 @@ class QuantizerInstance : public Quantizer { #else NGT::Index *index = new NGT::Index(property); #endif + if (globalCodebookIndex.getObjectRepositorySize() > invertedIndex.size()) { + std::cerr << "warning: the inverted index size is too small. Cannot build a global codebook with qid. " << invertedIndex.size() << ":" << globalCodebookIndex.getObjectRepositorySize() << std::endl; + return index; + } for (size_t id = 1; id < globalCodebookIndex.getObjectRepositorySize(); id++) { std::vector object; - if (id % 10000 == 0) { - std::cerr << "# of processed objects=" << id << std::endl; + if (id % 100000 == 0) { + std::cerr << "# of processed objects=" << id << ", vm size=" + << NGT::Common::getProcessVmSizeStr() << "/" + << NGT::Common::getProcessVmPeakStr() << std::endl; + + } + try { + globalCodebookIndex.getObjectSpace().getObject(id, object); +#ifdef NGTQBG_COARSE_BLOB + auto ivid = graphNodeToInvertedIndexEntries[id]; + object.push_back(invertedIndex.at(ivid)->subspaceID * QID_WEIGHT); +#else + object.push_back(invertedIndex.at(id)->subspaceID * QID_WEIGHT); +#endif + index->append(object); + } catch(NGT::Exception &err) { + stringstream msg; + msg << "buildGlobalCodebookWithQIDIndex: fatal inner error. " << err.what() << " : ID=" << id << " size=" << invertedIndex.size(); + NGTThrowException(msg); + NGTThrowException(msg); } - globalCodebookIndex.getObjectSpace().getObject(id, object); - object.push_back(invertedIndex.at(id)->subspaceID * QID_WEIGHT); - index->append(object); } + std::cerr << "creating the index..." << std::endl; index->createIndex(50); return index; #endif } #endif +#ifdef NGTQ_QBG + void decode(QuantizedObject &qobj, Object &object) { + auto globalCentroid = quantizationCodebook.data(qobj.subspaceID); + decode(qobj, globalCentroid, object.object); + } + + void decode(QuantizedObjectSet &qobjs, ObjectSet &objects) { + if (qobjs.size() == 0) { + return; + } + objects.resize(qobjs.size()); +#pragma omp parallel for + for (size_t i = 0; i < qobjs.size(); i++) { + decode(qobjs[i], objects[i]); + } + } + + void decode(InvertedIndexObject &quantizedObject, float *globalCentroid, std::vector &object) { + size_t dim = globalCodebookIndex.getObjectSpace().getPaddedDimension(); + size_t nOfSubvectors = property.getLocalCodebookNo(); + size_t dimOfSubvector = dim / nOfSubvectors; + auto *lcptr = static_cast(&localCodebooks[0]); + object.resize(dim, 0.0); + auto *optr = object.data(); + auto *gcptr = static_cast(globalCentroid); + for (size_t li = 0; li < nOfSubvectors; li++) { + auto lidx = (quantizedObject.localID[li] - 1) * dim; + for (size_t ld = 0; ld < dimOfSubvector; ld++) { + *optr++ = lcptr[lidx + ld] + *gcptr++; + } + lcptr += dimOfSubvector; + } + } + + void decode(QuantizedObject &quantizedObject, float *globalCentroid, std::vector &object) { + size_t dim = globalCodebookIndex.getObjectSpace().getPaddedDimension(); + size_t nOfSubvectors = property.getLocalCodebookNo(); + size_t dimOfSubvector = dim / nOfSubvectors; + auto *lcptr = static_cast(&localCodebooks[0]); + object.resize(dim, 0.0); + auto *optr = object.data(); + auto *gcptr = static_cast(globalCentroid); + for (size_t li = 0; li < nOfSubvectors; li++) { + auto lidx = (quantizedObject.object[li] - 1) * dim; + for (size_t ld = 0; ld < dimOfSubvector; ld++) { + *optr++ = lcptr[lidx + ld] + *gcptr++; + } + lcptr += dimOfSubvector; + } + } + + void encode(uint32_t subspaceID, Object &object, QuantizedObject &quantizedObject) { + if (object.object.empty()) { + return; + } +#ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS + if (!rotation.empty()) { + rotation.mul(object.object.data()); + } +#endif + (*generateResidualObject)(object.object, // object + subspaceID, + object.object.data()); // subspace objects +#ifndef NGTQG_ROTATED_GLOBAL_CODEBOOKS + rotation.mul(object.object.data()); +#endif + size_t paddedDimension = globalCodebookIndex.getObjectSpace().getPaddedDimension(); + size_t localCodebookNo = property.getLocalCodebookNo(); + size_t codebookSize = property.localCentroidLimit; + if (property.dimension % property.localDivisionNo != 0) { + std::stringstream msg; + msg << "Invalid dimension or # of subspaces. " << property.dimension << ":" << property.localDivisionNo; + NGTThrowException(msg); + } + size_t localDimension = property.dimension / property.localDivisionNo; + quantizedObject.objectID = object.objectID; + quantizedObject.subspaceID = object.subspaceID; + quantizedObject.object.resize(localCodebookNo); + for (size_t svi = 0; svi < localCodebookNo; svi++) { + float mind = std::numeric_limits::max(); + size_t minID = 0; + for (size_t cid = 0; cid < codebookSize; cid++) { + float localDistance = 0.0; + for (size_t ld = 0; ld < localDimension; ld++) { + size_t d = svi * localDimension + ld; + auto dist = object.object[d] - localCodebooks[cid * paddedDimension + d]; + dist *= dist; + localDistance += dist; + } + if (localDistance < mind) { + mind = localDistance; + minID = cid; + } + } + quantizedObject.object[svi] = minID + 1; + } + return; + } + + void encode(uint32_t subspaceID, ObjectSet &objects, IIEntry &invertedIndexEntry) { + QuantizedObjectSet qobjs; + encode(subspaceID, objects, qobjs); + invertedIndexEntry.set(qobjs); + } +#endif + +#ifdef NGTQ_QBG + void encode(uint32_t subspaceID, ObjectSet &objects, QuantizedObjectSet &qobjs) { +#ifdef NGTQ_SHARED_INVERTED_INDEX + std::cerr << "enode: Not implemented." << std::endl; + abort(); +#else + qobjs.resize(objects.size()); +#pragma omp parallel for + for (size_t i = 0; i < objects.size(); i++) { + // multiple local codebooks + encode(subspaceID, objects[i], qobjs[i]); + } +#endif + } +#endif + + #ifdef NGTQ_QBG #ifdef NGTQ_VECTOR_OBJECT - void insert(vector, size_t> > &objects, NGT::Index *gqindex) { + void insert(vector, size_t>> &objects, NGT::Index *gqindex) { #else - void insert(vector > &objects, NGT::Index *gqindex) { + void insert(vector> &objects, NGT::Index *gqindex) { #endif #ifdef NGTQ_SHARED_INVERTED_INDEX std::cerr << "insert: Not implemented." << std::endl; @@ -3363,8 +3824,9 @@ class QuantizerInstance : public Quantizer { getBlobIDFromObjectToBlobIndex(objects, ids); } } else { - std::cerr << "Quantizer::InsertQBG: Warning! invalid centroidCreationMode. " << property.centroidCreationMode << std::endl; - abort(); + std::stringstream msg; + msg << "Quantizer::InsertQBG: Warning! invalid centroidCreationMode. " << property.centroidCreationMode; + NGTThrowException(msg); } #ifdef NGTQ_SHARED_INVERTED_INDEX if (invertedIndex.getAllocatedSize() <= invertedIndex.size() + objects.size()) { @@ -3390,11 +3852,10 @@ class QuantizerInstance : public Quantizer { localData[i].iiIdx, // centroid:ID of global codebook localObjs); #endif -#else +#else #ifdef NGTQ_QBG #ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS - assert(!rotation.empty()); if (!rotation.empty()) { #ifdef NGTQ_VECTOR_OBJECT rotation.mul(objects[i].first.data()); @@ -3402,7 +3863,7 @@ class QuantizerInstance : public Quantizer { rotation.mul(static_cast(objects[i].first->getPointer())); #endif } -#endif +#endif #ifdef NGTQ_VECTOR_OBJECT (*generateResidualObject)(objects[i].first, // object invertedIndexEntry.subspaceID, @@ -3415,12 +3876,12 @@ class QuantizerInstance : public Quantizer { #ifndef NGTQG_ROTATED_GLOBAL_CODEBOOKS rotation.mul(subspaceObjects[i]); #endif -#else +#else (*generateResidualObject)(invertedIndexEntry[localData[i].iiLocalIdx].id, localData[i].iiIdx, // centroid:ID of global codebook localObjs); -#endif -#endif +#endif +#endif } @@ -3468,7 +3929,7 @@ class QuantizerInstance : public Quantizer { #ifndef NGTQ_QBG - void insert(vector &objvector, vector > &objects, size_t count) { + void insert(vector &objvector, vector> &objects, size_t count) { size_t id = count; if (count == 0) { id = objectList.size(); @@ -3500,8 +3961,13 @@ class QuantizerInstance : public Quantizer { #ifdef NGTQ_QBG void createIndex(size_t beginID = 1, size_t endID = 0) { + if (beginID == 0) { + return; + } NGT::Index *gqindex = 0; - if (property.centroidCreationMode == CentroidCreationModeStaticLayer) { + if ((property.centroidCreationMode == CentroidCreationModeStaticLayer || + property.centroidCreationMode == CentroidCreationModeStatic) && + objectToBlobIndex.empty()) { gqindex = buildGlobalCodebookWithQIDIndex(); } #ifdef NGTQ_VECTOR_OBJECT @@ -3518,8 +3984,9 @@ class QuantizerInstance : public Quantizer { for (size_t id = beginID; id <= endID; id++) { if (id % 1000000 == 0) { timer.stop(); - std::cerr << "# of processed objects=" << id << " Time=" << timer << ", vm size=" - << NGT::Common::getProcessVmSizeStr() << std::endl; + std::cerr << "# of processed objects=" << id << ", time=" << timer << ", vm size=" + << NGT::Common::getProcessVmSizeStr() << "/" + << NGT::Common::getProcessVmPeakStr() << std::endl; timer.restart(); } #ifdef NGTQ_VECTOR_OBJECT @@ -3549,14 +4016,14 @@ class QuantizerInstance : public Quantizer { #endif void setupInvertedIndex(std::vector> &qCodebook, - std::vector &codebookIndex, - std::vector &objectIndex) { + std::vector &codebookIndex, + std::vector &objectIndex) { #if !defined(NGTQ_QBG) std::cerr << "setupInvertedIndex: Not implemented." << std::endl; abort(); #else if (globalCodebookIndex.getObjectRepositorySize() != codebookIndex.size() + 1) { - std::cerr << "Error? " << globalCodebookIndex.getObjectRepositorySize() << ":" << codebookIndex.size() + 1 << std::endl; + std::cerr << "Warning: Error? " << globalCodebookIndex.getObjectRepositorySize() << ":" << codebookIndex.size() + 1 << std::endl; } if (!invertedIndex.empty()) { stringstream msg; @@ -3564,7 +4031,7 @@ class QuantizerInstance : public Quantizer { NGTThrowException(msg); } invertedIndex.reserve(codebookIndex.size() + 1); - std::cerr << "codebook Index size=" << codebookIndex.size()<< std::endl; + std::cerr << "codebook index size=" << codebookIndex.size()<< std::endl; for (size_t idx = 0; idx < codebookIndex.size(); idx++) { auto gid = idx + 1; #ifdef NGTQ_SHARED_INVERTED_INDEX @@ -3572,15 +4039,15 @@ class QuantizerInstance : public Quantizer { #else invertedIndex.put(gid, new InvertedIndexEntry(localCodebookIndexes.size())); #endif +#ifdef NGTQBG_COARSE_BLOB + invertedIndex.at(gid)->subspaceID = idx; +#else invertedIndex.at(gid)->subspaceID = codebookIndex[idx]; +#endif } - - - quantizationCodebook.setPaddedDimension(globalCodebookIndex.getObjectSpace().getPaddedDimension()); - quantizationCodebook = qCodebook; -#ifdef NGTQG_ROTATED_GLOBAL_CODEBOOKS - quantizationCodebook.rotate(rotation); +#ifdef NGTQBG_COARSE_BLOB + graphNodeToInvertedIndexEntries.setup(codebookIndex); #endif objectToBlobIndex = std::move(objectIndex); @@ -3658,6 +4125,13 @@ class QuantizerInstance : public Quantizer { lp.edgeSizeForSearch = 40; lp.objectType = NGT::Index::Property::ObjectType::Float; +#ifdef NGTQ_QBG + if (property.genuineDimension > property.dimension) { + stringstream msg; + msg << "NGTQ::Quantizer::create: dimension must be larger than genuineDimension. " << property.dimension << ":" << property.genuineDimension << std::endl; + NGTThrowException(msg); + } +#endif gp.dimension = property.dimension; if (gp.dimension == 0) { stringstream msg; @@ -3669,7 +4143,8 @@ class QuantizerInstance : public Quantizer { } if (property.localDivisionNo != 1 && property.dimension % property.localDivisionNo != 0) { stringstream msg; - msg << "NGTQ::Quantizer::create: dimension and localDivisionNo are not proper. " + msg << "NGTQ::Quantizer::create: The combination of dimension and localDivisionNo is invalid. " + << "the localDivisionNo must be a divisor of the dimension. " << property.dimension << ":" << property.localDivisionNo; NGTThrowException(msg); } @@ -3843,7 +4318,6 @@ class QuantizerInstance : public Quantizer { cerr << " Strange! "; cerr << result[0].distance << ":" << objects[resulti].distance << " "; } - globalCodebookIndex.getObjectSpace().getRepository().deleteObject(o); } cerr << " search object " << resulti << " ID=" << objects[resulti].id << " distance=" << objects[resulti].distance << endl; } @@ -3925,15 +4399,17 @@ class QuantizerInstance : public Quantizer { msg << "Quantizer::extractInvertedIndexObject: Fatal error! Invalid gid. " << invertedIndex.size() << ":" << gid; NGTThrowException(msg); } - if (invertedIndex[gid] == 0) { #ifdef NGTQ_SHARED_INVERTED_INDEX - std::cerr << "Not implemented" << std::endl; + std::cerr << "Not implemented" << std::endl; #else - invertedIndexObjects.clear(); + invertedIndexObjects.clear(); #endif + invertedIndexObjects.initialize(property.getLocalCodebookNo()); + if (invertedIndex[gid] == 0) { return; } invertedIndexObjects.subspaceID = invertedIndex[gid]->subspaceID; + invertedIndexObjects.numOfSubvectors = invertedIndex[gid]->numOfSubvectors; invertedIndexObjects.resize(invertedIndex[gid]->size()); for (size_t idx = 0; idx < invertedIndex[gid]->size(); idx++) { #ifdef NGTQ_SHARED_INVERTED_INDEX @@ -3999,7 +4475,7 @@ class QuantizerInstance : public Quantizer { std::cerr << "you should change the object ID type." << std::endl; abort(); } - for (size_t i = 0; i < invertedIndexObjects.numOfLocalIDs; i++) { + for (size_t i = 0; i < invertedIndexObjects.numOfSubvectors; i++) { invertedIndexObjects[entry.id].localID[i] = entry.localID[i]; } assert(invertedIndexObjects[entry.id].localID[0] == entry.localID[0]); @@ -4343,6 +4819,9 @@ class QuantizerInstance : public Quantizer { QuantizedObjectDistance &getQuantizedObjectDistance() { return *quantizedObjectDistance; } +#ifdef NGTQBG_COARSE_BLOB + GraphNodeToInvertedIndexEntries &getGraphNodeToInvertedIndexEntries() { return graphNodeToInvertedIndexEntries; } +#endif size_t getInvertedIndexSize() { return invertedIndex.size(); } #ifdef NGTQ_SHARED_INVERTED_INDEX @@ -4353,7 +4832,10 @@ class QuantizerInstance : public Quantizer { QuantizedObjectDistance *quantizedObjectDistance; GenerateResidualObject *generateResidualObject; float *localCodebooks; - bool silence; + bool verbose; +#ifdef NGTQBG_COARSE_BLOB + GraphNodeToInvertedIndexEntries graphNodeToInvertedIndexEntries; +#endif }; class Quantization { @@ -4377,7 +4859,6 @@ class Quantization { msg << "Not support the specified size of local ID. " << localIDByteSize; NGTThrowException(msg); } - } return quantizer; @@ -4435,10 +4916,10 @@ class Quantization { static void compress(const string &indexFile) { Index index; index.open(indexFile); - string tmpivt = indexFile + "/ivt-tmp"; + string tmpivt = indexFile + "/" + Quantizer::getInvertedIndexFile() + "-tmp"; index.getQuantizer().reconstructInvertedIndex(tmpivt); index.close(); - string ivt = indexFile + "/ivt"; + string ivt = indexFile + "/" + Quantizer::getInvertedIndexFile(); unlink(ivt.c_str()); rename(tmpivt.c_str(), ivt.c_str()); string ivtc = ivt + "c"; @@ -4501,7 +4982,7 @@ class Quantization { } #ifndef NGTQ_QBG - void insert(vector > &objects) { + void insert(vector> &objects) { std::cerr << "Not implemented." << std::endl; abort(); } @@ -4623,7 +5104,7 @@ class Quantization { NGTQ::Quantizer *quantizer; - bool silence; + bool verbose; }; } // namespace NGTQ diff --git a/lib/NGT/ObjectRepository.h b/lib/NGT/ObjectRepository.h index fb16ab7..64334c2 100644 --- a/lib/NGT/ObjectRepository.h +++ b/lib/NGT/ObjectRepository.h @@ -136,7 +136,7 @@ namespace NGT { } std::vector object; try { - extractObjectFromText(line, "\t ", object); + extractObjectFromText(line, "\t, ", object); PersistentObject *obj = 0; try { obj = allocateNormalizedPersistentObject(object); diff --git a/lib/NGT/ObjectSpace.h b/lib/NGT/ObjectSpace.h index 5254ab1..1368ef1 100644 --- a/lib/NGT/ObjectSpace.h +++ b/lib/NGT/ObjectSpace.h @@ -366,7 +366,46 @@ namespace NGT { assert(0); } } - + template + void set(std::vector &v, ObjectSpace &objectspace) { + const std::type_info &t = objectspace.getObjectType(); + size_t dimension = objectspace.getDimension(); + void *ref = (void*)&(*this)[0]; + if (ref == 0) { + NGTThrowException("BaseObject::set: vector is null"); + } + if (t == typeid(uint8_t)) { + for (size_t d = 0; d < dimension; d++) { + *(static_cast(ref) + d) = v[d]; + } + } else if (t == typeid(float)) { + for (size_t d = 0; d < dimension; d++) { + *(static_cast(ref) + d) = v[d]; + } +#ifdef NGT_HALF_FLOAT + } else if (t == typeid(float16)) { + for (size_t d = 0; d < dimension; d++) { + *(static_cast(ref) + d) = v[d]; + } +#endif + } else if (t == typeid(double)) { + for (size_t d = 0; d < dimension; d++) { + *(static_cast(ref) + d) = v[d]; + } + } else if (t == typeid(uint16_t)) { + for (size_t d = 0; d < dimension; d++) { + *(static_cast(ref) + d) = v[d]; + } + } else if (t == typeid(uint32_t)) { + for (size_t d = 0; d < dimension; d++) { + *(static_cast(ref) + d) = v[d]; + } + } else { + std::stringstream msg; + msg << "BaseObject::set: not supported data type. [" << t.name() << "]"; + NGTThrowException(msg); + } + } }; class Object : public BaseObject { @@ -379,11 +418,20 @@ namespace NGT { construct(s); } + template + Object(std::vector v, NGT::ObjectSpace &os):vector(0) { + size_t s = os.getByteSizeOfObject(); + construct(s); + set(v, os); + } + Object(size_t s):vector(0) { assert(s != 0); construct(s); } + virtual ~Object() { clear(); } + void attach(void *ptr) { vector = static_cast(ptr); } void detach() { vector = 0; } @@ -394,12 +442,12 @@ namespace NGT { } } - virtual ~Object() { clear(); } - uint8_t &operator[](size_t idx) const { return vector[idx]; } void *getPointer(size_t idx = 0) const { return vector + idx; } + bool isEmpty() { return vector == 0; } + static Object *allocate(ObjectSpace &objectspace) { return new Object(&objectspace); } private: void clear() { diff --git a/lib/NGT/Optimizer.h b/lib/NGT/Optimizer.h index 22b98e1..f58a03b 100644 --- a/lib/NGT/Optimizer.h +++ b/lib/NGT/Optimizer.h @@ -399,12 +399,12 @@ namespace NGT { } double accuracy = (double)relevantCount / (double)resultDataSize; double key; - if (epsilon != "") { - key = NGT::Common::strtod(epsilon); - keyValue = "Factor (Epsilon)"; - } else if (expansion != "") { + if (expansion != "") { key = NGT::Common::strtod(expansion); keyValue = "Expansion"; + } else if (epsilon != "") { + key = NGT::Common::strtod(epsilon); + keyValue = "Factor (Epsilon)"; } else { std::stringstream msg; msg << "check: inner error! " << epsilon; diff --git a/lib/NGT/defines.h.in b/lib/NGT/defines.h.in index 97087c6..8522a11 100644 --- a/lib/NGT/defines.h.in +++ b/lib/NGT/defines.h.in @@ -32,6 +32,7 @@ // Release Definitions for OSS //#define NGT_DISTANCE_COMPUTATION_COUNT +//#define NGT_SEARCH_TIMER #define NGT_CREATION_EDGE_SIZE 10 #define NGT_EXPLORATION_COEFFICIENT 1.1 diff --git a/python/src/ngtpy.cpp b/python/src/ngtpy.cpp index 5159cec..23e9d04 100644 --- a/python/src/ngtpy.cpp +++ b/python/src/ngtpy.cpp @@ -628,8 +628,9 @@ class QuantizedBlobIndex : public QBG::Index { size_t maxNoOfEdges, // the maximum number of quantized graph edges. bool zeroBasedNumbering, // object ID numbering. bool treeDisabled, // not use the tree index. - bool logDisabled // stderr log is disabled. - ):QBG::Index(path, true) { + bool logDisabled, // stderr log is disabled. + bool readOnly // open mode + ):QBG::Index(path, readOnly) { zeroNumbering = zeroBasedNumbering; numOfDistanceComputations = 0; treeIndex = !treeDisabled; @@ -644,7 +645,30 @@ class QuantizedBlobIndex : public QBG::Index { defaultNumOfProbes = 0; } -py::array_t batchSearchTmp( + void batchInsert( + py::array_t objects, + bool debug = false + ) { + py::buffer_info info = objects.request(); + if (debug) { + std::cerr << info.shape.size() << ":" << info.shape[0] << ":" << info.shape[1] << std::endl; + } + auto ptr = static_cast(info.ptr); + assert(info.shape.size() == 2); + for (int idx = 0; idx < info.shape[0]; idx++) { + if (debug) { + for (int i = 0; i < info.shape[1]; i++) { + std::cerr << *(ptr + i) << " "; + } + std::cerr << std::endl; + } + std::vector v(ptr, ptr + info.shape[1]); + ptr += info.shape[1]; + QBG::Index::append(v); + } + } + + py::array_t batchSearchTmp( py::array_t queries, size_t size ) { @@ -670,7 +694,7 @@ py::array_t batchSearchTmp( sc.setBlobEpsilon(defaultBlobEpsilon); sc.setEdgeSize(defaultEdgeSize); sc.setGraphExplorationSize(defaultExplorationSize); - QBG::Index::searchBlobGraph(sc); + QBG::Index::search(sc); NGT::ResultPriorityQueue &r = sc.getWorkingResult(); if (r.size() != size) { std::cerr << "result size is invalid? " << r.size() << ":" << size << std::endl; @@ -685,7 +709,7 @@ py::array_t batchSearchTmp( return results; } - void batchApproximateSearchWithoutGraph( + void batchSearchInTwoSteps( py::array_t queries, BatchResults &results, size_t size @@ -705,25 +729,27 @@ py::array_t batchSearchTmp( #pragma omp parallel for schedule(dynamic) for (int idx = 0; idx < nOfQueries; idx++) { - NGT::Object queryObject(psedoDimension * sizeof(float)); float *qptr = queryPtr + idx * dimension; - if (psedoDimension > dimension) { - memset(queryObject.getPointer(), 0, psedoDimension * sizeof(float)); - } - memcpy(queryObject.getPointer(), qptr, dimension * sizeof(float)); - QBG::SearchContainer sc(queryObject); + vector query(psedoDimension, 0); + memcpy(query.data(), qptr, dimension * sizeof(float)); + QBG::SearchContainer sc; + sc.setObjectVector(query); sc.setSize(size); sc.setEpsilon(defaultEpsilon); + sc.setBlobEpsilon(defaultBlobEpsilon); sc.setEdgeSize(defaultEdgeSize); sc.setNumOfProbes(defaultNumOfProbes); - QBG::Index::searchBlobNaively(sc); +#ifdef NGTQBG_FUNCTION_SELECTOR + sc.functionSelector = defaultFunctionSelector; ///////////////// +#endif + QBG::Index::searchInTwoSteps(sc); results.results[idx] = std::move(sc.getWorkingResult()); } results.size = results.results.size(); return; } - void batchApproximateSearch( + void batchSearchInOneStep( py::array_t queries, BatchResults &results, size_t size @@ -741,6 +767,12 @@ py::array_t batchSearchTmp( results.resultList.clear(); results.results.resize(nOfQueries); + size_t searchSize = size; + size_t searchExactResultSize = 0; + if (defaultExactResultExpansion >= 1.0) { + searchSize = static_cast(size) * defaultExactResultExpansion; + searchExactResultSize = size; + } #pragma omp parallel for schedule(dynamic) for (int idx = 0; idx < nOfQueries; idx++) { float *qptr = queryPtr + idx * dimension; @@ -748,12 +780,13 @@ py::array_t batchSearchTmp( memcpy(query.data(), qptr, dimension * sizeof(float)); QBG::SearchContainer sc; sc.setObjectVector(query); - sc.setSize(size); + sc.setSize(searchSize); + sc.setExactResultSize(searchExactResultSize); sc.setEpsilon(defaultEpsilon); sc.setBlobEpsilon(defaultBlobEpsilon); sc.setEdgeSize(defaultEdgeSize); sc.setGraphExplorationSize(defaultExplorationSize); - QBG::Index::searchBlobGraph(sc); + QBG::Index::search(sc); results.results[idx] = std::move(sc.getWorkingResult()); //QBG::Index::getQuantizer().globalCodebookIndex.deleteObject(queryObject); } @@ -761,61 +794,15 @@ py::array_t batchSearchTmp( return; } - void batchExactSearch( - py::array_t queries, - BatchResults &results, - size_t size - ) { - const py::buffer_info &qinfo = queries.request(); - const std::vector &qshape = qinfo.shape; - auto nOfQueries = qshape[0]; - size_t dimension = qshape[1]; - size_t psedoDimension = QBG::Index::getQuantizer().globalCodebookIndex.getObjectSpace().getPaddedDimension(); - auto *queryPtr = static_cast(qinfo.ptr); - - size = size > 0 ? size : defaultNumOfSearchObjects; - - results.results.clear(); - results.resultList.clear(); - results.resultList.resize(nOfQueries); - -#pragma omp parallel for schedule(dynamic) - for (int idx = 0; idx < nOfQueries; idx++) { - NGT::Object queryObject(psedoDimension * sizeof(float)); - float *qptr = queryPtr + idx * dimension; - if (psedoDimension > dimension) { - memset(queryObject.getPointer(), 0, psedoDimension * sizeof(float)); - } - memcpy(queryObject.getPointer(), qptr, dimension * sizeof(float)); - NGT::ObjectDistances rs; - QBG::SearchContainer sc(queryObject); - sc.setResults(&rs); - sc.setSize(static_cast(size) * defaultExactResultExpansion); - sc.setExactResultSize(size); - sc.setEpsilon(defaultEpsilon); - sc.setBlobEpsilon(defaultBlobEpsilon); - sc.setEdgeSize(defaultEdgeSize); - sc.setGraphExplorationSize(defaultExplorationSize); - QBG::Index::searchBlobGraph(sc); - results.resultList[idx] = std::move(rs); - } - results.size = results.resultList.size(); - return; - } - void batchSearch( py::array_t queries, BatchResults &results, size_t size ) { if (defaultNumOfProbes == 0) { - if (defaultExactResultExpansion > 1.0) { - batchExactSearch(queries, results, size); - } else { - batchApproximateSearch(queries, results, size); - } + batchSearchInOneStep(queries, results, size); } else { - batchApproximateSearchWithoutGraph(queries, results, size); + batchSearchInTwoSteps(queries, results, size); } return; } @@ -840,20 +827,18 @@ py::array_t batchSearchTmp( #pragma omp parallel for schedule(dynamic) for (int idx = 0; idx < nOfQueries; idx++) { - NGT::Object queryObject(dimension * sizeof(float)); float *qptr = queryPtr + idx * dimension; - if (psedoDimension > dimension) { - memset(queryObject.getPointer(), 0, psedoDimension * sizeof(float)); - } - memcpy(queryObject.getPointer(), qptr, dimension * sizeof(float)); - QBG::SearchContainer sc(queryObject); + vector query(psedoDimension, 0); + memcpy(query.data(), qptr, dimension * sizeof(float)); + QBG::SearchContainer sc; + sc.setObjectVector(query); sc.setSize(std::numeric_limits::max()); sc.setRadius(radius); sc.setEpsilon(defaultEpsilon); sc.setBlobEpsilon(defaultBlobEpsilon); sc.setEdgeSize(defaultEdgeSize); sc.setGraphExplorationSize(defaultExplorationSize); - QBG::Index::searchBlobGraph(sc); + QBG::Index::search(sc); results.results[idx] = std::move(sc.getWorkingResult()); } results.size = results.results.size(); @@ -863,31 +848,37 @@ py::array_t batchSearchTmp( py::object search( py::object query, size_t size, // the number of resultant objects - float epsilon, // search parameter epsilon. the adequate range is from 0.0 to 0.05. - float resultExpansion, // the number of inner resultant objects - int edgeSize // the number of used edges for each node during the exploration of the graph. + float epsilon // search parameter epsilon. the adequate range is from 0.0 to 0.05. ) { py::array_t qobject(query); py::buffer_info qinfo = qobject.request(); std::vector qvector(static_cast(qinfo.ptr), static_cast(qinfo.ptr) + qinfo.size); try { - NGT::Object *qobject = QBG::Index::getQuantizer().globalCodebookIndex.allocateObject(qvector); - QBG::SearchContainer sc(*qobject); + QBG::SearchContainer sc; + sc.setObjectVector(qvector); size = size > 0 ? size : defaultNumOfSearchObjects; epsilon = epsilon > -1.0 ? epsilon : defaultEpsilon; - sc.setSize(size); // the number of resulting objects. + if (defaultExactResultExpansion >= 1.0) { + sc.setSize(static_cast(size) * defaultExactResultExpansion); + sc.setExactResultSize(size); + } else { + sc.setSize(size); // the number of resulting objects. + } sc.setEpsilon(epsilon); // set exploration coefficient. sc.setBlobEpsilon(defaultBlobEpsilon); - std::cerr << "ngtpy search" << std::endl; - QBG::Index::searchBlobGraph(sc); + sc.setEdgeSize(defaultEdgeSize); + sc.setGraphExplorationSize(defaultExplorationSize); + sc.setNumOfProbes(defaultNumOfProbes); +#ifdef NGTQBG_FUNCTION_SELECTOR + sc.functionSelector = defaultFunctionSelector; ///////////////// +#endif + QBG::Index::searchInTwoSteps(sc); - QBG::Index::getQuantizer().globalCodebookIndex.deleteObject(qobject); numOfDistanceComputations += sc.distanceComputationCount; if (!withDistance) { - std::cerr << "without distances" << std::endl; + //-/std::cerr << "without distances" << std::endl; NGT::ResultPriorityQueue &r = sc.getWorkingResult(); - std::cerr << "size=" << r.size() << std::endl; py::array_t ids(r.size()); py::buffer_info idsinfo = ids.request(); int *endptr = reinterpret_cast(idsinfo.ptr); @@ -906,20 +897,16 @@ py::array_t batchSearchTmp( return ids; } - std::cerr << "with distances" << std::endl; py::list results; NGT::ObjectDistances r; r.moveFrom(sc.getWorkingResult()); - std::cerr << "size=" << r.size() << std::endl; if (zeroNumbering) { for (auto ri = r.begin(); ri != r.end(); ++ri) { results.append(py::make_tuple((*ri).id - 1, (*ri).distance)); - std::cerr << "ID(1)=" << (*ri).id << std::endl; } } else { for (auto ri = r.begin(); ri != r.end(); ++ri) { results.append(py::make_tuple((*ri).id, (*ri).distance)); - std::cerr << "ID(2)=" << (*ri).id << std::endl; } } return results; @@ -945,6 +932,9 @@ py::array_t batchSearchTmp( int explorationSize, // the maximum number of nodes that are searched float exactResultExpansion, int numOfProbes +#ifdef NGTQBG_FUNCTION_SELECTOR + , size_t functionSelector /////////////////// +#endif ) { defaultNumOfSearchObjects = numOfSearchObjects > 0 ? numOfSearchObjects : defaultNumOfSearchObjects; defaultEpsilon = epsilon > -1.0 ? epsilon : defaultEpsilon; @@ -955,6 +945,9 @@ py::array_t batchSearchTmp( defaultRadius = radius >= 0.0 ? radius : defaultRadius; defaultExactResultExpansion = exactResultExpansion > 0.0 ? exactResultExpansion : defaultExactResultExpansion; defaultNumOfProbes = numOfProbes > 0 ? numOfProbes : defaultNumOfProbes; +#ifdef NGTQBG_FUNCTION_SELECTOR + defaultFunctionSelector = functionSelector; //////////////// +#endif } bool zeroNumbering; // for object ID numbering. zero-based or one-based numbering. @@ -963,6 +956,7 @@ py::array_t batchSearchTmp( bool withDistance; size_t defaultNumOfSearchObjects; // k float defaultEpsilon; + float defaultBlobEpsilon; float defaultResultExpansion; int64_t defaultEdgeSize; @@ -970,6 +964,9 @@ py::array_t batchSearchTmp( float defaultRadius; float defaultExactResultExpansion; size_t defaultNumOfProbes; +#ifdef NGTQBG_FUNCTION_SELECTOR + size_t defaultFunctionSelector; ////////////////////////// +#endif }; PYBIND11_MODULE(ngtpy, m) { @@ -1118,29 +1115,32 @@ PYBIND11_MODULE(ngtpy, m) { py::class_(m, "QuantizedBlobIndex") - .def(py::init(), + .def(py::init(), py::arg("path"), py::arg("max_no_of_edges") = 128, py::arg("zero_based_numbering") = true, py::arg("tree_disabled") = false, - py::arg("log_disabled") = false) - .def("batchSearchTmp", &::QuantizedBlobIndex::batchSearchTmp, + py::arg("log_disabled") = false, + py::arg("read_only") = true) + .def("save", (void (QBG::Index::*)()) &QBG::Index::save) + .def("batch_insert", &::QuantizedBlobIndex::batchInsert, + py::arg("objects"), + py::arg("debug") = false) + .def("batch_search_tmp", &::QuantizedBlobIndex::batchSearchTmp, py::arg("query"), py::arg("size") = 0) - .def("batchSearch", &::QuantizedBlobIndex::batchSearch, + .def("batch_search", &::QuantizedBlobIndex::batchSearch, py::arg("query"), py::arg("results"), py::arg("size") = 0) - .def("batchRangeSearch", &::QuantizedBlobIndex::batchRangeSearch, + .def("batch_range_search", &::QuantizedBlobIndex::batchRangeSearch, py::arg("query"), py::arg("results"), py::arg("radius") = -FLT_MAX) .def("search", &::QuantizedBlobIndex::search, py::arg("query"), py::arg("size") = 0, - py::arg("epsilon") = -FLT_MAX, - py::arg("result_expansion") = -FLT_MAX, - py::arg("edge_size") = INT_MIN) + py::arg("epsilon") = -FLT_MAX) .def("set_with_distance", &::QuantizedBlobIndex::setWithDistance, py::arg("boolean") = true) .def("set", &::QuantizedBlobIndex::set, @@ -1152,16 +1152,20 @@ PYBIND11_MODULE(ngtpy, m) { py::arg("edge_size") = INT_MIN, py::arg("exploration_size") = 0, py::arg("exact_result_expansion") = 0.0, - py::arg("num_of_probes") = INT_MIN); + py::arg("num_of_probes") = INT_MIN +#ifdef NGTQBG_FUNCTION_SELECTOR + , py::arg("function_selector") = 0 /////////// +#endif + ); py::class_(m, "BatchResults") .def(py::init<>()) .def("get", &::BatchResults::get, py::arg("position")) - .def("getIDs", &::BatchResults::getIDs) - .def("getIndexedIDs", &::BatchResults::getIndexedIDs) - .def("getIndexedDistances", &::BatchResults::getIndexedDistances) - .def("getIndex", &::BatchResults::getIndex) - .def("getSize", &::BatchResults::getSize); + .def("get_ids", &::BatchResults::getIDs) + .def("get_indexed_ids", &::BatchResults::getIndexedIDs) + .def("get_indexed_distances", &::BatchResults::getIndexedDistances) + .def("get_index", &::BatchResults::getIndex) + .def("get_size", &::BatchResults::getSize); } diff --git a/samples/qbg-capi/qbg-capi.cpp b/samples/qbg-capi/qbg-capi.cpp index 52bd877..f5ecb58 100644 --- a/samples/qbg-capi/qbg-capi.cpp +++ b/samples/qbg-capi/qbg-capi.cpp @@ -4,6 +4,7 @@ int main(int argc, char **argv) { +#if !defined(NGT_SHARED_MEMORY_ALLOCATOR) std::string indexPath = "index"; std::string objectFile = "sift-128-euclidean.tsv"; std::string queryFile = "query.tsv"; @@ -133,6 +134,6 @@ main(int argc, char **argv) qbg_close_index(index); ngt_destroy_error_object(err); - +#endif return 0; } diff --git a/samples/qg-capi/qg-capi.cpp b/samples/qg-capi/qg-capi.cpp index 0218a2d..70264b4 100644 --- a/samples/qg-capi/qg-capi.cpp +++ b/samples/qg-capi/qg-capi.cpp @@ -4,6 +4,7 @@ int main(int argc, char **argv) { +#if !defined(NGT_SHARED_MEMORY_ALLOCATOR) std::string indexPath = "index"; std::string objectFile = "sift-128-euclidean.tsv"; std::string queryFile = "query.tsv"; @@ -140,6 +141,6 @@ main(int argc, char **argv) std::cerr << "close the quantized index" << std::endl; ngtqg_close_index(index); ngt_destroy_error_object(err); - +#endif return 0; }