diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 783e62a..056e0e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,7 @@ ADD_DEFINITIONS(-DNOMINMAX) -SET( srcs fbow.cpp vocabulary_creator.cpp ) -SET( hdrs fbow.h fbow_exports.h vocabulary_creator.h cpu.h) +SET( srcs fbow.cpp vocabulary_creator.cpp Database.cpp QueryResults.cpp) +SET( hdrs fbow.h fbow_exports.h vocabulary_creator.h cpu.h Database.h QueryResults.h) SET(THIS_LIBNAME fbow) diff --git a/src/Database.cpp b/src/Database.cpp new file mode 100644 index 0000000..af8fd6e --- /dev/null +++ b/src/Database.cpp @@ -0,0 +1,987 @@ +#include "Database.h" + +namespace fbow{ + +// -------------------------------------------------------------------------- + + +Database::Database + (bool use_di, int di_levels) + : m_voc(NULL), m_use_di(use_di), m_dilevels(di_levels), m_nentries(0) +{ +} + +// -------------------------------------------------------------------------- + +Database::Database + (const Vocabulary &voc, bool use_di, int di_levels) + : m_voc(NULL), m_use_di(use_di), m_dilevels(di_levels) +{ + setVocabulary(voc); + clear(); +} + +// -------------------------------------------------------------------------- + + +Database::Database + (const Database &db) + : m_voc(NULL) +{ + *this = db; +} + +// -------------------------------------------------------------------------- + + +Database::Database + (const std::string &filename) + : m_voc(NULL) +{ + load(filename); +} + +// -------------------------------------------------------------------------- + + +Database::Database + (const char *filename) + : m_voc(NULL) +{ + load(filename); +} + +// -------------------------------------------------------------------------- + + +Database::~Database(void) +{ + m_voc->clear(); + delete m_voc; +} + +// -------------------------------------------------------------------------- + + +Database& Database::operator= + (const Database &db) +{ + if(this != &db) + { + //m_dfile = db.m_dfile; + m_dilevels = db.m_dilevels; + m_ifile = db.m_ifile; + m_nentries = db.m_nentries; + m_use_di = db.m_use_di; + if (db.m_voc!=0) setVocabulary(*db.m_voc); + } + return *this; +} + +// -------------------------------------------------------------------------- + +EntryId Database::add( + const cv::Mat &features) +{ + fBow aux; + fBow v;// = (bowvec ? *bowvec : aux); + + v = m_voc->transform(features); // with features + + /* + for (std::pair e : v){ + std::cout << e.first << "," << e.second << std::endl;; + } + */ + + return add(v); +} + +// --------------------------------------------------------------------------- + + +EntryId Database::add(const fBow &v//, + //const FeatureVector &fv + ) +{ + EntryId entry_id = m_nentries++; + + fBow::const_iterator vit; + std::vector::const_iterator iit; + + /* + if(m_use_di) + { + // update direct file + if(entry_id == m_dfile.size()) + { + m_dfile.push_back(fv); + } + else + { + m_dfile[entry_id] = fv; + } + } + */ + + // update inverted file + for(vit = v.begin(); vit != v.end(); ++vit) + { + const WordId& word_id = vit->first; + const WordValue& word_weight = vit->second; + + IFRow& ifrow = m_ifile[word_id]; + ifrow.push_back(IFPair(entry_id, word_weight)); + } + + return entry_id; +} + +// -------------------------------------------------------------------------- + + + void Database::setVocabulary + (const Vocabulary& voc) +{ + delete m_voc; + m_voc = new Vocabulary(voc); + clear(); +} + +// -------------------------------------------------------------------------- + + + void Database::setVocabulary + (const Vocabulary& voc, bool use_di, int di_levels) +{ + m_use_di = use_di; + m_dilevels = di_levels; + delete m_voc; + m_voc = new Vocabulary(voc); + clear(); +} + +// -------------------------------------------------------------------------- + + + const Vocabulary* +Database::getVocabulary() const +{ + return m_voc; +} + +// -------------------------------------------------------------------------- + + + void Database::clear() +{ + // resize vectors + m_ifile.resize(0); + m_ifile.resize(m_voc->totalSize()); + //m_dfile.resize(0); + m_nentries = 0; +} + +// -------------------------------------------------------------------------- + + +void Database::allocate(int nd, int ni) +{ + // m_ifile already contains |words| items + if(ni > 0) + { + for(auto rit = m_ifile.begin(); rit != m_ifile.end(); ++rit) + { + int n = (int)rit->size(); + if(ni > n) + { + rit->resize(ni); + rit->resize(n); + } + } + } + + //if(m_use_di && (int)m_dfile.size() < nd) + //{ + // m_dfile.resize(nd); + //} +} + + + +// -------------------------------------------------------------------------- + +void Database::query( + const cv::Mat &features, + QueryResults &ret, int max_results, int max_id) const +{ + fBow vec = m_voc->transform(features); + query(vec, ret, max_results, max_id); +} + +// -------------------------------------------------------------------------- + + +void Database::query( + const fBow &vec, + QueryResults &ret, int max_results, int max_id) const +{ + ret.resize(0); + +/* +switch(m_voc->getScoringType()) + { + case L1_NORM: + queryL1(vec, ret, max_results, max_id); + break; + + case L2_NORM: + queryL2(vec, ret, max_results, max_id); + break; + + case CHI_SQUARE: + queryChiSquare(vec, ret, max_results, max_id); + break; + + case KL: + queryKL(vec, ret, max_results, max_id); + break; + + case BHATTACHARYYA: + queryBhattacharyya(vec, ret, max_results, max_id); + break; + + case DOT_PRODUCT: + queryDotProduct(vec, ret, max_results, max_id); + break; + } + */ + queryL2(vec, ret, max_results, max_id); +} + +// -------------------------------------------------------------------------- + + +void Database::queryL1(const fBow &vec, + QueryResults &ret, int max_results, int max_id) const +{ + fBow::const_iterator vit; + + std::map pairs; + std::map::iterator pit; + + for(vit = vec.begin(); vit != vec.end(); ++vit) + { + const WordId word_id = vit->first; + const WordValue& qvalue = vit->second; + + const IFRow& row = m_ifile[word_id]; + + // IFRows are sorted in ascending entry_id order + + for(auto rit = row.begin(); rit != row.end(); ++rit) + { + const EntryId entry_id = rit->entry_id; + const WordValue& dvalue = rit->word_weight; + + if((int)entry_id < max_id || max_id == -1) + { + double value = fabs(qvalue - dvalue) - fabs(qvalue) - fabs(dvalue); + + pit = pairs.lower_bound(entry_id); + if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) + { + pit->second += value; + } + else + { + pairs.insert(pit, + std::map::value_type(entry_id, value)); + } + } + + } // for each inverted row + } // for each query word + + // move to vector + ret.reserve(pairs.size()); + for(pit = pairs.begin(); pit != pairs.end(); ++pit) + { + ret.push_back(Result(pit->first, pit->second)); + } + + // resulting "scores" are now in [-2 best .. 0 worst] + + // sort vector in ascending order of score + std::sort(ret.begin(), ret.end()); + // (ret is inverted now --the lower the better--) + + // cut vector + if(max_results > 0 && (int)ret.size() > max_results) + ret.resize(max_results); + + // complete and scale score to [0 worst .. 1 best] + // ||v - w||_{L1} = 2 + Sum(|v_i - w_i| - |v_i| - |w_i|) + // for all i | v_i != 0 and w_i != 0 + // (Nister, 2006) + // scaled_||v - w||_{L1} = 1 - 0.5 * ||v - w||_{L1} + QueryResults::iterator qit; + for(qit = ret.begin(); qit != ret.end(); qit++) + qit->Score = -qit->Score/2.0; +} + +// -------------------------------------------------------------------------- + + +void Database::queryL2(const fBow &vec, + QueryResults &ret, int max_results, int max_id) const +{ + fBow::const_iterator vit; + + std::map pairs; + std::map::iterator pit; + + //map counters; + //map::iterator cit; + + for(vit = vec.begin(); vit != vec.end(); ++vit) + { + const WordId word_id = vit->first; + const WordValue& qvalue = vit->second; + + const IFRow& row = m_ifile[word_id]; + + // IFRows are sorted in ascending entry_id order + + for(auto rit = row.begin(); rit != row.end(); ++rit) + { + const EntryId entry_id = rit->entry_id; + const WordValue& dvalue = rit->word_weight; + + if((int)entry_id < max_id || max_id == -1) + { + double value = - qvalue * dvalue; // minus sign for sorting trick + + pit = pairs.lower_bound(entry_id); + //cit = counters.lower_bound(entry_id); + if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) + { + pit->second += value; + //cit->second += 1; + } + else + { + pairs.insert(pit, + std::map::value_type(entry_id, value)); + + //counters.insert(cit, + // map::value_type(entry_id, 1)); + } + } + + } // for each inverted row + } // for each query word + + // move to vector + ret.reserve(pairs.size()); + //cit = counters.begin(); + for(pit = pairs.begin(); pit != pairs.end(); ++pit)//, ++cit) + { + ret.push_back(Result(pit->first, pit->second));// / cit->second)); + } + + // resulting "scores" are now in [-1 best .. 0 worst] + + // sort vector in ascending order of score + std::sort(ret.begin(), ret.end()); + // (ret is inverted now --the lower the better--) + + // cut vector + if(max_results > 0 && (int)ret.size() > max_results) + ret.resize(max_results); + + // complete and scale score to [0 worst .. 1 best] + // ||v - w||_{L2} = sqrt( 2 - 2 * Sum(v_i * w_i) + // for all i | v_i != 0 and w_i != 0 ) + // (Nister, 2006) + QueryResults::iterator qit; + for(qit = ret.begin(); qit != ret.end(); qit++) + { + if(qit->Score <= -1.0) // rounding error + qit->Score = 1.0; + else + qit->Score = 1.0 - sqrt(1.0 + qit->Score); // [0..1] + // the + sign is ok, it is due to - sign in + // value = - qvalue * dvalue + } + +} + +// -------------------------------------------------------------------------- + + +void Database::queryChiSquare(const fBow &vec, + QueryResults &ret, int max_results, int max_id) const +{ + fBow::const_iterator vit; + + std::map > pairs; + std::map >::iterator pit; + + std::map > sums; // < sum vi, sum wi > + std::map >::iterator sit; + + // In the current implementation, we suppose vec is not normalized + + //map expected; + //map::iterator eit; + + for(vit = vec.begin(); vit != vec.end(); ++vit) + { + const WordId word_id = vit->first; + const WordValue& qvalue = vit->second; + + const IFRow& row = m_ifile[word_id]; + + // IFRows are sorted in ascending entry_id order + + for(auto rit = row.begin(); rit != row.end(); ++rit) + { + const EntryId entry_id = rit->entry_id; + const WordValue& dvalue = rit->word_weight; + + if((int)entry_id < max_id || max_id == -1) + { + // (v-w)^2/(v+w) - v - w = -4 vw/(v+w) + // we move the 4 out + double value = 0; + if(qvalue + dvalue != 0.0) // words may have weight zero + value = - qvalue * dvalue / (qvalue + dvalue); + + pit = pairs.lower_bound(entry_id); + sit = sums.lower_bound(entry_id); + //eit = expected.lower_bound(entry_id); + if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) + { + pit->second.first += value; + pit->second.second += 1; + //eit->second += dvalue; + sit->second.first += qvalue; + sit->second.second += dvalue; + } + else + { + pairs.insert(pit, + std::map >::value_type(entry_id, + std::make_pair(value, 1) )); + //expected.insert(eit, + // map::value_type(entry_id, dvalue)); + + sums.insert(sit, + std::map >::value_type(entry_id, + std::make_pair(qvalue, dvalue) )); + } + } + + } // for each inverted row + } // for each query word + + // move to vector + ret.reserve(pairs.size()); + sit = sums.begin(); + for(pit = pairs.begin(); pit != pairs.end(); ++pit, ++sit) + { + if(pit->second.second >= MIN_COMMON_WORDS) + { + ret.push_back(Result(pit->first, pit->second.first)); + ret.back().nWords = pit->second.second; + ret.back().sumCommonVi = sit->second.first; + ret.back().sumCommonWi = sit->second.second; + ret.back().expectedChiScore = + 2 * sit->second.second / (1 + sit->second.second); + } + + //ret.push_back(Result(pit->first, pit->second)); + } + + // resulting "scores" are now in [-2 best .. 0 worst] + // we have to add +2 to the scores to obtain the chi square score + + // sort vector in ascending order of score + std::sort(ret.begin(), ret.end()); + // (ret is inverted now --the lower the better--) + + // cut vector + if(max_results > 0 && (int)ret.size() > max_results) + ret.resize(max_results); + + // complete and scale score to [0 worst .. 1 best] + QueryResults::iterator qit; + for(qit = ret.begin(); qit != ret.end(); qit++) + { + // this takes the 4 into account + qit->Score = - 2. * qit->Score; // [0..1] + + qit->chiScore = qit->Score; + } + +} + +// -------------------------------------------------------------------------- + + +void Database::queryKL(const fBow &vec, + QueryResults &ret, int max_results, int max_id) const +{ + fBow::const_iterator vit; + + std::map pairs; + std::map::iterator pit; + + for(vit = vec.begin(); vit != vec.end(); ++vit) + { + const WordId word_id = vit->first; + const WordValue& vi = vit->second; + + const IFRow& row = m_ifile[word_id]; + + // IFRows are sorted in ascending entry_id order + + for(auto rit = row.begin(); rit != row.end(); ++rit) + { + const EntryId entry_id = rit->entry_id; + const WordValue& wi = rit->word_weight; + + if((int)entry_id < max_id || max_id == -1) + { + double value = 0; + if(vi != 0 && wi != 0) value = vi * log(vi/wi); + + pit = pairs.lower_bound(entry_id); + if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) + { + pit->second += value; + } + else + { + pairs.insert(pit, + std::map::value_type(entry_id, value)); + } + } + + } // for each inverted row + } // for each query word + + // resulting "scores" are now in [-X worst .. 0 best .. X worst] + // but we cannot make sure which ones are better without calculating + // the complete score + + // complete scores and move to vector + ret.reserve(pairs.size()); + for(pit = pairs.begin(); pit != pairs.end(); ++pit) + { + EntryId eid = pit->first; + double value = 0.0; + + for(vit = vec.begin(); vit != vec.end(); ++vit) + { + const WordValue &vi = vit->second; + const IFRow& row = m_ifile[vit->first]; + + if(vi != 0) + { + if(row.end() == find(row.begin(), row.end(), eid )) + { + value += vi * (log(vi) - log(DBL_EPSILON)); + } + } + } + + pit->second += value; + + // to vector + ret.push_back(Result(pit->first, pit->second)); + } + + // real scores are now in [0 best .. X worst] + + // sort vector in ascending order + // (scores are inverted now --the lower the better--) + std::sort(ret.begin(), ret.end()); + + // cut vector + if(max_results > 0 && (int)ret.size() > max_results) + ret.resize(max_results); + + // cannot scale scores + +} + +// -------------------------------------------------------------------------- + + +void Database::queryBhattacharyya( + const fBow &vec, QueryResults &ret, int max_results, int max_id) const +{ + fBow::const_iterator vit; + + //map pairs; + //map::iterator pit; + + std::map > pairs; // > + std::map >::iterator pit; + + for(vit = vec.begin(); vit != vec.end(); ++vit) + { + const WordId word_id = vit->first; + const WordValue& qvalue = vit->second; + + const IFRow& row = m_ifile[word_id]; + + // IFRows are sorted in ascending entry_id order + + for(auto rit = row.begin(); rit != row.end(); ++rit) + { + const EntryId entry_id = rit->entry_id; + const WordValue& dvalue = rit->word_weight; + + if((int)entry_id < max_id || max_id == -1) + { + double value = sqrt(qvalue * dvalue); + + pit = pairs.lower_bound(entry_id); + if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) + { + pit->second.first += value; + pit->second.second += 1; + } + else + { + pairs.insert(pit, + std::map >::value_type(entry_id, + std::make_pair(value, 1))); + } + } + + } // for each inverted row + } // for each query word + + // move to vector + ret.reserve(pairs.size()); + for(pit = pairs.begin(); pit != pairs.end(); ++pit) + { + if(pit->second.second >= MIN_COMMON_WORDS) + { + ret.push_back(Result(pit->first, pit->second.first)); + ret.back().nWords = pit->second.second; + ret.back().bhatScore = pit->second.first; + } + } + + // scores are already in [0..1] + + // sort vector in descending order + std::sort(ret.begin(), ret.end(), Result::gt); + + // cut vector + if(max_results > 0 && (int)ret.size() > max_results) + ret.resize(max_results); + +} + +// --------------------------------------------------------------------------- + + +void Database::queryDotProduct( + const fBow &vec, QueryResults &ret, int max_results, int max_id) const +{ + fBow::const_iterator vit; + + std::map pairs; + std::map::iterator pit; + + for(vit = vec.begin(); vit != vec.end(); ++vit) + { + const WordId word_id = vit->first; + const WordValue& qvalue = vit->second; + + const IFRow& row = m_ifile[word_id]; + + // IFRows are sorted in ascending entry_id order + + for(auto rit = row.begin(); rit != row.end(); ++rit) + { + const EntryId entry_id = rit->entry_id; + const WordValue& dvalue = rit->word_weight; + + if((int)entry_id < max_id || max_id == -1) + { + double value; + //if(this->m_voc->getWeightingType() == BINARY) + // value = 1; + //else + value = qvalue * dvalue; + + pit = pairs.lower_bound(entry_id); + if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) + { + pit->second += value; + } + else + { + pairs.insert(pit, + std::map::value_type(entry_id, value)); + } + } + + } // for each inverted row + } // for each query word + + // move to vector + ret.reserve(pairs.size()); + for(pit = pairs.begin(); pit != pairs.end(); ++pit) + { + ret.push_back(Result(pit->first, pit->second)); + } + + // scores are the greater the better + + // sort vector in descending order + std::sort(ret.begin(), ret.end(), Result::gt); + + // cut vector + if(max_results > 0 && (int)ret.size() > max_results) + ret.resize(max_results); + + // these scores cannot be scaled +} + +// --------------------------------------------------------------------------- + + +/* +const FeatureVector& Database::retrieveFeatures + (EntryId id) const +{ + assert(id < size()); + return m_dfile[id]; +} +*/ + +// -------------------------------------------------------------------------- + + +void Database::save(const std::string &filename) const +{ + cv::FileStorage fs(filename.c_str(), cv::FileStorage::WRITE); + if(!fs.isOpened()) throw std::string("Could not open file ") + filename; + + save(fs); + + m_voc->saveToFile(std::string("voc_") + filename); + +} + +// -------------------------------------------------------------------------- + +void Database::save(cv::FileStorage &fs, + const std::string &name) const +{ + // Format YAML: + // vocabulary { ... see TemplatedVocabulary::save } + // database + // { + // nEntries: + // usingDI: + // diLevels: + // invertedIndex + // [ + // [ + // { + // imageId: + // weight: + // } + // ] + // ] + // directIndex + // [ + // [ + // { + // nodeId: + // features: [ ] + // } + // ] + // ] + + // invertedIndex[i] is for the i-th word + // directIndex[i] is for the i-th entry + // directIndex may be empty if not using direct index + // + // imageId's and nodeId's must be stored in ascending order + // (according to the construction of the indexes) + + //m_voc->save(fs); + + fs << name << "{"; + + fs << "nEntries" << m_nentries; + fs << "usingDI" << (m_use_di ? 1 : 0); + fs << "diLevels" << m_dilevels; + + fs << "invertedIndex" << "["; + + for(auto iit = m_ifile.begin(); iit != m_ifile.end(); ++iit) + { + fs << "["; // word of IF + for(auto irit = iit->begin(); irit != iit->end(); ++irit) + { + fs << "{:" + << "imageId" << (int)irit->entry_id + << "weight" << irit->word_weight + << "}"; + } + fs << "]"; // word of IF + } + + fs << "]"; // invertedIndex + + fs << "directIndex" << "["; + + /* + for(auto dit = m_dfile.begin(); dit != m_dfile.end(); ++dit) + { + fs << "["; // entry of DF + + for(auto drit = dit->begin(); drit != dit->end(); ++drit) + { + NodeId nid = drit->first; + const std::vector& features = drit->second; + + // save info of last_nid + fs << "{"; + fs << "nodeId" << (int)nid; + // msvc++ 2010 with opencv 2.3.1 does not allow FileStorage::operator<< + // with vectors of unsigned int + fs << "features" << "[" + << *(const std::vector*)(&features) << "]"; + fs << "}"; + } + + + fs << "]"; // entry of DF + } + */ + + fs << "]"; // directIndex + + fs << "}"; // database +} + +// -------------------------------------------------------------------------- + +void Database::load(const std::string &filename) +{ + cv::FileStorage fs(filename.c_str(), cv::FileStorage::READ); + if(!fs.isOpened()) throw std::string("Could not open file ") + filename; + + Vocabulary voc; + voc.readFromFile(std::string("voc_") + filename); + setVocabulary(voc); + + load(fs); +} + +// -------------------------------------------------------------------------- + +void Database::load(const cv::FileStorage &fs, + const std::string &name) +{ + // load voc first + // subclasses must instantiate m_voc before calling this ::load + //if(!m_voc) m_voc = new Vocabulary; + + //m_voc->load(fs); + + // load database now + clear(); // resizes inverted file + + cv::FileNode fdb = fs[name]; + + m_nentries = (int)fdb["nEntries"]; + m_use_di = (int)fdb["usingDI"] != 0; + m_dilevels = (int)fdb["diLevels"]; + + cv::FileNode fn = fdb["invertedIndex"]; + for(WordId wid = 0; wid < fn.size(); ++wid) + { + cv::FileNode fw = fn[wid]; + + for(unsigned int i = 0; i < fw.size(); ++i) + { + EntryId eid = (int)fw[i]["imageId"]; + WordValue v = fw[i]["weight"]; + + m_ifile[wid].push_back(IFPair(eid, v)); + } + } + + /* + if(m_use_di) + { + fn = fdb["directIndex"]; + + m_dfile.resize(fn.size()); + assert(m_nentries == (int)fn.size()); + + FeatureVector::iterator dit; + for(EntryId eid = 0; eid < fn.size(); ++eid) + { + cv::FileNode fe = fn[eid]; + + m_dfile[eid].clear(); + for(unsigned int i = 0; i < fe.size(); ++i) + { + NodeId nid = (int)fe[i]["nodeId"]; + + dit = m_dfile[eid].insert(m_dfile[eid].end(), + make_pair(nid, std::vector() )); + + // this failed to compile with some opencv versions (2.3.1) + //fe[i]["features"] >> dit->second; + + // this was ok until OpenCV 2.4.1 + //std::vector aux; + //fe[i]["features"] >> aux; // OpenCV < 2.4.1 + //dit->second.resize(aux.size()); + //std::copy(aux.begin(), aux.end(), dit->second.begin()); + + cv::FileNode ff = fe[i]["features"][0]; + dit->second.reserve(ff.size()); + + cv::FileNodeIterator ffit; + for(ffit = ff.begin(); ffit != ff.end(); ++ffit) + { + dit->second.push_back((int)*ffit); + } + } + } // for each entry + } // if use_id + */ +} + +/* +std::ostream& operator<<(std::ostream &os, + const Database &db) +{ + os << "Database: Entries = " << db.size() << ", " + "Using direct index = " << (db.usingDirectIndex() ? "yes" : "no"); + + if(db.usingDirectIndex()) + os << ", Direct index levels = " << db.getDirectIndexLevels(); + + os << ". " << *db.getVocabulary(); + return os; +} +*/ + +} diff --git a/src/Database.h b/src/Database.h new file mode 100644 index 0000000..5cf7177 --- /dev/null +++ b/src/Database.h @@ -0,0 +1,364 @@ +/** + * File: Database.h + * Date: March 2011 + * Modified By Rafael Muñoz in 2016 + * Author: Dorian Galvez-Lopez + * Description: database of images + * License: see the LICENSE.txt file + * + */ + +#ifndef __D_T_DATABASE__ +#define __D_T_DATABASE__ + +#include +#include +#include +#include +#include +#include + +#include "fbow.h" +#include "QueryResults.h" +#include "exports.h" + +namespace fbow { + +typedef unsigned int EntryId; + +/// Id of words +typedef unsigned int WordId; + +/// Value of a word +typedef double WordValue; + + + +// For query functions +static int MIN_COMMON_WORDS = 5; + + /// Database +class FBOW_API Database +{ +public: + + /** + * Creates an empty database without vocabulary + * @param use_di a direct index is used to store feature indexes + * @param di_levels levels to go up the vocabulary tree to select the + * node id to store in the direct index when adding images + */ + explicit Database(bool use_di = true, int di_levels = 0); + + /** + * Creates a database with the given vocabulary + * @param T class inherited from Vocabulary + * @param voc vocabulary + * @param use_di a direct index is used to store feature indexes + * @param di_levels levels to go up the vocabulary tree to select the + * node id to store in the direct index when adding images + */ + + explicit Database(const Vocabulary &voc, bool use_di = true, + int di_levels = 0); + + /** + * Copy constructor. Copies the vocabulary too + * @param db object to copy + */ + Database(const Database &db); + + /** + * Creates the database from a file + * @param filename + */ + Database(const std::string &filename); + + /** + * Creates the database from a file + * @param filename + */ + Database(const char *filename); + + /** + * Destructor + */ + virtual ~Database(void); + + /** + * Copies the given database and its vocabulary + * @param db database to copy + */ + Database& operator=( + const Database &db); + + /** + * Sets the vocabulary to use and clears the content of the database. + * @param T class inherited from Vocabulary + * @param voc vocabulary to copy + */ + inline void setVocabulary(const Vocabulary &voc); + + /** + * Sets the vocabulary to use and the direct index parameters, and clears + * the content of the database + * @param T class inherited from Vocabulary + * @param voc vocabulary to copy + * @param use_di a direct index is used to store feature indexes + * @param di_levels levels to go up the vocabulary tree to select the + * node id to store in the direct index when adding images + */ + + void setVocabulary(const Vocabulary& voc, bool use_di, int di_levels = 0); + + /** + * Returns a pointer to the vocabulary used + * @return vocabulary + */ + inline const Vocabulary* getVocabulary() const; + + /** + * Allocates some memory for the direct and inverted indexes + * @param nd number of expected image entries in the database + * @param ni number of expected words per image + * @note Use 0 to ignore a parameter + */ + void allocate(int nd = 0, int ni = 0); + + /** + * Adds an entry to the database and returns its index + * @param features features of the new entry + * @param bowvec if given, the bow vector of these features is returned + * @param fvec if given, the vector of nodes and feature indexes is returned + * @return id of new entry + */ + EntryId add(const cv::Mat &features); + /** + * Adds an entry to the database and returns its index + * @param features features of the new entry, one per row + * @param bowvec if given, the bow vector of these features is returned + * @param fvec if given, the vector of nodes and feature indexes is returned + * @return id of new entry + */ + //EntryId add(const cv::Mat &features, + // fBow *bowvec = NULL, FeatureVector *fvec = NULL); + + /** + * Adss an entry to the database and returns its index + * @param vec bow vector + * @param fec feature vector to add the entry. Only necessary if using the + * direct index + * @return id of new entry + */ + EntryId add(const fBow &vec//, + //const FeatureVector &fec = FeatureVector() + ); + + /** + * Empties the database + */ + inline void clear(); + + /** + * Returns the number of entries in the database + * @return number of entries in the database + */ + unsigned int size() const{ return m_nentries;} + + + /** + * Checks if the direct index is being used + * @return true iff using direct index + */ + bool usingDirectIndex() const{ return m_use_di;} + + /** + * Returns the di levels when using direct index + * @return di levels + */ + int getDirectIndexLevels() const{ return m_dilevels;} + + /** + * Queries the database with some features + * @param features query features + * @param ret (out) query results + * @param max_results number of results to return. <= 0 means all + * @param max_id only entries with id <= max_id are returned in ret. + * < 0 means all + */ + //void query(const std::vector &features, QueryResults &ret, + // int max_results = 1, int max_id = -1) const; + /** + * Queries the database with some features + * @param features query features,one per row + * @param ret (out) query results + * @param max_results number of results to return. <= 0 means all + * @param max_id only entries with id <= max_id are returned in ret. + * < 0 means all + */ + void query(const cv::Mat &features, QueryResults &ret, + int max_results = 1, int max_id = -1) const; + + /** + * Queries the database with a vector + * @param vec bow vector already normalized + * @param ret results + * @param max_results number of results to return. <= 0 means all + * @param max_id only entries with id <= max_id are returned in ret. + * < 0 means all + */ + void query(const fBow &vec, QueryResults &ret, + int max_results = 1, int max_id = -1) const; + + /** + * Returns the a feature vector associated with a database entry + * @param id entry id (must be < size()) + * @return const reference to map of nodes and their associated features in + * the given entry + */ + //const FeatureVector& retrieveFeatures(EntryId id) const; + + /** + * Stores the database in a file + * @param filename + */ + void save(const std::string &filename) const; + + /** + * Loads the database from a file + * @param filename + */ + void load(const std::string &filename); + + /** + * Stores the database in the given file storage structure + * @param fs + * @param name node name + */ + virtual void save(cv::FileStorage &fs, + const std::string &name = "database") const; + + /** + * Loads the database from the given file storage structure + * @param fs + * @param name node name + */ + virtual void load(const cv::FileStorage &fs, + const std::string &name = "database"); + + // -------------------------------------------------------------------------- + + /** + * Writes printable information of the database + * @param os stream to write to + * @param db + */ + +// FBOW_API friend std::ostream& operator<<(std::ostream &os, +// const Database &db); + + + +protected: + + /// Query with L1 scoring + void queryL1(const fBow &vec, QueryResults &ret, + int max_results, int max_id) const; + + /// Query with L2 scoring + void queryL2(const fBow &vec, QueryResults &ret, + int max_results, int max_id) const; + + /// Query with Chi square scoring + void queryChiSquare(const fBow &vec, QueryResults &ret, + int max_results, int max_id) const; + + /// Query with Bhattacharyya scoring + void queryBhattacharyya(const fBow &vec, QueryResults &ret, + int max_results, int max_id) const; + + /// Query with KL divergence scoring + void queryKL(const fBow &vec, QueryResults &ret, + int max_results, int max_id) const; + + /// Query with dot product scoring + void queryDotProduct(const fBow &vec, QueryResults &ret, + int max_results, int max_id) const; + +protected: + + /* Inverted file declaration */ + + /// Item of IFRow + struct IFPair + { + /// Entry id + EntryId entry_id; + + /// Word weight in this entry + WordValue word_weight; + + /** + * Creates an empty pair + */ + IFPair(){} + + /** + * Creates an inverted file pair + * @param eid entry id + * @param wv word weight + */ + IFPair(EntryId eid, WordValue wv): entry_id(eid), word_weight(wv) {} + + /** + * Compares the entry ids + * @param eid + * @return true iff this entry id is the same as eid + */ + inline bool operator==(EntryId eid) const { return entry_id == eid; } + }; + + /// Row of InvertedFile + typedef std::list IFRow; + // IFRows are sorted in ascending entry_id order + + /// Inverted index + typedef std::vector InvertedFile; + // InvertedFile[word_id] --> inverted file of that word + + /* Direct file declaration */ + + /// Direct index + // typedef std::vector DirectFile; + // DirectFile[entry_id] --> [ directentry, ... ] + +protected: + + /// Associated vocabulary + Vocabulary *m_voc; + + /// Flag to use direct index + bool m_use_di; + + /// Levels to go up the vocabulary tree to select nodes to store + /// in the direct index + int m_dilevels; + + /// Inverted file (must have size() == |words|) + InvertedFile m_ifile; + + /// Direct file (resized for allocation) + //DirectFile m_dfile; + + /// Number of valid entries in m_dfile + int m_nentries; + +}; + + + +// -------------------------------------------------------------------------- + +} // namespace fbow + +#endif diff --git a/src/QueryResults.cpp b/src/QueryResults.cpp new file mode 100644 index 0000000..61ef646 --- /dev/null +++ b/src/QueryResults.cpp @@ -0,0 +1,63 @@ +/** + * File: QueryResults.cpp + * Date: March, November 2011 + * Author: Dorian Galvez-Lopez + * Description: structure to store results of database queries + * License: see the LICENSE.txt file + * + */ + +#include +#include +#include "QueryResults.h" + +using namespace std; + +namespace fbow +{ + +// --------------------------------------------------------------------------- + +ostream & operator<<(ostream& os, const Result& ret ) +{ + os << ""; + return os; +} + +// --------------------------------------------------------------------------- + +ostream & operator<<(ostream& os, const QueryResults& ret ) +{ + if(ret.size() == 1) + os << "1 result:" << endl; + else + os << ret.size() << " results:" << endl; + + QueryResults::const_iterator rit; + for(rit = ret.begin(); rit != ret.end(); ++rit) + { + os << *rit; + if(rit + 1 != ret.end()) os << endl; + } + return os; +} + +// --------------------------------------------------------------------------- + +void QueryResults::saveM(const std::string &filename) const +{ + fstream f(filename.c_str(), ios::out); + + QueryResults::const_iterator qit; + for(qit = begin(); qit != end(); ++qit) + { + f << qit->Id << " " << qit->Score << endl; + } + + f.close(); +} + +// --------------------------------------------------------------------------- + +} // namespace fbow + diff --git a/src/QueryResults.h b/src/QueryResults.h new file mode 100644 index 0000000..26d5402 --- /dev/null +++ b/src/QueryResults.h @@ -0,0 +1,205 @@ +/** + * File: QueryResults.h + * Date: March, November 2011 + * Author: Dorian Galvez-Lopez + * Description: structure to store results of database queries + * License: see the LICENSE.txt file + * + */ + +#ifndef __D_T_QUERY_RESULTS__ +#define __D_T_QUERY_RESULTS__ + +#include +#include "exports.h" +namespace fbow { + +/// Id of entries of the database +typedef unsigned int EntryId; + +/// Single result of a query +class FBOW_API Result +{ +public: + + /// Entry id + EntryId Id; + + /// Score obtained + double Score; + + /// debug + int nWords; // words in common + // !!! this is filled only by Bhatt score! + // (and for BCMatching, BCThresholding then) + + double bhatScore, chiScore; + /// debug + + // only done by ChiSq and BCThresholding + double sumCommonVi; + double sumCommonWi; + double expectedChiScore; + /// debug + + /** + * Empty constructors + */ + inline Result(){} + + /** + * Creates a result with the given data + * @param _id entry id + * @param _score score + */ + inline Result(EntryId _id, double _score): Id(_id), Score(_score){} + + /** + * Compares the scores of two results + * @return true iff this.score < r.score + */ + inline bool operator<(const Result &r) const + { + return this->Score < r.Score; + } + + /** + * Compares the scores of two results + * @return true iff this.score > r.score + */ + inline bool operator>(const Result &r) const + { + return this->Score > r.Score; + } + + /** + * Compares the entry id of the result + * @return true iff this.id == id + */ + inline bool operator==(EntryId id) const + { + return this->Id == id; + } + + /** + * Compares the score of this entry with a given one + * @param s score to compare with + * @return true iff this score < s + */ + inline bool operator<(double s) const + { + return this->Score < s; + } + + /** + * Compares the score of this entry with a given one + * @param s score to compare with + * @return true iff this score > s + */ + inline bool operator>(double s) const + { + return this->Score > s; + } + + /** + * Compares the score of two results + * @param a + * @param b + * @return true iff a.Score > b.Score + */ + static inline bool gt(const Result &a, const Result &b) + { + return a.Score > b.Score; + } + + /** + * Compares the scores of two results + * @return true iff a.Score > b.Score + */ + inline static bool ge(const Result &a, const Result &b) + { + return a.Score > b.Score; + } + + /** + * Returns true iff a.Score >= b.Score + * @param a + * @param b + * @return true iff a.Score >= b.Score + */ + static inline bool geq(const Result &a, const Result &b) + { + return a.Score >= b.Score; + } + + /** + * Returns true iff a.Score >= s + * @param a + * @param s + * @return true iff a.Score >= s + */ + static inline bool geqv(const Result &a, double s) + { + return a.Score >= s; + } + + + /** + * Returns true iff a.Id < b.Id + * @param a + * @param b + * @return true iff a.Id < b.Id + */ + static inline bool ltId(const Result &a, const Result &b) + { + return a.Id < b.Id; + } + + /** + * Prints a string version of the result + * @param os ostream + * @param ret Result to print + */ + friend std::ostream & operator<<(std::ostream& os, const Result& ret ); +}; + +/// Multiple results from a query +class QueryResults: public std::vector +{ +public: + + /** + * Multiplies all the scores in the vector by factor + * @param factor + */ + inline void scaleScores(double factor); + + /** + * Prints a string version of the results + * @param os ostream + * @param ret QueryResults to print + */ + FBOW_API friend std::ostream & operator<<(std::ostream& os, const QueryResults& ret ); + + /** + * Saves a matlab file with the results + * @param filename + */ + void saveM(const std::string &filename) const; + +}; + +// -------------------------------------------------------------------------- + +inline void QueryResults::scaleScores(double factor) +{ + for(QueryResults::iterator qit = begin(); qit != end(); ++qit) + qit->Score *= factor; +} + +// -------------------------------------------------------------------------- + +} // namespace fbow + +#endif + diff --git a/src/fbow.cpp b/src/fbow.cpp index 4f574f9..0234d3d 100644 --- a/src/fbow.cpp +++ b/src/fbow.cpp @@ -7,6 +7,18 @@ namespace fbow{ +Vocabulary::Vocabulary(const Vocabulary& voc){ + *this = voc; +} + +Vocabulary& Vocabulary::operator = (const Vocabulary &voc){ + this->clear(); + this->_params = voc._params; + _data=(char*)AlignedAlloc(_params._aligment,_params._total_size); + memcpy(this->_data, voc._data, _params._total_size); + this->cpu_info = voc.cpu_info; + return *this; +} Vocabulary::~Vocabulary(){ if (_data!=0) AlignedFree( _data); diff --git a/src/fbow.h b/src/fbow.h index 7ebd44c..dbd44b7 100644 --- a/src/fbow.h +++ b/src/fbow.h @@ -56,6 +56,7 @@ class FBOW_API Vocabulary { + static inline void * AlignedAlloc(int __alignment,int size){ assert(__alignment<256); @@ -82,7 +83,10 @@ class FBOW_API Vocabulary friend class VocabularyCreator; public: - ~Vocabulary(); + Vocabulary(){}; + ~Vocabulary(); + Vocabulary(const Vocabulary& voc); + Vocabulary& operator = (const Vocabulary &voc); //transform the features stored as rows in the returned BagOfWords fBow transform(const cv::Mat &features); @@ -106,6 +110,10 @@ class FBOW_API Vocabulary bool isValid()const{return _data!=0;} //total number of blocks size_t size()const{return _params._nblocks;} + //total size of data + size_t totalSize()const{ + return _params._total_size; + } //removes all data void clear(); //returns a hash value idinfying the vocabulary diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 82bed12..de9105d 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -8,4 +8,5 @@ ADD_EXECUTABLE(fbow_create_voc_step1 fbow_create_voc_step1.cpp) #ADD_EXECUTABLE(fbow_create_voc_step1_opencv fbow_create_voc_step1_opencv.cpp) ADD_EXECUTABLE(fbow_transform fbow_transform.cpp) ADD_EXECUTABLE(image_matching image_matching.cpp) +ADD_EXECUTABLE(fbow_demo fbow_demo.cpp) INSTALL(TARGETS fbow_create_voc_step0 fbow_create_voc_step1 fbow_transform RUNTIME DESTINATION bin) diff --git a/utils/fbow_demo.cpp b/utils/fbow_demo.cpp new file mode 100644 index 0000000..24c2307 --- /dev/null +++ b/utils/fbow_demo.cpp @@ -0,0 +1,202 @@ +#include +#include + +// fbow +#include "vocabulary_creator.h" +#include "fbow.h" +#include "Database.h" + +// OpenCV +#include +#include +#include +#ifdef USE_CONTRIB +#include +#include +#endif + +using namespace fbow; +using namespace std; + +//command line parser +class CmdLineParser{int argc; char **argv; public: CmdLineParser(int _argc,char **_argv):argc(_argc),argv(_argv){} bool operator[] ( string param ) {int idx=-1; for ( int i=0; i readImagePaths(int argc,char **argv,int start){ + vector paths; + for(int i=start;i loadFeatures( std::vector path_to_images,string descriptor="") throw (std::exception){ + //select detector + cv::Ptr fdetector; + if (descriptor=="orb") fdetector=cv::ORB::create(); + else if (descriptor=="brisk") fdetector=cv::BRISK::create(); +#ifdef OPENCV_VERSION_3 + else if (descriptor=="akaze") fdetector=cv::AKAZE::create(); +#endif +#ifdef USE_CONTRIB + else if(descriptor=="surf" ) fdetector=cv::xfeatures2d::SURF::create(400, 4, 2, EXTENDED_SURF); +#endif + + else throw std::runtime_error("Invalid descriptor"); + assert(!descriptor.empty()); + vector features; + + + cout << "Extracting features..." << endl; + for(size_t i = 0; i < path_to_images.size(); ++i) + { + vector keypoints; + cv::Mat descriptors; + cout<<"reading image: "<detectAndCompute(image, cv::Mat(), keypoints, descriptors); + features.push_back(descriptors); + cout<<"done detecting features"< &features) +{ + // branching factor and depth levels + const int k = 9; + const int L = 3; + //const WeightingType weight = TF_IDF; + //const ScoringType score = L1_NORM; + const int nThreads = 1; + + srand(0); + fbow::VocabularyCreator voc_creator; + fbow::Vocabulary voc; + cout << "Creating a " << k << "^" << L << " vocabulary..." << endl; + voc_creator.create(voc, features, "orb", fbow::VocabularyCreator::Params(k, L, nThreads)); + + // lets do something with this vocabulary + cout << "Matching images against themselves (0 low, 1 high): " << endl; + fBow v1, v2; + for(size_t i = 0; i < features.size(); i++) + { + v1 = voc.transform(features[i]); + for(size_t j = 0; j < features.size(); j++) + { + v2 = voc.transform(features[j]); + + double score = fBow::score(v1, v2); + cout << "Image " << i << " vs Image " << j << ": " << score << endl; + } + } + + // save the vocabulary to disk + cout << endl << "Saving vocabulary..." << endl; + voc.saveToFile("small_voc.yml.gz"); + cout << "Done" << endl; + + return; +} + +void testDatabase(const vector &features) +{ + cout << "Creating a small database..." << endl; + + // load the vocabulary from disk + fbow::Vocabulary voc; + voc.readFromFile("small_voc.yml.gz"); + + Database db(voc, false, 0); // false = do not use direct index + // db creates a copy of the vocabulary, we may get rid of "voc" now + + // add images to the database + for(size_t i = 0; i < features.size(); i++) + db.add(features[i]); + + cout << "database created!" << endl; + + // and query the database + cout << "Querying the database: " << endl; + + QueryResults ret; + for(size_t i = 0; i < features.size(); i++) + { + db.query(features[i], ret, 4); + cout << "Searching for Image " << i << ". " << ret << endl; + } + + cout << endl; + + cout << "Saving database..." << endl; + db.save("small_db.yml.gz"); + cout << "database saved!" << endl; +} + +void testLoadedDatabase(const vector &features){ + + // once saved, we can load it again + cout << "Retrieving database once again..." << endl; + Database db2("small_db.yml.gz"); + cout << "... loaded! endl" << std::endl; + //cout << "... done! This is: " << endl << db2 << endl; + + QueryResults ret; + for (size_t i = 0; i < features.size(); i++) + { + db2.query(features[i], ret, 1); + + // ret[0] is always the same image in this case, because we added it to the + // database. ret[1] is the second best match. + + QueryResults::iterator qit = ret.begin(); + cout << "Matching Image " << i << "-> id=" << qit->Id << " score=" << qit->Score << endl; + } + + +} + + + +// ---------------------------------------------------------------------------- + +int main(int argc,char **argv) +{ + + try{ + CmdLineParser cml(argc,argv); + if (cml["-h"] || argc<=2){ + cerr<<"Usage: descriptor_name image0 image1 ... \n\t descriptors:brisk,surf,orb ,akaze(only if using opencv 3)"< features= loadFeatures(images,descriptor); + testVocCreation(features); + + testDatabase(features); + + testLoadedDatabase(features); + + }catch(std::exception &ex){ + cerr<