diff --git a/documentation/database.man b/documentation/database.man index 00b0f9df..fb4941e0 100644 --- a/documentation/database.man +++ b/documentation/database.man @@ -95,6 +95,14 @@ Sets the maximum allowable angle between stars in a Tetra pattern to \fImax\fP ( .SH OTHER OPTIONS +.TP +\fB--swap-integer-endianness\fP +If true, generate databases with all integer values having opposite endianness than the generating machine. It will not be possible to use the generated databases on the system they were generated on. + +.TP +\fB--swap-float-endianness\fP +If true, generate databases with all floating point values having opposite endianness than the generating machine. It will not be possible to use the generated databases on the system they were generated on. + .TP \fB--output\fP \fIoutput-path\fP The file to output the database to. Defaults to stdout. diff --git a/src/attitude-estimators.cpp b/src/attitude-estimators.cpp index 1e80f692..34b45a05 100644 --- a/src/attitude-estimators.cpp +++ b/src/attitude-estimators.cpp @@ -125,7 +125,7 @@ float QuestCharPoly(float x, float a, float b, float c, float d, float s) {retur float QuestCharPolyPrime(float x, float a, float b, float c) {return 4*pow(x,3) - 2*(a+b)*x - c;} /** - * Approximates roots of a real function using the Newton-Raphson algorithm + * Approximates roots of a real function using the Newton-Raphson algorithm * @see https://www.geeksforgeeks.org/program-for-newton-raphson-method/ */ float QuestEigenvalueEstimator(float guess, float a, float b, float c, float d, float s) { diff --git a/src/attitude-utils.cpp b/src/attitude-utils.cpp index c9e397cb..4757e466 100644 --- a/src/attitude-utils.cpp +++ b/src/attitude-utils.cpp @@ -4,6 +4,8 @@ #include #include +#include "serialize-helpers.hpp" + namespace lost { /// Multiply two quaternions using the usual definition of quaternion multiplication (effectively composes rotations) @@ -460,25 +462,19 @@ bool Attitude::IsKnown() const { } } -/// The length that a Vec3 will take up when serialized -long SerializeLengthVec3() { - return sizeof(float)*3; +/// Serialize a Vec3 to buffer. Takes up space according to SerializeLengthVec3 +void SerializeVec3(SerializeContext *ser, const Vec3 &vec) { + SerializePrimitive(ser, vec.x); + SerializePrimitive(ser, vec.y); + SerializePrimitive(ser, vec.z); } -/// Serialize a Vec3 to buffer. Takes up space according to SerializeLengthVec3 -void SerializeVec3(const Vec3 &vec, unsigned char *buffer) { - float *fBuffer = (float *)buffer; - *fBuffer++ = vec.x; - *fBuffer++ = vec.y; - *fBuffer = vec.z; -} - -Vec3 DeserializeVec3(const unsigned char *buffer) { - Vec3 result; - const float *fBuffer = (float *)buffer; - result.x = *fBuffer++; - result.y = *fBuffer++; - result.z = *fBuffer; +Vec3 DeserializeVec3(DeserializeContext *des) { + Vec3 result = { + DeserializePrimitive(des), + DeserializePrimitive(des), + DeserializePrimitive(des), + }; return result; } diff --git a/src/attitude-utils.hpp b/src/attitude-utils.hpp index 7e108ec3..dd7ee358 100644 --- a/src/attitude-utils.hpp +++ b/src/attitude-utils.hpp @@ -7,6 +7,8 @@ #include // iota #include +#include "serialize-helpers.hpp" + namespace lost { // At first, I wanted to have two separate Attitude classes, one storing Euler angles and converting @@ -78,9 +80,8 @@ class Mat3 { extern const Mat3 kIdentityMat3; -long SerializeLengthVec3(); -void SerializeVec3(const Vec3 &, unsigned char *); -Vec3 DeserializeVec3(const unsigned char *); +void SerializeVec3(SerializeContext *, const Vec3 &); +Vec3 DeserializeVec3(DeserializeContext *des); float Distance(const Vec2 &, const Vec2 &); float Distance(const Vec3 &, const Vec3 &); diff --git a/src/database-options.hpp b/src/database-options.hpp index ab35565c..4224531f 100644 --- a/src/database-options.hpp +++ b/src/database-options.hpp @@ -2,13 +2,15 @@ #include -LOST_CLI_OPTION("min-mag" , float , minMag , 100 , atof(optarg) , kNoDefaultArgument) -LOST_CLI_OPTION("max-stars" , int , maxStars , 10000 , atoi(optarg) , kNoDefaultArgument) -LOST_CLI_OPTION("min-separation" , float , minSeparation , 0.05 , atof(optarg) , kNoDefaultArgument) -LOST_CLI_OPTION("kvector" , bool , kvector , false , atobool(optarg), true) -LOST_CLI_OPTION("kvector-min-distance" , float , kvectorMinDistance , 0.5 , atof(optarg) , kNoDefaultArgument) -LOST_CLI_OPTION("kvector-max-distance" , float , kvectorMaxDistance , 15 , atof(optarg) , kNoDefaultArgument) -LOST_CLI_OPTION("kvector-distance-bins", long , kvectorNumDistanceBins, 10000 , atol(optarg) , kNoDefaultArgument) -LOST_CLI_OPTION("tetra" , bool , tetra , false , atobool(optarg), true) -LOST_CLI_OPTION("tetra-max-angle" , float , tetraMaxAngle , 12 , atof(optarg) , kNoDefaultArgument) -LOST_CLI_OPTION("output" , std::string, outputPath , "-" , optarg , kNoDefaultArgument) +LOST_CLI_OPTION("min-mag" , float , minMag , 100 , atof(optarg) , kNoDefaultArgument) +LOST_CLI_OPTION("max-stars" , int , maxStars , 10000 , atoi(optarg) , kNoDefaultArgument) +LOST_CLI_OPTION("min-separation" , float , minSeparation , 0.08 , atof(optarg) , kNoDefaultArgument) +LOST_CLI_OPTION("kvector" , bool , kvector , false , atobool(optarg), true) +LOST_CLI_OPTION("kvector-min-distance" , float , kvectorMinDistance , 0.5 , atof(optarg) , kNoDefaultArgument) +LOST_CLI_OPTION("kvector-max-distance" , float , kvectorMaxDistance , 15 , atof(optarg) , kNoDefaultArgument) +LOST_CLI_OPTION("kvector-distance-bins" , long , kvectorNumDistanceBins, 10000 , atol(optarg) , kNoDefaultArgument) +LOST_CLI_OPTION("swap-integer-endianness", bool , swapIntegerEndianness , false , atobool(optarg), true) +LOST_CLI_OPTION("swap-float-endianness" , bool , swapFloatEndianness , false , atobool(optarg), true) +LOST_CLI_OPTION("tetra" , bool , tetra , false , atobool(optarg), true) +LOST_CLI_OPTION("tetra-max-angle" , float , tetraMaxAngle , 12 , atof(optarg) , kNoDefaultArgument) +LOST_CLI_OPTION("output" , std::string, outputPath , "-" , optarg , kNoDefaultArgument) diff --git a/src/databases.cpp b/src/databases.cpp index 5108b98e..1c0ab7a1 100644 --- a/src/databases.cpp +++ b/src/databases.cpp @@ -15,10 +15,13 @@ #include #include "attitude-utils.hpp" +#include "serialize-helpers.hpp" #include "star-utils.hpp" namespace lost { +const int32_t PairDistanceKVectorDatabase::kMagicValue = 0x2536f009; + struct KVectorPair { int16_t index1; int16_t index2; @@ -44,11 +47,6 @@ bool CompareKVectorPairs(const KVectorPair &p1, const KVectorPair &p2) { | | | min+i*(max-min)/numBins | */ -/// The number of bytes that a kvector index will take up whe serialized -long SerializeLengthKVectorIndex(long numBins) { - return 4+sizeof(float)+sizeof(float)+4+4*(numBins+1); -} - // apparently there's no easy way to accept an iterator argument. Hate java all you want, but at // least it can do that! // https://stackoverflow.com/questions/5054087/declare-a-function-accepting-generic-iterator or @@ -65,8 +63,8 @@ long SerializeLengthKVectorIndex(long numBins) { * @param numBins the number of "bins" the KVector should use. A higher number makes query results "tighter" but takes up more disk space. Usually should be set somewhat smaller than (max-min) divided by the "width" of the typical query. * @param buffer[out] index is written here. */ -void SerializeKVectorIndex(const std::vector &values, float min, float max, long numBins, unsigned char *buffer) { - std::vector kVector(numBins+1); // numBins = length, all elements zero +void SerializeKVectorIndex(SerializeContext *ser, const std::vector &values, float min, float max, long numBins) { + std::vector kVector(numBins+1); // We store sums before and after each bin float binWidth = (max - min) / numBins; // generate the k-vector part @@ -98,45 +96,31 @@ void SerializeKVectorIndex(const std::vector &values, float min, float ma lastBinVal = bin; } - unsigned char *bufferStart = buffer; // metadata fields - *(int32_t *)buffer = values.size(); - buffer += sizeof(int32_t); - *(float *)buffer = min; - buffer += sizeof(float); - *(float *)buffer = max; - buffer += sizeof(float); - *(int32_t *)buffer = numBins; - buffer += sizeof(int32_t); + SerializePrimitive(ser, values.size()); + SerializePrimitive(ser, min); + SerializePrimitive(ser, max); + SerializePrimitive(ser, numBins); // kvector index field - // you could probably do this with memcpy instead, but the explicit loop is necessary for endian - // concerns? TODO endianness for (const int32_t &bin : kVector) { - *(int32_t *)buffer = bin; - buffer += sizeof(int32_t); + SerializePrimitive(ser, bin); } - - // verify length - assert(buffer - bufferStart == SerializeLengthKVectorIndex(numBins)); } /// Construct from serialized buffer. -KVectorIndex::KVectorIndex(const unsigned char *buffer) { - numValues = *(int32_t *)buffer; - buffer += sizeof(int32_t); - min = *(float *)buffer; - buffer += sizeof(float); - max = *(float *)buffer; - buffer += sizeof(float); - numBins = *(int32_t *)buffer; - buffer += sizeof(int32_t); +KVectorIndex::KVectorIndex(DeserializeContext *des) { + + numValues = DeserializePrimitive(des); + min = DeserializePrimitive(des); + max = DeserializePrimitive(des); + numBins = DeserializePrimitive(des); assert(min >= 0.0f); assert(max > min); binWidth = (max - min) / numBins; - bins = (const int32_t *)buffer; + bins = DeserializeArray(des, numBins+1); } /** @@ -209,20 +193,11 @@ std::vector CatalogToPairDistances(const Catalog &catalog, float mi return result; } -long SerializeLengthPairDistanceKVector(long numPairs, long numBins) { - return SerializeLengthKVectorIndex(numBins) + 2*sizeof(int16_t)*numPairs; -} - -/// Number of bytes that a serialized KVectorDatabase will take up -long SerializeLengthPairDistanceKVector(const Catalog &catalog, float minDistance, float maxDistance, long numBins) { - return SerializeLengthPairDistanceKVector(CatalogToPairDistances(catalog, minDistance, maxDistance).size(), numBins); -} - /** * Serialize a pair-distance KVector into buffer. * Use SerializeLengthPairDistanceKVector to determine how large the buffer needs to be. See command line documentation for other options. */ -void SerializePairDistanceKVector(const Catalog &catalog, float minDistance, float maxDistance, long numBins, unsigned char *buffer) { +void SerializePairDistanceKVector(SerializeContext *ser, const Catalog &catalog, float minDistance, float maxDistance, long numBins) { std::vector kVector(numBins+1); // numBins = length, all elements zero std::vector pairs = CatalogToPairDistances(catalog, minDistance, maxDistance); @@ -234,22 +209,14 @@ void SerializePairDistanceKVector(const Catalog &catalog, float minDistance, flo distances.push_back(pair.distance); } - unsigned char *bufferStart = buffer; - // index field - SerializeKVectorIndex(distances, minDistance, maxDistance, numBins, buffer); - buffer += SerializeLengthKVectorIndex(numBins); + SerializeKVectorIndex(ser, distances, minDistance, maxDistance, numBins); // bulk pairs field for (const KVectorPair &pair : pairs) { - *(int16_t *)buffer = pair.index1; - buffer += sizeof(int16_t); - *(int16_t *)buffer = pair.index2; - buffer += sizeof(int16_t); + SerializePrimitive(ser, pair.index1); + SerializePrimitive(ser, pair.index2); } - - // verify length - assert(buffer - bufferStart == SerializeLengthPairDistanceKVector(pairs.size(), numBins)); } std::pair, std::vector> TetraPreparePattCat(const Catalog &catalog, @@ -376,18 +343,15 @@ std::pair, std::vector> TetraPreparePattCat(cons return std::pair, std::vector>{finalCatIndices, pattStarIndices}; } -TetraDatabase::TetraDatabase(const unsigned char *buffer) : buffer_(buffer) { - maxAngle_ = *(float*)buffer; - buffer += sizeof(float); - catalogSize_ = *(int32_t *)buffer; - // std::cout << "Tetra database, size= " << catalogSize_ << std::endl; +TetraDatabase::TetraDatabase(DeserializeContext *des) { + // maxAngle_ = *(float*)buffer; + // buffer += sizeof(float); + // catalogSize_ = *(int32_t *)buffer; + maxAngle_ = DeserializePrimitive(des); + catalogSize_ = DeserializePrimitive(des); } -float TetraDatabase::MaxAngle() const { return maxAngle_; } - -int TetraDatabase::PattCatSize() const { return catalogSize_; } - -std::vector TetraDatabase::GetPattern(int index) const { +TetraPatt TetraDatabase::GetPattern(int index) const { std::vector res; const unsigned char *p = buffer_ + headerSize; for (int i = 0; i < 4; i++) { @@ -403,12 +367,12 @@ uint16_t TetraDatabase::GetTrueCatInd(int tetraInd) const { return *((uint16_t *)p + tetraInd); } -std::vector TetraDatabase::GetPatternMatches(int index) const { - std::vector res; +std::vector TetraDatabase::GetPatternMatches(int index) const { + std::vector res; for (int c = 0;; c++) { int i = (index + c * c) % PattCatSize(); - Pattern tableRow = GetPattern(i); + TetraPatt tableRow = GetPattern(i); if (tableRow[0] == 0 && tableRow[1] == 0) { break; @@ -420,12 +384,10 @@ std::vector TetraDatabase::GetPatternMatches(int index) const { } /// Create the database from a serialized buffer. -PairDistanceKVectorDatabase::PairDistanceKVectorDatabase(const unsigned char *buffer) - : index(KVectorIndex(buffer)) { +PairDistanceKVectorDatabase::PairDistanceKVectorDatabase(DeserializeContext *des) + : index(KVectorIndex(des)) { - // TODO: errors? (not even sure what i meant by this comment anymore) - buffer += SerializeLengthKVectorIndex(index.NumBins()); - pairs = (const int16_t *)buffer; + pairs = DeserializeArray(des, 2*index.NumValues()); } /// Return the value in the range [low,high] which is closest to num @@ -509,40 +471,42 @@ std::vector PairDistanceKVectorDatabase::StarDistances(int16_t star, ///////////////////// Tetra database ////////////////////// -static long SerializeTetraHelper(const Catalog &catalog, float maxFovDeg, unsigned char *buffer, +void SerializeTetraDatabase(SerializeContext *ser, Catalog &catalog, float maxFovDeg, const std::vector &pattStarIndices, - const std::vector &catIndices, bool ser) { - const float maxFov = DegToRad(maxFovDeg); + const std::vector &catIndices) { + const float maxFovRad = DegToRad(maxFovDeg); - // TODO: pattBins here and numPattBins in TetraStarIDAlgorithm::numPattBins should be the same - // Otherwise we break things + // TODO: these are hardcoded values + // pattBins here and numPattBins in TetraStarIDAlgorithm::numPattBins must be the same const int pattBins = 50; const int tempBins = 4; + const int pattSize = 4; - // Preprocessed star table for Tetra Catalog tetraCatalog; - for (int ind : catIndices) { + for (int ind : catIndices){ tetraCatalog.push_back(catalog[ind]); } - struct CmpSpatialHash { - bool operator()(const Vec3 &vec1, const Vec3 &vec2) const { - if (vec1.x != vec2.x) return (vec1.x < vec2.x); - if (vec1.y != vec2.y) return (vec1.y < vec2.y); - return vec1.z < vec2.z; - }; + auto spatialHash = [](const Vec3 &vec){ + std::hash hasher; + return hasher(vec.x) ^ hasher(vec.y) ^ hasher(vec.z); }; - - std::map, CmpSpatialHash> tempCoarseSkyMap; - - for (const int starID : pattStarIndices) { - Vec3 v(tetraCatalog[starID].spatial); - Vec3 hash{floor((v.x + 1) * tempBins), floor((v.y + 1) * tempBins), - floor((v.z + 1) * tempBins)}; + auto spatialEquals = [](const Vec3 &vec1, const Vec3 &vec2){ + return vec1.x==vec2.x && vec1.y==vec2.y && vec1.z==vec2.z; + }; + std::unordered_map, decltype(spatialHash), decltype(spatialEquals)> + tempCoarseSkyMap(8, spatialHash, spatialEquals); + + for (int starID : pattStarIndices){ + Vec3 v{tetraCatalog[starID].spatial}; + Vec3 hash{ + floor((v.x+1) * tempBins), + floor((v.y+1) * tempBins), + floor((v.z+1) * tempBins) + }; tempCoarseSkyMap[hash].push_back(starID); } - // Return a list of stars that are nearby auto tempGetNearbyStars = [&tempCoarseSkyMap, &tetraCatalog](const Vec3 &vec, float radius) { std::vector components{vec.x, vec.y, vec.z}; @@ -557,11 +521,8 @@ static long SerializeTetraHelper(const Catalog &catalog, float maxFovDeg, unsign range.push_back(hi); hcSpace.push_back(range); } - - // hashcode space has 3 ranges, one for each of [x, y, z] - - std::vector nearbyStarIDs; - + // Hashcode space has 3 ranges, one for each of [x, y, z] + std::vector nearbyStarIDs; // TODO: typedef this for (int a = hcSpace[0][0]; a < hcSpace[0][1]; a++) { for (int b = hcSpace[1][0]; b < hcSpace[1][1]; b++) { for (int c = hcSpace[2][0]; c < hcSpace[2][1]; c++) { @@ -578,38 +539,28 @@ static long SerializeTetraHelper(const Catalog &catalog, float maxFovDeg, unsign } } } - return nearbyStarIDs; }; - - const int pattSize = 4; - typedef std::array Pattern; + using Pattern = std::array; std::vector pattList; Pattern patt{0, 0, 0, 0}; // Construct all possible patterns - for (int ind = 0; ind < (int)pattStarIndices.size(); ind++) { - uint16_t firstStarID = pattStarIndices[ind]; - patt[0] = firstStarID; + for(int firstStarInd : pattStarIndices){ + patt[0] = firstStarInd; - Vec3 v(tetraCatalog[firstStarID].spatial); + Vec3 v{tetraCatalog[firstStarInd].spatial}; Vec3 hashCode{floor((v.x + 1) * tempBins), floor((v.y + 1) * tempBins), floor((v.z + 1) * tempBins)}; - // Remove star i from its sky map partition + // Remove star=firstStarInd from its sky map partition auto removeIt = std::find(tempCoarseSkyMap[hashCode].begin(), - tempCoarseSkyMap[hashCode].end(), firstStarID); - - // TODO: an assertion is more appropriate - if (removeIt == tempCoarseSkyMap[hashCode].end()) { - std::cerr << "Fatal error with hashing" << std::endl; - exit(EXIT_FAILURE); - } + tempCoarseSkyMap[hashCode].end(), firstStarInd); + assert(removeIt != tempCoarseSkyMap[hashCode].end()); tempCoarseSkyMap[hashCode].erase(removeIt); - auto nearbyStars = tempGetNearbyStars(v, maxFov); + auto nearbyStars = tempGetNearbyStars(v, maxFovRad); const int n = nearbyStars.size(); - for (int i = 0; i < n; i++) { for (int j = i + 1; j < n; j++) { for (int k = j + 1; k < n; k++) { @@ -618,18 +569,16 @@ static long SerializeTetraHelper(const Catalog &catalog, float maxFovDeg, unsign patt[3] = nearbyStars[k]; bool pattFits = true; - // TODO: why do we need to check this? - // If we do nearbyStars, we already guarantee that this spatial is at most maxFOV away from all in nearby? - // No guarantee that all stars in nearBy are at most maxFOV from each other though for (int pair1 = 0; pair1 < pattSize; pair1++) { for (int pair2 = pair1 + 1; pair2 < pattSize; pair2++) { float dotProd = tetraCatalog[patt[pair1]].spatial * tetraCatalog[patt[pair2]].spatial; - if (dotProd <= std::cos(maxFov)) { + if (dotProd <= std::cos(maxFovRad)) { pattFits = false; break; } } + if (!pattFits) break; } if (pattFits) { @@ -640,42 +589,39 @@ static long SerializeTetraHelper(const Catalog &catalog, float maxFovDeg, unsign } } - std::cerr << "Found " << pattList.size() << " patterns" << std::endl; - - // Load factor of 0.5 - long long pattCatalogLength = 2 * (int)pattList.size(); - std::vector pattCatalog(pattCatalogLength); - + std::cerr << "Tetra found " << pattList.size() << " patterns" << std::endl; + // Ensure load factor just < 0.5 + long long pattCatalogLen = 2 * (int)pattList.size() + 1; + std::vector pattCatalog(pattCatalogLen); for (Pattern patt : pattList) { - std::vector pattEdgeLengths; + std::vector pattEdgeLengths; for (int i = 0; i < pattSize; i++) { CatalogStar star1 = tetraCatalog[patt[i]]; for (int j = i + 1; j < pattSize; j++) { // calculate distance between vectors CatalogStar star2 = tetraCatalog[patt[j]]; - float edgeLen = (star2.spatial - star1.spatial).Magnitude(); + double edgeLen = (star2.spatial - star1.spatial).Magnitude(); pattEdgeLengths.push_back(edgeLen); } } std::sort(pattEdgeLengths.begin(), pattEdgeLengths.end()); - - float pattLargestEdge = pattEdgeLengths[(int)(pattEdgeLengths.size() - 1)]; - std::vector pattEdgeRatios; - for (int i = 0; i < (int)pattEdgeLengths.size() - 1; - i++) { // size()-1, since we ignore the largest edge + double pattLargestEdge = pattEdgeLengths.back(); + std::vector pattEdgeRatios; + // Skip last edge since ratio is just 1 + for (int i = 0; i < pattEdgeLengths.size() - 1; i++) { pattEdgeRatios.push_back(pattEdgeLengths[i] / pattLargestEdge); } std::vector key; - for (float edgeRatio : pattEdgeRatios) { + for (double edgeRatio : pattEdgeRatios) { key.push_back(int(edgeRatio * pattBins)); } - int hashIndex = KeyToIndex(key, pattBins, pattCatalogLength); + int hashIndex = KeyToIndex(key, pattBins, pattCatalogLen); long long offset = 0; // Quadratic probing to find next available bucket for element with key=hashIndex while (true) { - int index = int(hashIndex + std::pow(offset, 2)) % pattCatalogLength; + int index = int(hashIndex + std::pow(offset, 2)) % pattCatalogLen; offset++; if (pattCatalog[index][0] == 0 && pattCatalog[index][1] == 0) { pattCatalog[index] = patt; @@ -683,55 +629,28 @@ static long SerializeTetraHelper(const Catalog &catalog, float maxFovDeg, unsign } } } - - // Done with everything, write to buffer - // See TetraDatabase for layout of db - - if (ser) { - *((float *)buffer) = maxFovDeg; - buffer += sizeof(float); - *((int32_t *)buffer) = (int)pattCatalog.size(); - buffer += sizeof(int32_t); - - for (Pattern patt : pattCatalog) { - for (int i = 0; i < pattSize; i++) { - *((uint16_t *)buffer) = patt[i]; - buffer += sizeof(uint16_t); - } - } - - for (uint16_t ind : catIndices) { - *((uint16_t *)buffer) = ind; - buffer += sizeof(uint16_t); + SerializePrimitive(ser, maxFovDeg); + SerializePrimitive(ser, pattCatalog.size()); + for (Pattern patt : pattCatalog) { + for (int i = 0; i < pattSize; i++) { + SerializePrimitive(ser, patt[i]); } - - return -1; - } else { - return sizeof(float) + sizeof(int32_t) + sizeof(uint16_t) * pattCatalog.size() * pattSize + - catIndices.size() * sizeof(uint16_t); } -} - -long SerializeLengthTetraDatabase(const Catalog &cat, float maxFov, - const std::vector &pattStarIndices, - const std::vector &catIndices) { - return SerializeTetraHelper(cat, maxFov, nullptr, pattStarIndices, catIndices, false); -} - -void SerializeTetraDatabase(const Catalog &cat, float maxFov, unsigned char *buffer, - const std::vector &pattStarIndices, - const std::vector &catIndices) { - SerializeTetraHelper(cat, maxFov, buffer, pattStarIndices, catIndices, true); + for (uint16_t ind : catIndices) { + SerializePrimitive(ser, ind); + } } /** MultiDatabase memory layout: - | size | name | description | - |----------------+-------------------+---------------------------------------------------------| - | 8*maxDatabases | table of contents | each 8-byte entry is the 4-byte magic value followed by | - | | | a 4-byte index into the bulk where that db begins | - | Large | databases | the database contents | + | size | name | description | + |------+----------------+---------------------------------------------| + | 4 | magicValue | unique database identifier | + | 4 | databaseLength | length in bytes (32-bit unsigned) | + | n | database | the entire database. 8-byte aligned | + | ... | ... | More databases (each has value, length, db) | + | 4 | caboose | 4 null bytes indicate the end | */ /** @@ -741,61 +660,36 @@ void SerializeTetraDatabase(const Catalog &cat, float maxFov, unsigned char *buf * @return Returns a pointer to the start of the database type indicated by the magic value, null if not found */ const unsigned char *MultiDatabase::SubDatabasePointer(int32_t magicValue) const { - long databaseIndex = -1; - int32_t *toc = (int32_t *)buffer; - for (int i = 0; i < kMultiDatabaseMaxDatabases; i++) { - int32_t curMagicValue = *toc; - toc++; + DeserializeContext desValue(buffer); + DeserializeContext *des = &desValue; // just for naming consistency with how we use `des` elsewhere + + assert(magicValue != 0); + while (true) { + int32_t curMagicValue = DeserializePrimitive(des); + if (curMagicValue == 0) { + return nullptr; + } + uint32_t dbLength = DeserializePrimitive(des); + assert(dbLength > 0); + DeserializePadding(des); // align to an 8-byte boundary + const unsigned char *curSubDatabasePointer = DeserializeArray(des, dbLength); if (curMagicValue == magicValue) { - databaseIndex = *toc; - break; + return curSubDatabasePointer; } - toc++; - } - // the database was not found - if (databaseIndex < 0) { - return NULL; } - - return buffer+kMultiDatabaseTocLength+databaseIndex; + // shouldn't ever make it here. Compiler should remove this assertion as unreachable. + assert(false); } -/** - * Add a database to a MultiDatabase - * @param magicValue A value unique to this type of database which is used to extract it out of the database later. - * @param length The number of bytes to allocate for this database. - * @return Pointer to the start of the space allocated for said database. Return null if full (too many databases). - */ -unsigned char *MultiDatabaseBuilder::AddSubDatabase(int32_t magicValue, long length) { - // find unused spot in toc and take it! - int32_t *toc = (int32_t *)buffer; - bool foundSpot = false; - for (int i = 0; i < kMultiDatabaseMaxDatabases; i++) { - if (*toc == 0) { - *toc = magicValue; - toc++; - *toc = bulkLength; - foundSpot = true; - break; - } - // skip the entry - toc += 2; - } - - // database is full - if (!foundSpot) { - return NULL; +void SerializeMultiDatabase(SerializeContext *ser, + const MultiDatabaseDescriptor &dbs) { + for (const MultiDatabaseEntry &multiDbEntry : dbs) { + SerializePrimitive(ser, multiDbEntry.magicValue); + SerializePrimitive(ser, multiDbEntry.bytes.size()); + SerializePadding(ser); + std::copy(multiDbEntry.bytes.cbegin(), multiDbEntry.bytes.cend(), std::back_inserter(ser->buffer)); } - - buffer = (unsigned char *)realloc(buffer, kMultiDatabaseTocLength+bulkLength+length); - // just past the end of the last database - unsigned char *result = buffer+kMultiDatabaseTocLength+bulkLength; - bulkLength += length; - return result; -} - -MultiDatabaseBuilder::~MultiDatabaseBuilder() { - free(buffer); + SerializePrimitive(ser, 0); // caboose } } diff --git a/src/databases.hpp b/src/databases.hpp index 57d54999..ffb9d981 100644 --- a/src/databases.hpp +++ b/src/databases.hpp @@ -8,6 +8,7 @@ #include #include "star-utils.hpp" +#include "serialize-helpers.hpp" namespace lost { @@ -15,13 +16,13 @@ const int32_t kCatalogMagicValue = 0xF9A283BC; /** * A data structure enabling constant-time range queries into fixed numerical data. - * + * * @note Not an instantiable database on its own -- used in other databases */ // TODO: QueryConservative, QueryExact, QueryTrapezoidal? class KVectorIndex { public: - explicit KVectorIndex(const unsigned char *); + explicit KVectorIndex(DeserializeContext *des); long QueryLiberal(float minQueryDistance, float maxQueryDistance, long *upperIndex) const; @@ -43,8 +44,7 @@ class KVectorIndex { const int32_t *bins; }; -long SerializeLengthPairDistanceKVector(const Catalog &, float minDistance, float maxDistance, long numBins); -void SerializePairDistanceKVector(const Catalog &, float minDistance, float maxDistance, long numBins, unsigned char *buffer); +void SerializePairDistanceKVector(SerializeContext *, const Catalog &, float minDistance, float maxDistance, long numBins); /** * A database storing distances between pairs of stars. @@ -53,7 +53,7 @@ void SerializePairDistanceKVector(const Catalog &, float minDistance, float maxD */ class PairDistanceKVectorDatabase { public: - explicit PairDistanceKVectorDatabase(const unsigned char *databaseBytes); + explicit PairDistanceKVectorDatabase(DeserializeContext *des); const int16_t *FindPairsLiberal(float min, float max, const int16_t **end) const; const int16_t *FindPairsExact(const Catalog &, float min, float max, const int16_t **end) const; @@ -67,9 +67,10 @@ class PairDistanceKVectorDatabase { long NumPairs() const; /// Magic value to use when storing inside a MultiDatabase - static const int32_t kMagicValue = 0x2536f009; - - private: + static const int32_t kMagicValue; // 0x2536f009 + // apparently you're supposed to not actually put the value of the static variables here, but + // rather in a cpp implementation file. +private: KVectorIndex index; // TODO: endianness const int16_t *pairs; @@ -84,61 +85,55 @@ Pre-processing for Tetra star-id algorithm std::pair, std::vector> TetraPreparePattCat(const Catalog &, const float maxFovDeg); -// long SerializeTetraDatabase(const Catalog &, float maxFov, unsigned char *buffer, -// const std::vector &, -// const std::vector &, bool ser); - -long SerializeLengthTetraDatabase(const Catalog &, float maxFov, - const std::vector &, - const std::vector &); - -void SerializeTetraDatabase(const Catalog &, float maxFov, unsigned char *buffer, - const std::vector &, - const std::vector &); - -typedef std::vector Pattern; - -/* -Layout: +void SerializeTetraDatabase(SerializeContext *, Catalog &, float maxFovDeg, + const std::vector& pattStarIndices, + const std::vector& catIndices); -/////////////////// Header ////////////////////////////// -- Max FOV (float) -- Number of patterns in pattern catalog (int) -//////////////////////////////////////////////////////// -- All patterns (number of patterns * pattSize=4 * sizeof(uint16_t)) -- List of Catalog indices to use for Tetra star ID algo -///////////////////////////////////////////////////////// +/// Tetra star pattern = vector of 4 star IDs +using TetraPatt = std::vector; -*/ +/** + * A database storing Tetra star patterns + * TODO: implement something to prevent cycling in quadratic probe + * (or guarantee load factor < 0.5) + * + * Layout: + * | size (bytes) | name | description | + * |-----------------+--------------+-------------------------------------------------------------| | + * | sizeof float | maxFov | max angle (degrees) allowed between any 2 stars | + * | | | in the same pattern | + * | 8 | pattCatSize | number of rows in pattern catalog | + * | 4*pattCatSize*2 | pattCat | hash table for Tetra star patternss | + * | 2*tetraCatSize | tetraStarCat | list of catalog indices to use for Tetra star-id algo | + */ class TetraDatabase { public: - explicit TetraDatabase(const unsigned char *buffer); + explicit TetraDatabase(DeserializeContext *des); - // Get max angle (in degrees) allowed between stars in the same pattern - float MaxAngle() const; + /// Max angle (in degrees) allowed between stars in the same pattern + float MaxAngle() const {return maxAngle_;} /// Number of rows in pattern catalog - // With load factor of 0.5, size = number of patterns * 2 - int PattCatSize() const; + // With load factor of just under 0.5, size = numPatterns*2 + 1 + int PattCatSize() const {return catalogSize_;} - // Get the 4-tuple pattern at row=index, 0-based - Pattern GetPattern(int index) const; + /// Get the 4-tuple pattern at row=index, 0-based + TetraPatt GetPattern(int index) const; - /* - Returns a list of rows (4-tuples) from the Pattern Catalog - Start from index and does quadratic probing - */ - std::vector GetPatternMatches(int index) const; + /// Returns a list of patterns from the pattern catalog + // Starts from index and does quadratic probing + // We assume that collisions mean the pattern stored there COULD be a match + std::vector GetPatternMatches(int index) const; uint16_t GetTrueCatInd(int tetraIndex) const; // TODO: should probably have a field describing number of indices for future updates to db /// Magic value to use when storing inside a MultiDatabase static const int32_t kMagicValue = 0xDEADBEEF; - static const int headerSize = sizeof(float) + sizeof(int32_t); + static const int headerSize = sizeof(float) + sizeof(uint64_t); private: - const unsigned char *buffer_; + // const unsigned char *buffer_; float maxAngle_; uint32_t catalogSize_; }; @@ -160,11 +155,6 @@ class TetraDatabase { // int16_t *triples; // }; -/// maximum number of databases in a MultiDatabase -const int kMultiDatabaseMaxDatabases = 64; -/// The size of the table of contents in a multidatabase (stores subdatabase locations) -const long kMultiDatabaseTocLength = 8*kMultiDatabaseMaxDatabases; - /** * A database that contains multiple databases * This is almost always the database that is actually passed to star-id algorithms in the real world, since you'll want to store at least the catalog plus one specific database. @@ -179,27 +169,19 @@ class MultiDatabase { const unsigned char *buffer; }; -/// Class for easily creating a MultiDatabase -class MultiDatabaseBuilder { +class MultiDatabaseEntry { public: - MultiDatabaseBuilder() - : buffer((unsigned char *)calloc(1, kMultiDatabaseTocLength)), bulkLength(0) { }; - ~MultiDatabaseBuilder(); + MultiDatabaseEntry(int32_t magicValue, std::vector bytes) // I wonder if making `bytes` a reference would avoid making two copies, or maybe it would be worse by preventing copy-elision + : magicValue(magicValue), bytes(bytes) { } - unsigned char *AddSubDatabase(int32_t magicValue, long length); - - /// When done adding databases, use this to get the buffer you should write to disk. - unsigned char *Buffer() { return buffer; }; - /// The length of the buffer returned by Buffer - long BufferLength() { return kMultiDatabaseTocLength+bulkLength; }; -private: - // Throughout LOST, most dynamic memory is managed with `new` and `delete` to make it easier to - // use unique pointers. Here, however, we use realloc, so C-style memory management. - unsigned char *buffer; - // how many bytes are presently allocated for databases (excluding map) - long bulkLength; + int32_t magicValue; + std::vector bytes; }; +using MultiDatabaseDescriptor = std::vector; + +void SerializeMultiDatabase(SerializeContext *, const MultiDatabaseDescriptor &dbs); + } #endif diff --git a/src/io.cpp b/src/io.cpp index c1ee7bf2..2b505746 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -265,52 +265,53 @@ typedef StarIdAlgorithm *(*StarIdAlgorithmFactory)(); typedef AttitudeEstimationAlgorithm *(*AttitudeEstimationAlgorithmFactory)(); -/// Add a pair-distance KVector database to the given builder. -void BuildPairDistanceKVectorDatabase(MultiDatabaseBuilder *builder, const Catalog &catalog, float minDistance, float maxDistance, long numBins) { - // TODO: calculating the length of the vector duplicates a lot of the work, slowing down - // database generation - long length = SerializeLengthPairDistanceKVector(catalog, minDistance, maxDistance, numBins); - unsigned char *buffer = builder->AddSubDatabase(PairDistanceKVectorDatabase::kMagicValue, length); - if (buffer == NULL) { - std::cerr << "No room for another database." << std::endl; - } - SerializePairDistanceKVector(catalog, minDistance, maxDistance, numBins, buffer); - - // TODO: also parse it and print out some stats before returning +SerializeContext serFromDbValues(const DatabaseOptions &values) { + return SerializeContext(values.swapIntegerEndianness, values.swapFloatEndianness); } -void BuildTetraDatabase(MultiDatabaseBuilder *builder, const Catalog &catalog, float maxAngle, - const std::vector &pattStars, - const std::vector &catIndices) { - // long length = SerializeTetraDatabase(catalog, maxAngle, nullptr, pattStars, catIndices, false); - long length = SerializeLengthTetraDatabase(catalog, maxAngle, pattStars, catIndices); - unsigned char *buffer = builder->AddSubDatabase(TetraDatabase::kMagicValue, length); - if (buffer == nullptr) { - std::cerr << "Error: No room for Tetra database" << std::endl; - } - // SerializeTetraDatabase(catalog, maxAngle, buffer, pattStars, catIndices, true); - SerializeTetraDatabase(catalog, maxAngle, buffer, pattStars, catIndices); -} +// void BuildTetraDatabase(MultiDatabaseBuilder *builder, const Catalog &catalog, float maxAngle, +// const std::vector &pattStars, +// const std::vector &catIndices) { +// // long length = SerializeTetraDatabase(catalog, maxAngle, nullptr, pattStars, catIndices, false); +// long length = SerializeLengthTetraDatabase(catalog, maxAngle, pattStars, catIndices); +// unsigned char *buffer = builder->AddSubDatabase(TetraDatabase::kMagicValue, length); +// if (buffer == nullptr) { +// std::cerr << "Error: No room for Tetra database" << std::endl; +// } +// // SerializeTetraDatabase(catalog, maxAngle, buffer, pattStars, catIndices, true); +// SerializeTetraDatabase(catalog, maxAngle, buffer, pattStars, catIndices); +// } -void GenerateTetraDatabases(MultiDatabaseBuilder *builder, const Catalog &catalog, - const DatabaseOptions &values, const std::vector &pattStars, - const std::vector &catIndices) { - float maxAngle = values.tetraMaxAngle; - BuildTetraDatabase(builder, catalog, maxAngle, pattStars, catIndices); -} -/// Generate and add databases to the given multidatabase builder according to the command line options in `values` -void GenerateDatabases(MultiDatabaseBuilder *builder, const Catalog &catalog, const DatabaseOptions &values) { +// void GenerateTetraDatabases(MultiDatabaseBuilder *builder, const Catalog &catalog, +// const DatabaseOptions &values, const std::vector &pattStars, +// const std::vector &catIndices) { +// float maxAngle = values.tetraMaxAngle; +// BuildTetraDatabase(builder, catalog, maxAngle, pattStars, catIndices); +// } +// /// Generate and add databases to the given multidatabase builder according to the command line options in `values` +// void GenerateDatabases(MultiDatabaseBuilder *builder, const Catalog &catalog, const DatabaseOptions &values) { + +MultiDatabaseDescriptor GenerateDatabases(const Catalog &catalog, const DatabaseOptions &values) { + MultiDatabaseDescriptor dbEntries; + + SerializeContext catalogSer = serFromDbValues(values); + // TODO decide why we have this inclMagnitude and inclName and if we should change that + SerializeCatalog(&catalogSer, catalog, false, true); + dbEntries.emplace_back(kCatalogMagicValue, catalogSer.buffer); if (values.kvector) { float minDistance = DegToRad(values.kvectorMinDistance); float maxDistance = DegToRad(values.kvectorMaxDistance); long numBins = values.kvectorNumDistanceBins; - BuildPairDistanceKVectorDatabase(builder, catalog, minDistance, maxDistance, numBins); + SerializeContext ser = serFromDbValues(values); + SerializePairDistanceKVector(&ser, catalog, minDistance, maxDistance, numBins); + dbEntries.emplace_back(PairDistanceKVectorDatabase::kMagicValue, ser.buffer); } else { std::cerr << "No database builder selected -- no database generated." << std::endl; exit(1); } + return dbEntries; } /// Print information about the camera in machine and human-readable form. @@ -938,7 +939,8 @@ PipelineOutput Pipeline::Go(const PipelineInput &input) { MultiDatabase multiDatabase(database.get()); const unsigned char *catalogBuffer = multiDatabase.SubDatabasePointer(kCatalogMagicValue); if (catalogBuffer != NULL) { - result.catalog = DeserializeCatalog(multiDatabase.SubDatabasePointer(kCatalogMagicValue), NULL, NULL); + DeserializeContext des(catalogBuffer); + result.catalog = DeserializeCatalog(&des, NULL, NULL); } else { std::cerr << "WARNING: That database does not include a catalog. Proceeding with the full catalog." << std::endl; result.catalog = input.GetCatalog(); @@ -1583,13 +1585,13 @@ static void PipelineComparatorAttitude(std::ostream &os, os << "attitude_error_rate " << fractionIncorrect << std::endl; } -static void PrintTimeStats(std::ostream &os, const std::string &prefix, const std::vector ×) { +static void PrintTimeStats(std::ostream &os, const std::string &prefix, const std::vector ×) { assert(times.size() > 0); // print average, min, max, and 95% max - long sum = 0; - long min = LONG_MAX; - long max = 0; + long long sum = 0; + long long min = LONG_MAX; + long long max = 0; for (int i = 0; i < (int)times.size(); i++) { assert(times[i] > 0); sum += times[i]; @@ -1597,14 +1599,14 @@ static void PrintTimeStats(std::ostream &os, const std::string &prefix, const st max = std::max(max, times[i]); } long average = sum / times.size(); - std::vector sortedTimes = times; + std::vector sortedTimes = times; std::sort(sortedTimes.begin(), sortedTimes.end()); // what really is the 95th percentile? Being conservative, we want to pick a value that at least // 95% of the times are less than. This means: (1) finding the number of times, (2) Finding // Math.ceil(0.95 * numTimes), and (3) subtracting 1 to get the index. int ninetyFiveIndex = (int)std::ceil(0.95 * times.size()) - 1; assert(ninetyFiveIndex >= 0); - long ninetyFifthPercentile = sortedTimes[ninetyFiveIndex]; + long long ninetyFifthPercentile = sortedTimes[ninetyFiveIndex]; os << prefix << "_average_ns " << average << std::endl; os << prefix << "_min_ns " << min << std::endl; @@ -1617,12 +1619,12 @@ static void PipelineComparatorPrintSpeed(std::ostream &os, const PipelineInputList &, const std::vector &actual, const PipelineOptions &) { - std::vector centroidingTimes; - std::vector starIdTimes; - std::vector attitudeTimes; - std::vector totalTimes; + std::vector centroidingTimes; + std::vector starIdTimes; + std::vector attitudeTimes; + std::vector totalTimes; for (int i = 0; i < (int)actual.size(); i++) { - long totalTime = 0; + long long totalTime = 0; if (actual[i].centroidingTimeNs > 0) { centroidingTimes.push_back(actual[i].centroidingTimeNs); totalTime += actual[i].centroidingTimeNs; diff --git a/src/io.hpp b/src/io.hpp index 26252054..29852d37 100644 --- a/src/io.hpp +++ b/src/io.hpp @@ -192,9 +192,9 @@ struct PipelineOutput { /// How many nanoseconds the centroiding stage of the pipeline took. Similarly for the other /// fields. If negative, the centroiding stage was not run. - long centroidingTimeNs = -1; - long starIdTimeNs = -1; - long attitudeEstimationTimeNs = -1; + long long centroidingTimeNs = -1; + long long starIdTimeNs = -1; + long long attitudeEstimationTimeNs = -1; /** * @brief The catalog that the indices in starIds refer to @@ -289,10 +289,11 @@ class DatabaseOptions { #undef LOST_CLI_OPTION }; -// unlike the other algorithm prompters, db builders aren't a -// typedef void (*DbBuilder)(MultiDatabaseBuilder &, const Catalog &); -void GenerateDatabases(MultiDatabaseBuilder *, const Catalog &, const DatabaseOptions &values); -// void PromptDatabases(MultiDatabaseBuilder &, const Catalog &); +SerializeContext serFromDbValues(const DatabaseOptions &values); + +/// Appropriately create descriptors for all requested databases according to command-line options. +/// @sa SerializeMultiDatabase +MultiDatabaseDescriptor GenerateDatabases(const Catalog &, const DatabaseOptions &values); // TODO: can we avoid the split? void GenerateTetraDatabases(MultiDatabaseBuilder *, const Catalog &, const DatabaseOptions &values, diff --git a/src/main.cpp b/src/main.cpp index 87cc3c96..4ae9980f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -29,36 +29,35 @@ static void DatabaseBuild(const DatabaseOptions &values) { Catalog narrowedCatalog = NarrowCatalog(CatalogRead(), (int) (values.minMag * 100), values.maxStars, DegToRad(values.minSeparation)); std::cerr << "Narrowed catalog has " << narrowedCatalog.size() << " stars." << std::endl; - MultiDatabaseBuilder builder; - // TODO: allow magnitude and weird - unsigned char - *catalogBuffer = - builder.AddSubDatabase(kCatalogMagicValue, SerializeLengthCatalog(narrowedCatalog, false, true)); - SerializeCatalog(narrowedCatalog, false, true, catalogBuffer); - - if (values.tetra) { - std::cerr << "Tetra max angle is: " << values.tetraMaxAngle << std::endl; - auto tetraStuff = TetraPreparePattCat(narrowedCatalog, values.tetraMaxAngle); - std::vector catIndices = tetraStuff.first; - std::vector pattStars = tetraStuff.second; - - std::cerr << "Tetra processed catalog has " << catIndices.size() << " stars." << std::endl; - std::cerr << "Number of pattern stars: " << pattStars.size() << std::endl; - - GenerateTetraDatabases(&builder, narrowedCatalog, values, pattStars, catIndices); - std::cerr << "Generated TETRA database with " << builder.BufferLength() << " bytes" - << std::endl; - } - // We should allow for multiple databases at the same tme - // Do NOT make this an if...else if...else - if (values.kvector) { - GenerateDatabases(&builder, narrowedCatalog, values); - std::cerr << "Generated kvector database with " << builder.BufferLength() << " bytes" - << std::endl; - } + MultiDatabaseDescriptor dbEntries = GenerateDatabases(narrowedCatalog, values); + SerializeContext ser = serFromDbValues(values); + SerializeMultiDatabase(&ser, dbEntries); + + // if (values.tetra) { + // std::cerr << "Tetra max angle is: " << values.tetraMaxAngle << std::endl; + // auto tetraStuff = TetraPreparePattCat(narrowedCatalog, values.tetraMaxAngle); + // std::vector catIndices = tetraStuff.first; + // std::vector pattStars = tetraStuff.second; + + // std::cerr << "Tetra processed catalog has " << catIndices.size() << " stars." << std::endl; + // std::cerr << "Number of pattern stars: " << pattStars.size() << std::endl; + + // GenerateTetraDatabases(&builder, narrowedCatalog, values, pattStars, catIndices); + // std::cerr << "Generated TETRA database with " << builder.BufferLength() << " bytes" + // << std::endl; + // } + // // We should allow for multiple databases at the same tme + // // Do NOT make this an if...else if...else + // if (values.kvector) { + // GenerateDatabases(&builder, narrowedCatalog, values); + // std::cerr << "Generated kvector database with " << builder.BufferLength() << " bytes" + // << std::endl; + // } + + std::cerr << "Generated database with " << ser.buffer.size() << " bytes" << std::endl; UserSpecifiedOutputStream pos = UserSpecifiedOutputStream(values.outputPath, true); - pos.Stream().write((char *) builder.Buffer(), builder.BufferLength()); + pos.Stream().write((char *) ser.buffer.data(), ser.buffer.size()); } diff --git a/src/serialize-helpers.hpp b/src/serialize-helpers.hpp new file mode 100644 index 00000000..13a0a3fb --- /dev/null +++ b/src/serialize-helpers.hpp @@ -0,0 +1,136 @@ +/** + * Helpers to serialize and deserialize arbitrary data types to disk + * + * The serialization and deserialization helpers here assume that (a) integers and floating point + * numbers are stored in the same format on the source and target systems except for endianness, and + * (b) the target system requires no greater than n-byte alignment for n-byte values (eg, an int32_t + * only needs 4-byte alignment, not 8-byte), and (c) the database itself will be aligned at a + * multiple of the largest size of any value stored within (because we only align values relative to + * the start of the database, so everything breaks if the database itself is misaligned). + * + * Generally, the "bulk" of any database will not be explicitly deserialized into a data structure + * in memory, but instead will just effectively memory-mapped, by just storing a pointer to a + * certain offset into the database. The serialization functions in this file can still be used to + * create such a "bulk" section, but deserialization must be handled manually. + */ + +#ifndef SERIALIZE_HELPERS_H +#define SERIALIZE_HELPERS_H + +#include + +#include +#include +#include + +namespace lost { + +class SerializeContext { +public: + SerializeContext(bool swapIntegerEndianness, bool swapFloatEndianness) + : swapIntegerEndianness(swapIntegerEndianness), swapFloatEndianness(swapFloatEndianness) { } + SerializeContext() : SerializeContext(false, false) { } + + bool swapIntegerEndianness; + bool swapFloatEndianness; + std::vector buffer; +}; + +class DeserializeContext { +public: + explicit DeserializeContext(const unsigned char *buffer) : buffer(buffer), cursor(buffer) { } + + size_t GetOffset() const { + return cursor - buffer; + } + + void MoveForward(size_t howMuch) { + cursor += howMuch; + } + + const unsigned char *GetCursor() { + return cursor; + } + +private: + const unsigned char *buffer; /// the start of the buffer + const unsigned char *cursor; /// the current location of the "read head" into the buffer +}; + +/// Unconditionally swap the endianness of a value (uses sizeof T). +template +void SwapEndianness(unsigned char *buffer) { + for (int i = 0; i < (int)(sizeof(T)/2); i++) { + std::swap(buffer[i], buffer[sizeof(T)-1-i]); + } +} + +/// Swap the endianness of a value if necessary. Uses +/// LOST_DATABASE_{SOURCE,TARGET}_INTEGER_ENDIANNESS to determine to switch all values but float and +/// double, which use LOST_DATABASE_{SOURCE,TARGET}_FLOAT_ENDIANNESS. +template +void SwapEndiannessIfNecessary(unsigned char *buffer, SerializeContext *ser) { + if (ser->swapIntegerEndianness) { + SwapEndianness(buffer); + } +} + +// template specializations + +template <> +inline void SwapEndiannessIfNecessary(unsigned char *buffer, SerializeContext *ser) { + if (ser->swapFloatEndianness) { + SwapEndianness(buffer); + } +} + +template <> +inline void SwapEndiannessIfNecessary(unsigned char *buffer, SerializeContext *ser) { + if (ser->swapFloatEndianness) { + SwapEndianness(buffer); + } +} + +/// Move the cursor forward past any padding that would appear before a value of type T +template +void DeserializePadding(DeserializeContext *des) { + des->MoveForward((sizeof(T) - des->GetOffset()%sizeof(T))%sizeof(T)); +} + +template +T DeserializePrimitive(DeserializeContext *des) { + DeserializePadding(des); + const T *result = (T *)des->GetCursor(); // endianness should have been taken care of during serialization + des->MoveForward(sizeof(T)); + return *result; +} + +/// return an array of items as a pointer. Will point into the buffer (mmap style). +template +const T *DeserializeArray(DeserializeContext *des, long arrLength) { + DeserializePadding(des); // Perhaps we should always 8-align arrays? Does that possibly make + // any offset calculation or accesses faster? + const T *result = (T *)des->GetCursor(); + des->MoveForward(sizeof(T)*arrLength); + return result; +} + +template +void SerializePadding(SerializeContext *ser) { + for (int i = ser->buffer.size(); i%sizeof(T) != 0; i++) { + ser->buffer.push_back(0); + } +} + +template +void SerializePrimitive(SerializeContext *ser, const T &val) { + unsigned char buf[sizeof(T)]; + memcpy(&buf, &val, sizeof(T)); + SwapEndiannessIfNecessary(buf, ser); + SerializePadding(ser); + std::copy(buf, buf+sizeof(T), std::back_inserter(ser->buffer)); +} + +} + +#endif diff --git a/src/star-id.cpp b/src/star-id.cpp index 422849b1..b08aaadc 100644 --- a/src/star-id.cpp +++ b/src/star-id.cpp @@ -356,7 +356,8 @@ StarIdentifiers GeometricVotingStarIdAlgorithm::Go( if (databaseBuffer == NULL) { return identified; } - PairDistanceKVectorDatabase vectorDatabase(databaseBuffer); + DeserializeContext des(databaseBuffer); + PairDistanceKVectorDatabase vectorDatabase(&des); for (int i = 0; i < (int)stars.size(); i++) { std::vector votes(catalog.size(), 0); @@ -898,7 +899,8 @@ StarIdentifiers PyramidStarIdAlgorithm::Go( std::cerr << "Not enough stars, or database missing." << std::endl; return identified; } - PairDistanceKVectorDatabase vectorDatabase(databaseBuffer); + DeserializeContext des(databaseBuffer); + PairDistanceKVectorDatabase vectorDatabase(&des); // smallest normal single-precision float is around 10^-38 so we should be all good. See // Analytic_Star_Pattern_Probability on the HSL wiki for details. diff --git a/src/star-utils.cpp b/src/star-utils.cpp index 850d67a6..304347f9 100644 --- a/src/star-utils.cpp +++ b/src/star-utils.cpp @@ -8,6 +8,8 @@ #include #include +#include "serialize-helpers.hpp" + namespace lost { // brightest star first @@ -75,18 +77,6 @@ Catalog::const_iterator FindNamedStar(const Catalog &catalog, int name) { return catalog.cend(); } -/// @sa SerializeCatalogStar -long SerializeLengthCatalogStar(bool inclMagnitude, bool inclName) { - long starSize = SerializeLengthVec3(); - if (inclMagnitude) { - starSize += sizeof(float); - } - if (inclName) { - starSize += sizeof(int16_t); - } - return starSize; -} - /** * Serialize a CatalogStar into a byte buffer. * Use SerializeLengthCatalogStar() to determine how many bytes to allocate in `buffer` @@ -94,20 +84,23 @@ long SerializeLengthCatalogStar(bool inclMagnitude, bool inclName) { * @param inclName Whether to include the (numerical) name of the star. * @param buffer[out] Where the serialized star is stored. */ +<<<<<<< HEAD // TODO: make inclusion of name/magnitude true by default? // Actually why give the option in the first place, algos like Tetra need this to work void SerializeCatalogStar(const CatalogStar &catalogStar, bool inclMagnitude, bool inclName, unsigned char *buffer) { SerializeVec3(catalogStar.spatial, buffer); buffer += SerializeLengthVec3(); +======= +void SerializeCatalogStar(SerializeContext *ser, const CatalogStar &catalogStar, bool inclMagnitude, bool inclName) { + SerializeVec3(ser, catalogStar.spatial); +>>>>>>> master if (inclMagnitude) { - *(float *)buffer = catalogStar.magnitude; - buffer += sizeof(float); + SerializePrimitive(ser, catalogStar.magnitude); } if (inclName) { // TODO: double check that bools aren't some special bitwise thing in C++ - *(int16_t *)buffer = catalogStar.name; - buffer += sizeof(int16_t); + SerializePrimitive(ser, catalogStar.name); } } @@ -116,85 +109,62 @@ void SerializeCatalogStar(const CatalogStar &catalogStar, bool inclMagnitude, bo * @warn The `inclMagnitude` and `inclName` parameters must be the same as passed to SerializeCatalogStar() * @sa SerializeCatalogStar */ -CatalogStar DeserializeCatalogStar(const unsigned char *buffer, bool inclMagnitude, bool inclName) { +CatalogStar DeserializeCatalogStar(DeserializeContext *des, bool inclMagnitude, bool inclName) { CatalogStar result; - result.spatial = DeserializeVec3(buffer); - buffer += SerializeLengthVec3(); + result.spatial = DeserializeVec3(des); if (inclMagnitude) { - result.magnitude = *(float *)buffer; - buffer += sizeof(float); + result.magnitude = DeserializePrimitive(des); } else { result.magnitude = -424242; // TODO, what to do about special values, since there's no good ones for ints. } if (inclName) { - result.name = *(int16_t *)buffer; - buffer += sizeof(int16_t); + result.name = DeserializePrimitive(des); } else { result.name = -1; } return result; } -/// @sa SerializeCatalog -long SerializeLengthCatalog(const Catalog &catalog, bool inclMagnitude, bool inclName) { - return sizeof(int16_t) + sizeof(int8_t) + catalog.size()*SerializeLengthCatalogStar(inclMagnitude, inclName); -} - /** * Serialize the catalog to `buffer`. * Use SerializeLengthCatalog() to determine how many bytes to allocate in `buffer` * @param inclMagnitude,inclName See SerializeCatalogStar() */ -void SerializeCatalog(const Catalog &catalog, bool inclMagnitude, bool inclName, unsigned char *buffer) { - unsigned char *bufferStart = buffer; - - // size - *(int16_t *)buffer = catalog.size(); - buffer += sizeof(int16_t); +void SerializeCatalog(SerializeContext *ser, const Catalog &catalog, bool inclMagnitude, bool inclName) { + SerializePrimitive(ser, catalog.size()); // flags int8_t flags = (inclMagnitude) | (inclName << 1); - *(int8_t *)buffer = flags; - buffer += sizeof(int8_t); + SerializePrimitive(ser, flags); - long catalogStarLength = SerializeLengthCatalogStar(inclMagnitude, inclName); for (const CatalogStar &catalogStar : catalog) { - SerializeCatalogStar(catalogStar, inclMagnitude, inclName, buffer); - buffer += catalogStarLength; + SerializeCatalogStar(ser, catalogStar, inclMagnitude, inclName); } - - assert(buffer-bufferStart == SerializeLengthCatalog(catalog, inclMagnitude, inclName)); } -// TODO (longer term): don't deserialize the catalog, store it on disk using the in-memory format so -// we can just copy it to memory then cast - /** * Deserialize a catalog. * @param[out] inclMagnitudeReturn,inclNameReturn Will store whether `inclMagnitude` and `inclNameReturn` were set in the corresponding SerializeCatalog() call. */ -Catalog DeserializeCatalog(const unsigned char *buffer, bool *inclMagnitudeReturn, bool *inclNameReturn) { +Catalog DeserializeCatalog(DeserializeContext *des, bool *inclMagnitudeReturn, bool *inclNameReturn) { bool inclName, inclMagnitude; Catalog result; - int16_t numStars = *(int16_t *)buffer; - buffer += sizeof(int16_t); + int16_t numStars = DeserializePrimitive(des); - int8_t flags = *(int8_t *)buffer; + int8_t flags = DeserializePrimitive(des); inclMagnitude = (flags) & 1; inclName = (flags>>1) & 1; + if (inclMagnitudeReturn != NULL) { *inclMagnitudeReturn = inclMagnitude; } if (inclNameReturn != NULL) { *inclNameReturn = inclName; } - buffer += sizeof(int8_t); - int catalogStarLength = SerializeLengthCatalogStar(inclMagnitude, inclName); for (int i = 0; i < numStars; i++) { - result.push_back(DeserializeCatalogStar(buffer, inclMagnitude, inclName)); - buffer += catalogStarLength; + result.push_back(DeserializeCatalogStar(des, inclMagnitude, inclName)); } return result; diff --git a/src/star-utils.hpp b/src/star-utils.hpp index a65cf0de..2fc1269e 100644 --- a/src/star-utils.hpp +++ b/src/star-utils.hpp @@ -5,6 +5,7 @@ #include #include "attitude-utils.hpp" +#include "serialize-helpers.hpp" namespace lost { @@ -101,10 +102,9 @@ typedef std::vector Catalog; typedef std::vector Stars; typedef std::vector StarIdentifiers; -long SerializeLengthCatalog(const Catalog &, bool inclMagnitude, bool inclName); -void SerializeCatalog(const Catalog &, bool inclMagnitude, bool inclName, unsigned char *buffer); +void SerializeCatalog(SerializeContext *, const Catalog &, bool inclMagnitude, bool inclName); // sets magnited and name to whether the catalog in the database contained magnitude and name -Catalog DeserializeCatalog(const unsigned char *buffer, bool *inclMagnitudeReturn, bool *inclNameReturn); +Catalog DeserializeCatalog(DeserializeContext *des, bool *inclMagnitudeReturn, bool *inclNameReturn); Catalog::const_iterator FindNamedStar(const Catalog &catalog, int name); /// returns some relative brightness measure, which is proportional to the total number of photons received from a star. diff --git a/test/identify-remaining-stars.cpp b/test/identify-remaining-stars.cpp index c36db261..20c2c468 100644 --- a/test/identify-remaining-stars.cpp +++ b/test/identify-remaining-stars.cpp @@ -2,6 +2,7 @@ #include +#include "databases.hpp" #include "star-id.hpp" #include "star-id-private.hpp" @@ -51,16 +52,17 @@ TEST_CASE("IRUnidentifiedCentroid obtuse angle", "[identify-remaining] [fast]") std::vector IdentifyThirdStarTest(const Catalog &catalog, int16_t catalogName1, int16_t catalogName2, float dist1, float dist2, float tolerance) { - unsigned char *dbBytes = BuildPairDistanceKVectorDatabase(integralCatalog, NULL, 0, M_PI, 1000); + SerializeContext ser; + SerializePairDistanceKVector(&ser, integralCatalog, 0, M_PI, 1000); + DeserializeContext des(ser.buffer.data()); auto cs1 = FindNamedStar(catalog, catalogName1); auto cs2 = FindNamedStar(catalog, catalogName2); - PairDistanceKVectorDatabase db(dbBytes); + PairDistanceKVectorDatabase db(&des); auto result = IdentifyThirdStar(db, catalog, cs1 - catalog.cbegin(), cs2 - catalog.cbegin(), dist1, dist2, tolerance); - delete[] dbBytes; return result; } @@ -163,13 +165,13 @@ TEST_CASE("IdentifyRemainingStars fuzz", "[identify-remaining] [fuzz]") { someFakeStarIds.push_back(fakeStarIdsCopy[i]); } - unsigned char *dbBytes = BuildPairDistanceKVectorDatabase(fakeCatalog, NULL, 0, M_PI, 1000); - PairDistanceKVectorDatabase db(dbBytes); + SerializeContext ser; + SerializePairDistanceKVector(&ser, fakeCatalog, 0, M_PI, 1000); + DeserializeContext des(ser.buffer.data()); + PairDistanceKVectorDatabase db(&des); int numIdentified = IdentifyRemainingStarsPairDistance(&someFakeStarIds, fakeCentroids, db, fakeCatalog, smolCamera, 1e-5); - delete[] dbBytes; - REQUIRE(numIdentified == numFakeStars - fakePatternSize); REQUIRE(AreStarIdentifiersEquivalent(fakeStarIds, someFakeStarIds)); } diff --git a/test/kvector.cpp b/test/kvector.cpp index e05b2b3d..40b05793 100644 --- a/test/kvector.cpp +++ b/test/kvector.cpp @@ -3,17 +3,19 @@ #include "databases.hpp" #include "io.hpp" #include "attitude-utils.hpp" +#include "serialize-helpers.hpp" #include "utils.hpp" using namespace lost; // NOLINT TEST_CASE("Kvector full database stuff", "[kvector]") { - long length; const Catalog &catalog = CatalogRead(); - unsigned char *dbBytes = BuildPairDistanceKVectorDatabase(catalog, &length, DegToRad(1.0), DegToRad(2.0), 100); - REQUIRE(length < 999999); - PairDistanceKVectorDatabase db(dbBytes); + std::vector dbBytes; + SerializeContext ser; + SerializePairDistanceKVector(&ser, catalog, DegToRad(1.0), DegToRad(2.0), 100); + DeserializeContext des(ser.buffer.data()); + PairDistanceKVectorDatabase db(&des); SECTION("basic consistency checks") { long lastNumReturnedPairs = 999999; @@ -47,16 +49,14 @@ TEST_CASE("Kvector full database stuff", "[kvector]") { } REQUIRE(totalReturnedPairs == db.NumPairs()); } - - delete[] dbBytes; } TEST_CASE("Tighter tolerance test", "[kvector]") { - long length; const Catalog &catalog = CatalogRead(); - unsigned char *dbBytes = BuildPairDistanceKVectorDatabase(catalog, &length, DegToRad(0.5), DegToRad(5.0), 1000); - REQUIRE(length < 999999); - PairDistanceKVectorDatabase db(dbBytes); + SerializeContext ser; + SerializePairDistanceKVector(&ser, catalog, DegToRad(0.5), DegToRad(5.0), 1000); + DeserializeContext des(ser.buffer.data()); + PairDistanceKVectorDatabase db(&des); // radius we'll request float delta = 0.0001; // radius we expect back: radius we request + width of a bin @@ -99,8 +99,6 @@ TEST_CASE("Tighter tolerance test", "[kvector]") { } CHECK(!outsideRangeReturned); } - - delete[] dbBytes; } TEST_CASE("3-star database, check exact results", "[kvector] [fast]") { @@ -109,8 +107,10 @@ TEST_CASE("3-star database, check exact results", "[kvector] [fast]") { CatalogStar(DegToRad(4), DegToRad(7), 2.0, 43), CatalogStar(DegToRad(2), DegToRad(6), 4.0, 44), }; - unsigned char *dbBytes = BuildPairDistanceKVectorDatabase(tripleCatalog, NULL, DegToRad(0.5), DegToRad(20.0), 1000); - PairDistanceKVectorDatabase db(dbBytes); + SerializeContext ser; + SerializePairDistanceKVector(&ser, tripleCatalog, DegToRad(0.5), DegToRad(20.0), 1000); + DeserializeContext des(ser.buffer.data()); + PairDistanceKVectorDatabase db(&des); REQUIRE(db.NumPairs() == 3); float distances[] = {0.038825754, 0.15707963, 0.177976474}; @@ -132,6 +132,4 @@ TEST_CASE("3-star database, check exact results", "[kvector] [fast]") { CHECK(AngleUnit(tripleCatalog[pairs[0]].spatial, tripleCatalog[pairs[1]].spatial) == Approx(distance).epsilon(1e-4)); } } - - delete[] dbBytes; } diff --git a/test/serialize.cpp b/test/serialize.cpp new file mode 100644 index 00000000..c8003875 --- /dev/null +++ b/test/serialize.cpp @@ -0,0 +1,99 @@ +// Tests for serialization and multidatabases + +#include + +#include + +#include "serialize-helpers.hpp" + +using namespace lost; // NOLINT + +TEST_CASE("Simple serialization, deserialization of primitives", "[fast] [serialize]") { + int64_t val64 = 27837492938; + float valFloat = 23.71728; + SerializeContext ser; + SerializePrimitive(&ser, val64); + SerializePrimitive(&ser, valFloat); + DeserializeContext des(ser.buffer.data()); + int64_t deserializedVal64 = DeserializePrimitive(&des); + float deserializedFloat = DeserializePrimitive(&des); + CHECK(val64 == deserializedVal64); + CHECK(valFloat == deserializedFloat); +} + +TEST_CASE("Endian-swapped serialization, deserialization of primitives", "[fast] [serialize]") { + int64_t val64 = 27837492938; + float valFloat = 23.71728; + SerializeContext ser1(true, true); + SerializePrimitive(&ser1, val64); + SerializePrimitive(&ser1, valFloat); + DeserializeContext des(ser1.buffer.data()); + int64_t deserializedVal64 = DeserializePrimitive(&des); + float deserializedValFloat = DeserializePrimitive(&des); + CHECK(val64 != deserializedVal64); + CHECK(valFloat != deserializedValFloat); + // but if we serialize it again, it should be back to normal! + + SerializeContext ser2(true, true); + SerializePrimitive(&ser2, deserializedVal64); + SerializePrimitive(&ser2, deserializedValFloat); + DeserializeContext des2(ser2.buffer.data()); + int64_t redeserializedVal64 = DeserializePrimitive(&des2); + float redeserializedValFloat = DeserializePrimitive(&des2); + CHECK(val64 == redeserializedVal64); + CHECK(valFloat == redeserializedValFloat); +} + +TEST_CASE("Endian-swapped floats only", "[fast] [serialize]") { + int64_t val64 = 27837492938; + float valFloat = 23.71728; + SerializeContext ser1(false, true); + SerializePrimitive(&ser1, val64); + SerializePrimitive(&ser1, valFloat); + DeserializeContext des(ser1.buffer.data()); + int64_t deserializedVal64 = DeserializePrimitive(&des); + float deserializedValFloat = DeserializePrimitive(&des); + CHECK(val64 == deserializedVal64); + CHECK(valFloat != deserializedValFloat); + + SerializeContext ser2(false, true); + SerializePrimitive(&ser2, deserializedValFloat); + DeserializeContext des2(ser2.buffer.data()); + float redeserializedValFloat = DeserializePrimitive(&des2); + CHECK(valFloat == redeserializedValFloat); +} + +TEST_CASE("Padding", "[fast] [serialize]") { + int8_t val8 = 23; + int32_t val32 = 1234567; + SerializeContext ser; + SerializePrimitive(&ser, val8); + SerializePrimitive(&ser, val32); + CHECK(ser.buffer.size() == 8); + CHECK(ser.buffer[0] == 23); + CHECK(ser.buffer[1] == 0); + CHECK(ser.buffer[2] == 0); + CHECK(ser.buffer[3] == 0); + + DeserializeContext des(ser.buffer.data()+4); + int32_t deserializedVal32 = DeserializePrimitive(&des); + CHECK(val32 == deserializedVal32); +} + +TEST_CASE("Array", "[fast] [serialize]") { + SerializeContext ser; + // serialize a single byte first, to ensure that array adds padding. + SerializePrimitive(&ser, 42); + for (int16_t i = 0; i < 16; i++) { + SerializePrimitive(&ser, i); + } + + DeserializeContext des(ser.buffer.data()); + int8_t firstByte = DeserializePrimitive(&des); + CHECK(firstByte == 42); + const int16_t *arr = DeserializeArray(&des, 8); + CHECK(arr[0] == 0); + CHECK(arr[1] == 1); + int8_t ninthByte = DeserializePrimitive(&des); + CHECK(ninthByte == 8); +} diff --git a/test/utils.cpp b/test/utils.cpp index e781f12a..337d7094 100644 --- a/test/utils.cpp +++ b/test/utils.cpp @@ -1,24 +1,12 @@ #include "utils.hpp" #include +#include #include "databases.hpp" namespace lost { -unsigned char *BuildPairDistanceKVectorDatabase( - const Catalog &catalog, long *length, float minDistance, float maxDistance, long numBins) { - - long dummyLength; - if (length == NULL) - length = &dummyLength; - - *length = SerializeLengthPairDistanceKVector(catalog, minDistance, maxDistance, numBins); - unsigned char *result = new unsigned char[*length]; - SerializePairDistanceKVector(catalog, minDistance, maxDistance, numBins, result); - return result; -} - bool AreStarIdentifiersEquivalent(const StarIdentifiers &ids1, const StarIdentifiers &ids2) { if (ids1.size() != ids2.size()) { return false; diff --git a/test/utils.hpp b/test/utils.hpp index a01e9a55..d99bd4a0 100644 --- a/test/utils.hpp +++ b/test/utils.hpp @@ -5,7 +5,6 @@ namespace lost { -unsigned char *BuildPairDistanceKVectorDatabase(const Catalog &catalog, long *length, float minDistance, float maxDistance, long numBins); /// simple O(n^2) check bool AreStarIdentifiersEquivalent(const StarIdentifiers &, const StarIdentifiers &);