Implementation of mmapped data structures

src/MMappedCircuit.cpp

@@ -6,31 +6,9 @@ extern "C" {
 
 static MMappedCircuit *c;
 
-MMappedCircuit::MMappedCircuit(bool read_only)
-{
-  if(read_only) {
-    gates_file = new bip::managed_mapped_file(bip::open_read_only, GATES_FILENAME);
-    wires_file = new bip::managed_mapped_file(bip::open_read_only, WIRES_FILENAME);
-    mapping_file = new bip::managed_mapped_file(bip::open_read_only, MAPPING_FILENAME);
-  } else {
-    gates_file = new bip::managed_mapped_file(bip::open_or_create, GATES_FILENAME, INITIAL_FILE_SIZE);
-    wires_file = new bip::managed_mapped_file(bip::open_or_create, WIRES_FILENAME, INITIAL_FILE_SIZE);
-    mapping_file = new bip::managed_mapped_file(bip::open_or_create, MAPPING_FILENAME, INITIAL_FILE_SIZE);
-  }
-  mapping = mapping_file->find_or_construct<mmapped_unordered_map>("mapping")(mapping_file->get_segment_manager());
-  // Get a raw pointer to access gates and wires
-}
-
-MMappedCircuit::~MMappedCircuit()
-{
-  delete mapping_file;
-  delete gates_file;
-  delete wires_file;
-}
-
 void initialize_provsql_mmap()
 {
-  c = new MMappedCircuit(false);
+  c = new MMappedCircuit();
 }
 
 void destroy_provsql_mmap()

src/MMappedCircuit.h

@@ -1,15 +1,13 @@
 #ifndef MMAPPED_CIRCUIT_H
 #define MMAPPED_CIRCUIT_H
 
-#include <boost/interprocess/managed_mapped_file.hpp>
-#include <boost/unordered_map.hpp>
+#include "MMappedUUIDHashTable.h"
+#include "MMappedVector.hpp"
 
 extern "C" {
 #include "provsql_utils.h"
 }
 
-namespace bip=boost::interprocess;
-
 typedef struct GateInformation
 {
   gate_type type;
@@ -22,30 +20,18 @@ typedef struct GateInformation
 
 class MMappedCircuit {
 private:
-bip::managed_mapped_file *gates_file;
-bip::managed_mapped_file *wires_file;
-bip::managed_mapped_file *mapping_file;
-
-using mmapped_unordered_map =
-  boost::unordered_map<
-    pg_uuid_t,
-    std::pair<unsigned, unsigned>,
-    boost::hash<pg_uuid_t>,
-    std::equal_to<pg_uuid_t>,
-    bip::allocator<pg_uuid_t, bip::managed_mapped_file::segment_manager> >;
+MMappedUUIDHashTable mapping;
+MMappedVector<GateInformation> gates;
+MMappedVector<pg_uuid_t> wires;
 
-mmapped_unordered_map *mapping;
-GateInformation *gates;
-pg_uuid_t *wires;
-
-static constexpr const char *GATES_FILENAME="gates.mmap";
-static constexpr const char *WIRES_FILENAME="wires.mmap";
-static constexpr const char *MAPPING_FILENAME="mapping.mmap";
-static constexpr unsigned INITIAL_FILE_SIZE = 1<<16;
+static constexpr const char *GATES_FILENAME="provsql_gates.mmap";
+static constexpr const char *WIRES_FILENAME="provsql_wires.mmap";
+static constexpr const char *MAPPING_FILENAME="provsql_mapping.mmap";
 
 public:
-explicit MMappedCircuit(bool read_only);
-~MMappedCircuit();
+explicit MMappedCircuit() :
+  mapping(MAPPING_FILENAME), gates(GATES_FILENAME), wires(WIRES_FILENAME) {
+}
 };
 
 #endif /* MMAPPED_CIRCUIT_H */

src/MMappedUUIDHashTable.cpp

@@ -0,0 +1,127 @@
+#include "MMappedUUIDHashTable.h"
+
+#include <cassert>
+#include <cerrno>
+#include <cstring>
+#include <new>
+#include <stdexcept>
+#include <vector>
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+
+MMappedUUIDHashTable::MMappedUUIDHashTable(const char *filename)
+{
+  fd=open(filename, O_CREAT|O_RDWR, 0600); // flawfinder: ignore
+  if(fd==-1)
+    throw std::runtime_error(strerror(errno));
+
+  auto size = lseek(fd, 0, SEEK_END);
+  lseek(fd, 0, SEEK_SET);
+
+  bool empty=false;
+
+  if(size==0) {
+    empty=true;
+    size=table_t::sizeForLogSize(STARTING_LOG_SIZE);
+    if(ftruncate(fd, size))
+      throw std::runtime_error(strerror(errno));
+  }
+
+  mmap(size);
+
+  if(empty) {
+    table->log_size = table_t::logSizeForSize(size);
+    table->nb_elements = 0;
+    table->next_value = 0;
+    for(unsigned i=0; i<table->capacity(); ++i) {
+      table->t[i].value = NOTHING;
+    }
+  }
+}
+
+void MMappedUUIDHashTable::mmap(size_t length)
+{
+  table = reinterpret_cast<table_t *>(::mmap(
+                                        NULL,
+                                        length,
+                                        PROT_READ|PROT_WRITE,
+                                        MAP_SHARED_VALIDATE,
+                                        fd,
+                                        0));
+  if(table == MAP_FAILED)
+    throw std::runtime_error(strerror(errno));
+}
+
+void MMappedUUIDHashTable::grow()
+{
+  std::vector<value_t> elements;
+  elements.reserve(table->nb_elements);
+  for(unsigned i=0; i<table->capacity(); ++i)
+    if(table->t[i].value != NOTHING)
+      elements.push_back(table->t[i]);
+
+  auto new_log_size = table->log_size+1;
+  assert(new_log_size<sizeof(unsigned)*8);
+  munmap(table, table_t::sizeForLogSize(table->log_size));
+
+  auto new_size = table_t::sizeForLogSize(new_log_size);
+  if(ftruncate(fd, new_size))
+    throw std::runtime_error(strerror(errno));
+  mmap(new_size);
+
+  table->log_size = new_log_size;
+  for(unsigned i=0; i<table->capacity(); ++i) {
+    table->t[i].value = NOTHING;
+  }
+  for(const auto &u: elements)
+    set(u.uuid, u.value);
+}
+
+MMappedUUIDHashTable::~MMappedUUIDHashTable()
+{
+  munmap(table, table_t::sizeForLogSize(table->log_size));
+  close(fd);
+}
+
+unsigned MMappedUUIDHashTable::find(pg_uuid_t u) const
+{
+  unsigned k = hash(u);
+  while(table->t[k].value != NOTHING &&
+        std::memcmp(&table->t[k].uuid, &u, sizeof(pg_uuid_t))) {
+    k = (k+1) % table->capacity();
+  }
+
+  return k;
+}
+
+unsigned MMappedUUIDHashTable::get(pg_uuid_t u) const
+{
+  unsigned k = find(u);
+
+  return table->t[k].value;
+}
+
+unsigned MMappedUUIDHashTable::get(pg_uuid_t u)
+{
+  unsigned k = find(u);
+
+  if(table->t[k].value == NOTHING) {
+    if(table->nb_elements >= MAXIMUM_LOAD_FACTOR * table->capacity()) {
+      grow();
+    }
+    k = find(u);
+    ++table->nb_elements;
+    table->t[k].uuid = u;
+    return table->t[k].value = table->next_value++;
+  } else
+    return table->t[k].value;
+}
+
+// Only used when growing the table, so no need to check/update nb_elements
+void MMappedUUIDHashTable::set(pg_uuid_t u, unsigned i)
+{
+  table->t[find(u)] = {u, i};
+}

src/MMappedUUIDHashTable.h

@@ -0,0 +1,71 @@
+#ifndef MMAPPED_UUID_HASH_TABLE_H
+#define MMAPPED_UUID_HASH_TABLE_H
+
+#include <cstddef>
+#include "provsql_utils.h"
+
+/*
+ * Open-addressing hash table meant to map UUIDs to integers, stored in
+ * an mmaped file; can only grow, impossible to remove elements; we use
+ * the fact that keys are UUIDs to have a trivial hash function
+ */
+class MMappedUUIDHashTable
+{
+struct value_t {
+  pg_uuid_t uuid;
+  unsigned value;
+};
+
+struct table_t {
+  static constexpr std::size_t sizeForLogSize(unsigned ls) {
+    return offsetof(table_t, t) + (1 << ls)*sizeof(value_t);
+  }
+  static constexpr unsigned logSizeForSize(std::size_t size) {
+    size -= offsetof(table_t, t);
+    size /= sizeof(value_t);
+    size >>= 1;
+    unsigned log_size=0;
+    while(size) {
+      size >>= 1;
+      ++log_size;
+    }
+    return log_size;
+  }
+  constexpr unsigned capacity() {
+    return 1u << log_size;
+  }
+
+  unsigned log_size;
+  unsigned nb_elements;
+  unsigned next_value;
+  value_t t[];
+};
+
+int fd;
+table_t *table;
+
+static constexpr unsigned STARTING_LOG_SIZE=16;
+static constexpr double MAXIMUM_LOAD_FACTOR=.5;
+static constexpr unsigned NOTHING=static_cast<unsigned>(-1);
+
+inline unsigned hash(pg_uuid_t u) const {
+  return *reinterpret_cast<unsigned*>(&u) % (1 << table->log_size);
+};
+
+unsigned find(pg_uuid_t u) const;
+void mmap(size_t length);
+void grow();
+void set(pg_uuid_t u, unsigned i);
+
+public:
+explicit MMappedUUIDHashTable(const char *filename);
+~MMappedUUIDHashTable();
+
+unsigned get(pg_uuid_t u) const;
+unsigned get(pg_uuid_t u);
+inline unsigned nbElements() const {
+  return table->nb_elements;
+}
+};
+
+ #endif /* MMAPPED_UUID_HASH_TABLE_H */

src/MMappedVector.h

@@ -0,0 +1,39 @@
+#ifndef MMAPPED_VECTOR_H
+#define MMAPPED_VECTOR_H
+
+#include <cstddef>
+#include "provsql_utils.h"
+
+/*
+ * Vector template, stored in an mmaped file; can only grow, impossible
+ * to remove elements
+ */
+template <typename T>
+class MMappedVector {
+struct data_t {
+  unsigned nb_elements;
+  unsigned capacity;
+  T d[];
+};
+
+int fd;
+data_t *data;
+
+static constexpr unsigned STARTING_CAPACITY=(1u << 16);
+
+void mmap(size_t length);
+void grow();
+void set(pg_uuid_t u, unsigned i);
+
+public:
+explicit MMappedVector(const char *filename);
+~MMappedVector();
+
+T get(unsigned k) const;
+void add(T value);
+inline unsigned nbElements() const {
+  return data->nb_elements;
+}
+};
+
+ #endif /* MMAPPED_VECTOR_H */