Merge branch 'master' of github.com:infoforcefeed/OlegDB

Makefile

@@ -1,4 +1,4 @@
-CFLAGS=-Wall -Werror -g -O2 -Wstrict-aliasing=2
+CFLAGS=-Wall -Werror -g -O2 -Wstrict-aliasing=2 -Wno-format-truncation
 uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
 
 ifndef CC
@@ -13,9 +13,11 @@ PREFIX?=/usr/local
 INSTALL_LIB=$(PREFIX)/lib/
 INSTALL_BIN=$(PREFIX)/bin/
 INSTALL_INCLUDE=$(PREFIX)/include/olegdb/
+OBJ_FILES=vector.o murmur3.o oleg.o logging.o aol.o rehash.o file.o utils.o tree.o lz4.o stack.o cursor.o transaction.o
 
 TEST_OUT=$(BIN_DIR)oleg_test
 LIB_OUT=$(LIB_DIR)liboleg.so
+STATIC_LIB_OUT=$(LIB_DIR)liboleg.a
 BIN_OUT=$(BIN_DIR)olegdb
 export CGO_LDFLAGS=-L$(BUILD_DIR)lib
 
@@ -54,9 +56,12 @@ $(TEST_OUT): test.o main.o
 	$(CC) $(INCLUDES) -L$(LIB_DIR) -o $(TEST_OUT) test.o main.o $(MATH_LINKER) -loleg
 
 liboleg: $(LIB_DIR) $(LIB_OUT)
-$(LIB_OUT): vector.o murmur3.o oleg.o logging.o aol.o rehash.o file.o utils.o tree.o lz4.o stack.o cursor.o transaction.o
+$(LIB_OUT): $(OBJ_FILES)
 	$(CC) $(INCLUDES) -o $(LIB_OUT) $^ -fpic -shared $(MATH_LINKER)
 
+static: $(LIB_OUT) $(OBJ_FILES)
+	ar rcs $(STATIC_LIB_OUT) $(OBJ_FILES)
+
 uninstall:
 	rm -rf $(INSTALL_LIB)liboleg*
 	rm -rf $(INSTALL_BIN)olegdb
@@ -88,3 +93,5 @@ clean:
 	rm -f $(BIN_DIR)*
 	rm -f $(LIB_DIR)*
 	rm -f *.o
+
+.PHONY: clean test libinstall install uninstall static all

README.md

@@ -78,6 +78,6 @@ Roadmap
 Roadmap is full of lies and half-truths, please ignore.
 
 - [ ] Witch hunt
-- [ ] Wordard generation
+- [ ] Wordart generation
 - [ ] Feeding tube integration
 - [ ] Being more stable than redis

frontend/dispatcher.go

@@ -42,6 +42,7 @@ func DBRequester() chan DBOpenRequest {
 				if config.SplayTreeEnabled {
 					flags = flags | goleg.F_SPLAYTREE
 				}
+				flags = flags | goleg.F_AOL_FFLUSH
 				databases[dbname], dberr = goleg.Open(config.DataDir, dbname, flags)
 				database = databases[dbname]
 			}

include/aol.h

@@ -5,6 +5,14 @@
 /* Filemode used for writing AOL commands. Restore just uses "r" */
 #define AOL_FILEMODE "ab+"
 
+typedef struct bogo_file {
+    int fd;
+    size_t filesize;
+    size_t read_offset;
+    unsigned char *filestart;
+    unsigned char *read_ptr;
+} bogo_file;
+
 int ol_aol_init(ol_database *db);
 int ol_aol_write_cmd(ol_database *db, const char *cmd, ol_bucket *bucket);
 /* Restore this database's AOL file. */

include/file.h

@@ -14,6 +14,9 @@ void _ol_close_values(ol_database *db);
 /* Wraps mmap so that we mmap() consistently. */
 void *_ol_mmap(size_t to_mmap, int fd);
 
+/* _ol_mmap with different mmap flags. */
+void *_ol_mmap_readonly(size_t to_mmap, int fd);
+
 /* Called whenever a value is inserted into the database. Truncates the value
  * file to the correct size, aligned by VALUES_DEFAULT_SIZE chunks. */
 int _ol_ensure_values_file_size(ol_database *db, const size_t desired_size);

include/test.h

@@ -14,6 +14,8 @@
     if (test_return_val != 0 || errno != 0) {\
         tests_failed++;\
         ol_log_msg(LOG_ERR, "%c[%dmFailed.%c[%dm", 0x1B, 31, 0x1B, 0);\
+        if (errno != 0) ol_log_msg(LOG_ERR, "errno is not 0!");\
+        if (test_return_val != 0) ol_log_msg(LOG_ERR, "Retval: %d", test_return_val);\
         ol_log_msg(LOG_ERR, "ERRNO: %s\n", clean_errno());\
         goto error;\
     } else {\

src/file.c

@@ -35,11 +35,11 @@ size_t _ol_get_file_size(const char *filepath) {
     return 0;
 }
 
-void *_ol_mmap(size_t to_mmap, int fd) {
+void *_ol_mmap_internal(size_t to_mmap, int fd, const int flags) {
     /* TODO: Investigate usage of madvise here. */
     void *to_return = NULL;
 
-    to_return = mmap(NULL, to_mmap, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    to_return = mmap(NULL, to_mmap, flags, MAP_SHARED, fd, 0);
     check(to_return != MAP_FAILED, "Could not mmap file.");
 
     const int check = madvise(to_return, to_mmap, MADV_RANDOM);
@@ -52,6 +52,14 @@ void *_ol_mmap(size_t to_mmap, int fd) {
     return NULL;
 }
 
+void *_ol_mmap_readonly(size_t to_mmap, int fd) {
+    return _ol_mmap_internal(to_mmap, fd, PROT_READ);
+}
+
+void *_ol_mmap(size_t to_mmap, int fd) {
+    return _ol_mmap_internal(to_mmap, fd, PROT_READ | PROT_WRITE);
+}
+
 int _ol_open_values_with_fd(ol_database *db, const int fd, const size_t filesize) {
     const size_t to_mmap = filesize == 0 ? VALUES_DEFAULT_SIZE : filesize;
 
@@ -89,7 +97,8 @@ void _ol_close_values(ol_database *db) {
     db->get_db_file_name(db, VALUES_FILENAME, values_filename);
     const size_t siz = _ol_get_file_size(values_filename);
 
-    munmap(db->values, siz);
+    if (db->values)
+        munmap(db->values, siz);
     flock(db->valuesfd, LOCK_UN);
     close(db->valuesfd);
 }
@@ -144,7 +153,7 @@ int _ol_open_values(ol_database *db) {
 
     /* Figure out the filename */
     db->get_db_file_name(db, VALUES_FILENAME, values_filename);
-    size_t filesize = _ol_get_file_size(values_filename);
+    const size_t filesize = _ol_get_file_size(values_filename);
 
     debug("Opening %s for values", values_filename);
     values_fd = open(values_filename, O_RDWR | O_CREAT, S_IWUSR | S_IRUSR);

src/oleg.c

@@ -269,13 +269,16 @@ int ol_unjar(ol_database *db, const char *key, size_t klen, unsigned char **data
         return olt_unjar(&stack_tx, key, klen, data, dsize);
     }
 
+    debug("Beginning ol_unjar tx");
     ol_transaction *tx = olt_begin(db);
     int unjar_ret = OL_SUCCESS;
     check(tx != NULL, "Could not begin transaction.");
 
+    debug("Beginning olt_unjar");
     unjar_ret = olt_unjar(tx, key, klen, data, dsize);
     check(unjar_ret == OL_SUCCESS, "Could not unjar.");
 
+    debug("Commiting olt_unjar");
     check(olt_commit(tx) == OL_SUCCESS, "Could not commit transaction.");
 
     return OL_SUCCESS;
@@ -337,9 +340,11 @@ int ol_spoil(ol_database *db, const char *key, size_t klen, struct tm *expiratio
     int spoil_ret = OL_SUCCESS;
     check(tx != NULL, "Could not begin implicit transaction.");
 
+    debug("ol_spoilt tx created.");
     spoil_ret = olt_spoil(tx, key, klen, expiration_date);
     check(spoil_ret == OL_SUCCESS, "Could not spoil value. Aborting.");
 
+    debug("Spoil succeeded, now commiting.");
     check(olt_commit(tx) == OL_SUCCESS, "Could not commit transaction.");
     debug("End of ol_spoil.");
 

src/transaction.c

@@ -188,21 +188,27 @@ int olt_unjar(ol_transaction *tx, const char *key, size_t klen, unsigned char **
         operating_db = tx->transaction_db;
     }
 
+    debug("Checking expiry");
     if (bucket != NULL) {
         if (!_has_bucket_expired(bucket)) {
             /* We don't need to fill out the data so just return 'we found the key'. */
-            if (data == NULL)
+            if (data == NULL) {
+                debug("Data is NULL");
                 return OL_SUCCESS;
+            }
 
+            debug("Getting value from bucket");
             const int ret = _ol_get_value_from_bucket(operating_db, bucket, data, dsize);
             check(ret == 0, "Could not retrieve value from bucket.");
 
             /* Key found, tell somebody. */
+            debug("Everything worked");
             return OL_SUCCESS;
         } else {
             /* It's dead, get rid of it. */
             /* NOTE: We explicitly say the transaction_db here because ITS A
              * FUCKING TRANSACTION. ACID, bro. */
+            debug("Bucket has expired.");
             check(olt_scoop(tx, key, klen) == 0, "Scoop failed");
         }
     }
@@ -220,12 +226,14 @@ int olt_exists(ol_transaction *tx, const char *key, size_t klen) {
 }
 
 int olt_jar(ol_transaction *tx, const char *key, size_t klen, const unsigned char *value, size_t vsize) {
-    int ret;
+    int ret = 0;
     char _key[KEY_SIZE] = {'\0'};
     size_t _klen = 0;
     ol_database *db = tx->transaction_db;
 
-    ol_bucket *bucket = ol_get_bucket(db, key, klen, &_key, &_klen);
+    ol_bucket *new_bucket = NULL;
+    ol_bucket *bucket = NULL;
+    bucket = ol_get_bucket(db, key, klen, &_key, &_klen);
     check(_klen > 0, "Key length of zero not allowed.");
 
     /* We only want to hit this codepath within the same database, otherwise
@@ -238,17 +246,19 @@ int olt_jar(ol_transaction *tx, const char *key, size_t klen, const unsigned cha
     }
 
     /* Looks like we don't have an old hash */
-    ol_bucket *new_bucket = calloc(1, sizeof(ol_bucket));
-    if (new_bucket == NULL)
-        return OL_FAILURE;
+    new_bucket = calloc(1, sizeof(ol_bucket));
+    if (new_bucket == NULL) {
+        ol_log_msg(LOG_ERR, "Could not allocate new bucket.");
+        goto error;
+    }
 
     /* copy _key into new bucket */
     new_bucket->key = malloc(_klen + 1);
     check_mem(new_bucket->key);
     new_bucket->key[_klen] = '\0';
     if (strncpy(new_bucket->key, _key, _klen) != new_bucket->key) {
-        free(new_bucket);
-        return OL_FAILURE;
+        ol_log_msg(LOG_ERR, "Could not copy key into bucket.");
+        goto error;
     }
 
     new_bucket->klen = _klen;
@@ -271,9 +281,8 @@ int olt_jar(ol_transaction *tx, const char *key, size_t klen, const unsigned cha
             size_t cmsize = (size_t)LZ4_compress((char*)value, (char*)new_data_ptr,
                                                  (int)vsize);
             if (cmsize == 0) {
-                /* Free allocated data */
-                free(new_bucket);
-                return OL_FAILURE;
+                ol_log_msg(LOG_ERR, "Could not compress data.");
+                goto error;
             }
 
             new_bucket->data_size = cmsize;
@@ -284,9 +293,8 @@ int olt_jar(ol_transaction *tx, const char *key, size_t klen, const unsigned cha
             memset(new_data_ptr, '\0', new_bucket->data_size);
 
             if (memcpy(new_data_ptr, value, vsize) != new_data_ptr) {
-                /* Free allocated memory since we're not going to use them */
-                free(new_bucket);
-                return OL_FAILURE;
+                ol_log_msg(LOG_ERR, "Could not copy data into bucket.");
+                goto error;
             }
         }
     } else {
@@ -321,21 +329,15 @@ int olt_jar(ol_transaction *tx, const char *key, size_t klen, const unsigned cha
     /* TODO: rehash this shit at 80% */
     if (db->rcrd_cnt > 0 && db->rcrd_cnt == bucket_max) {
         debug("Record count is now %i; growing hash table.", db->rcrd_cnt);
-        ret = _ol_grow_and_rehash_db(db);
-        if (ret > 0) {
-            ol_log_msg(LOG_ERR, "Problem rehashing DB. Error code: %i", ret);
-            free(new_bucket);
-            return OL_FAILURE;
-        }
+        check(_ol_grow_and_rehash_db(db) <= 0, "Problem rehashing DB. Error code: %i", ret);
     }
 
 
     uint32_t hash;
     MurmurHash3_x86_32(_key, _klen, DEVILS_SEED, &hash);
     ret = _ol_set_bucket(db, new_bucket, hash);
 
-    if(ret > 0)
-        ol_log_msg(LOG_ERR, "Problem inserting item: Error code: %i", ret);
+    check(ret <= 0, "Problem inserting item: Error code: %i", ret);
 
     if(db->is_enabled(OL_F_APPENDONLY, &db->feature_set) &&
             db->state != OL_S_STARTUP) {
@@ -348,6 +350,8 @@ int olt_jar(ol_transaction *tx, const char *key, size_t klen, const unsigned cha
     return OL_SUCCESS;
 
 error:
+    if (new_bucket != NULL)
+        free(new_bucket);
     return OL_FAILURE;
 }
 
@@ -447,39 +451,48 @@ int olt_spoil(ol_transaction *tx, const char *key, size_t klen, struct tm *expir
 
     ol_database *operating_db = tx->transaction_db;
 
+    debug("Fetching bucket");
     ol_bucket *bucket = ol_get_bucket(operating_db, key, klen, &_key, &_klen);
     check(_klen > 0, "Key length of zero not allowed.");
 
     if (bucket == NULL && tx->parent_db != NULL) {
         /* Transaction DB doesn't have this key, but the parent does. */
+        debug("Have to get bucket from parent.");
         operating_db = tx->parent_db;
         bucket = ol_get_bucket(operating_db, key, klen, &_key, &_klen);
         if (bucket != NULL) {
             /* Copy that value into our current transaction db,
              * and then spoil it.
              */
+            debug("Retrieved buckey.");
             uint32_t hash;
             MurmurHash3_x86_32(_key, _klen, DEVILS_SEED, &hash);
             ol_bucket *copied = malloc(sizeof(ol_bucket));
             check_mem(copied);
 
+            debug("Copying into new buckey.");
             memcpy(copied, bucket, sizeof(ol_bucket));
             copied->next = NULL;
 
+            debug("Malloc()ing key");
             copied->key = malloc(_klen + 1);
             check_mem(copied->key);
             copied->key[_klen] = '\0';
+            debug("Copying into key.");
             memcpy(copied->key, bucket->key, _klen);
 
+            debug("Setting buckey.");
             _ol_set_bucket_no_incr(tx->transaction_db, copied, hash);
         }
     }
 
     if (bucket != NULL) {
+        debug("Setting the expiration.");
         if (bucket->expiration == NULL)
-            bucket->expiration = malloc(sizeof(struct tm));
+            bucket->expiration = calloc(1, sizeof(struct tm));
         else
             debug("Hmmm, bucket->expiration wasn't null.");
+        debug("Copying into expiration.");
         memcpy(bucket->expiration, expiration_date, sizeof(struct tm));
         debug("New expiration time: %lu", (long)mktime(bucket->expiration));