Merge pull request #21 from Intel-HLS/dev

Thread-/process-safety and bug fixes

Makefile

@@ -42,7 +42,7 @@ ifeq ($(BUILD),release)
 endif
 
 ifeq ($(BUILD),debug)
-  CPPFLAGS += -DDEBUG -gdwarf-3 -g3
+  CPPFLAGS += -DDEBUG -gdwarf-3 -g3 -Wall
 endif
 
 # --- Verbose mode handler --- #
@@ -239,7 +239,8 @@ clean_libtiledb:
 $(EXAMPLES_OBJ_DIR)/%.o: $(EXAMPLES_SRC_DIR)/%.cc
 	@mkdir -p $(EXAMPLES_OBJ_DIR)
 	@echo "Compiling $<"
-	@$(CXX) $(CPPFLAGS) $(INCLUDE_PATHS) $(EXAMPLES_INCLUDE_PATHS) \
+	@$(CXX) $(CPPFLAGS) -fopenmp $(INCLUDE_PATHS) \
+                $(EXAMPLES_INCLUDE_PATHS) \
 		$(CORE_INCLUDE_PATHS) -c $< -o $@
 	@$(CXX) -MM $(EXAMPLES_INCLUDE_PATHS) \
                     $(CORE_INCLUDE_PATHS) $< > $(@:.o=.d)
@@ -273,7 +274,7 @@ clean_examples:
 $(TEST_OBJ_DIR)/%.o: $(TEST_SRC_DIR)/%.cc
 	@mkdir -p $(dir $@) 
 	@echo "Compiling $<"
-	@$(CXX) $(CPPFLAGS) $(TEST_INCLUDE_PATHS) -c $< -o $@
+	@$(CXX) $(CPPFLAGS) -fopenmp $(TEST_INCLUDE_PATHS) -c $< -o $@
 	@$(CXX) -MM $(TEST_INCLUDE_PATHS) \
                     $(CORE_INCLUDE_PATHS) $< > $(@:.o=.d)
 	@mv -f $(@:.o=.d) $(@:.o=.d.tmp)

README.md

@@ -1,4 +1,9 @@
-The TileDB documentation for users is hosted as 
-a Github wiki: https://github.com/Intel-HLS/TileDB/wiki
+# TileDB
 
-TileDB official website: http://istc-bigdata.org/tiledb
+[![Travis](https://img.shields.io/travis/Intel-HLS/TileDB.svg?maxAge=2592000)]
+(https://travis-ci.org/Intel-HLS/TileDB)
+
+The TileDB documentation for users is hosted as a [Github
+wiki](https://github.com/Intel-HLS/TileDB/wiki).
+
+[TileDB official website](http://istc-bigdata.org/tiledb).

core/include/array/array.h

@@ -35,6 +35,7 @@
 
 #include "array_read_state.h"
 #include "array_schema.h"
+#include "book_keeping.h"
 #include "constants.h"
 #include "fragment.h"
 
@@ -140,10 +141,17 @@ class Array {
 
   /**
    * Consolidates all fragments into a new single one, on a per-attribute basis.
+   * Returns the new fragment (which has to be finalized outside this functions),
+   * along with the names of the old (consolidated) fragments (which also have
+   * to be deleted outside this function).
    *
+   * @param new_fragment The new fragment to be returned.
+   * @param old_fragment_names The names of the old fragments to be returned.
    * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error.
    */
-  int consolidate();
+  int consolidate(
+      Fragment*& new_fragment, 
+      std::vector<std::string>& old_fragment_names);
 
   /**
    * Consolidates all fragment into a new single one, focusing on a specific
@@ -167,6 +175,9 @@ class Array {
    * Initializes a TileDB array object.
    *
    * @param array_schema The array schema.
+   * @param fragment_names The names of the fragments of the array.
+   * @param book_keeping The book-keeping structures of the fragments
+   *     of the array.
    * @param mode The mode of the array. It must be one of the following:
    *    - TILEDB_ARRAY_WRITE 
    *    - TILEDB_ARRAY_WRITE_UNSORTED 
@@ -184,6 +195,8 @@ class Array {
    */
   int init(
       const ArraySchema* array_schema, 
+      const std::vector<std::string>& fragment_names,
+      const std::vector<BookKeeping*>& book_keeping,
       int mode,
       const char** attributes,
       int attribute_num,
@@ -299,21 +312,21 @@ class Array {
    * After the new fragmemt is finalized, the array will change its name
    * by removing the leading '.' character. 
    *
-   * @return A new special fragment name.
+   * @return A new special fragment name on success, or "" (empty string) on
+   *     error.
    */
   std::string new_fragment_name() const;
 
   /**
    * Opens the existing fragments in TILEDB_ARRAY_READ_MODE.
    *
+   * @param fragment_names The vector with the fragment names.
+   * @param book_keeping The book-keeping of the array fragments.
    * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error.
    */
-  int open_fragments();
-
-  /** 
-   * Appropriately sorts the fragment names based on their name timestamps.
-   */
-  void sort_fragment_names(std::vector<std::string>& fragment_names) const;
+  int open_fragments(
+      const std::vector<std::string>& fragment_names,
+      const std::vector<BookKeeping*>& book_keeping);
 };
 
 #endif

core/include/array/array_iterator.h

@@ -71,6 +71,9 @@ class ArrayIterator {
   /*             ACCESSORS             */
   /* ********************************* */
 
+  /** Return the array name. */
+  const std::string& array_name() const;
+
   /**
    * Checks if the the iterator has reached its end.
    *

core/include/array/array_schema.h

@@ -511,7 +511,7 @@ class ArraySchema {
    *     may be a sub-domain of the array domain).
    * @param tile_coords The tile coordinates. 
    * @return The tile position of *tile_coords* along the tile order of the
-   *     array inside the input domain.
+   *     array inside the input domain, or TILEDB_AS_ERR on error.
    */
   template<class T> 
   int64_t get_tile_pos(

core/include/c_api/c_api.h

@@ -462,10 +462,13 @@ TILEDB_EXPORT int tiledb_array_overflow(
 /**
  * Consolidates the fragments of an array into a single fragment. 
  * 
- * @param tiledb_array The TileDB array to be consolidated.
+ * @param tiledb_ctx The TileDB context.
+ * @param array The name of the TileDB array to be consolidated.
  * @return TILEDB_OK on success, and TILEDB_ERR on error.
  */
-TILEDB_EXPORT int tiledb_array_consolidate(const TileDB_Array* tiledb_array);
+TILEDB_EXPORT int tiledb_array_consolidate(
+    const TileDB_CTX* tiledb_ctx,
+    const char* array);
 
 /** 
  * Finalizes a TileDB array, properly freeing its memory space. 
@@ -819,11 +822,13 @@ TILEDB_EXPORT int tiledb_metadata_overflow(
 /**
  * Consolidates the fragments of a metadata object into a single fragment. 
  * 
- * @param tiledb_metadata The TileDB metadata to be consolidated.
+ * @param tiledb_ctx The TileDB context.
+ * @param metadata The name of the TileDB metadata to be consolidated.
  * @return TILEDB_OK on success, and TILEDB_ERR on error.
  */
 TILEDB_EXPORT int tiledb_metadata_consolidate(
-    const TileDB_Metadata* tiledb_metadata);
+    const TileDB_CTX* tiledb_ctx,
+    const char* metadata);
 
 /** 
  * Finalizes a TileDB metadata object, properly freeing the memory space. 

core/include/fragment/book_keeping.h

@@ -33,7 +33,8 @@
 #ifndef __BOOK_KEEPING_H__
 #define __BOOK_KEEPING_H__
 
-#include "fragment.h"
+#include "array_schema.h"
+#include "constants.h"
 #include <vector>
 #include <zlib.h>
 
@@ -53,8 +54,6 @@
 
 
 
-class Fragment;
-
 /** Stores the book-keeping structures of a fragment. */
 class BookKeeping {
  public:
@@ -65,9 +64,16 @@ class BookKeeping {
   /** 
    * Constructor. 
    *
-   * @param fragment The fragment the book-keeping structure belongs to.
+   * @param array_schema The array schema.
+   * @param dense True if the fragment is dense, and false otherwise.
+   * @param fragment_name The name of the fragment this book-keeping belongs to.
+   * @param mode The mode in which the fragment was initialized in.
    */
-  BookKeeping(const Fragment* fragment);
+  BookKeeping(
+      const ArraySchema* array_schema, 
+      bool dense, 
+      const std::string& fragment_name,
+      int mode);
 
   /** Destructor. */
   ~BookKeeping();
@@ -85,6 +91,12 @@ class BookKeeping {
   /** Returns the number of cells in the tile at the input position. */
   int64_t cell_num(int64_t tile_pos) const;
 
+  /** 
+   * Returns ture if the corresponding fragment is dense, and false if it
+   * is sparse.
+   */
+  bool dense() const;
+
   /** Returns the (expanded) domain in which the fragment is constrained. */
   const void* domain() const;
 
@@ -202,21 +214,27 @@ class BookKeeping {
   /*         PRIVATE ATTRIBUTES        */
   /* ********************************* */
 
+  /** The array schema */
+  const ArraySchema* array_schema_;
   /** The first and last coordinates of each tile. */
   std::vector<void*> bounding_coords_;
+  /** True if the fragment is dense, and false if it is sparse. */
+  bool dense_;
   /**
    * The (expanded) domain in which the fragment is constrained. "Expanded"
    * means that the domain is enlarged minimally to coincide with tile 
    * boundaries (if there is a tile grid imposed by tile extents). Note that the
    * type of the domain must be the same as the type of the array coordinates.
    */
   void* domain_;
-  /** The fragment the book-keeping belongs to. */
-  const Fragment* fragment_;
+  /** The name of the fragment the book-keeping belongs to. */
+  std::string fragment_name_;
   /** Number of cells in the last tile (meaningful only in the sparse case). */
   int64_t last_tile_cell_num_;
   /** The MBRs (applicable only to the sparse case with irregular tiles). */
   std::vector<void*> mbrs_;
+  /** The mode in which the fragment was initialized. */
+  int mode_;
   /** The offsets of the next tile for each attribute. */
   std::vector<off_t> next_tile_offsets_;
   /** The offsets of the next variable tile for each attribute. */

core/include/fragment/fragment.h

@@ -36,7 +36,6 @@
 #include "array.h"
 #include "array_schema.h"
 #include "book_keeping.h"
-#include "constants.h"
 #include "read_state.h"
 #include "write_state.h"
 #include <vector>
@@ -125,11 +124,10 @@ class Fragment {
   int finalize();
 
   /**
-   * Initializes a fragment.
+   * Initializes a fragment in write mode.
    *
    * @param fragment_name The name that will be given to the fragment.
    * @param mode The fragment mode. It can be one of the following: 
-   *    - TILEDB_READ
    *    - TILEDB_WRITE 
    *    - TILEDB_WRITE_UNSORTED 
    * @param subarray The subarray the fragment is constrained on.
@@ -140,6 +138,17 @@ class Fragment {
       int mode,
       const void* subarray);
 
+  /**
+   * Initializes a fragment in read mode.
+   *
+   * @param fragment_name The name that will be given to the fragment.
+   * @param book_keeping The book-keeping of the fragment.
+   * @return TILEDB_FG_OK on success and TILEDB_FG_ERR on error. 
+   */
+  int init(
+      const std::string& fragment_name, 
+      BookKeeping* book_keeping);
+
   /** Resets the read state (typically to start a new read). */
   void reset_read_state();
 

core/include/fragment/read_state.h

@@ -53,7 +53,7 @@
 
 
 
-class BookKeeping;
+class Fragment;
 
 /** Stores the state necessary when reading cells from a fragment. */
 class ReadState {

core/include/metadata/metadata.h

@@ -117,11 +117,18 @@ class Metadata {
   /* ********************************* */
 
   /**
-   * Consolidates the fragments of a metadata object into a single fragment. 
-   * 
-   * @return TILEDB_MT_OK on success, and TILEDB_MT_ERR on error.
+   * Consolidates all fragments into a new single one, on a per-attribute basis.
+   * Returns the new fragment (which has to be finalized outside this functions),
+   * along with the names of the old (consolidated) fragments (which also have
+   * to be deleted outside this function).
+   *
+   * @param new_fragment The new fragment to be returned.
+   * @param old_fragment_names The names of the old fragments to be returned.
+   * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error.
    */
-  int consolidate();
+  int consolidate(
+      Fragment*& new_fragment, 
+      std::vector<std::string>& old_fragment_names);
 
   /**
    * Finalizes the metadata, properly freeing up the memory space.
@@ -134,6 +141,9 @@ class Metadata {
    * Initializes a TileDB metadata object.
    *
    * @param array_schema This essentially encapsulates the metadata schema.
+   * @param fragment_names The names of the fragments of the array.
+   * @param book_keeping The book-keeping structures of the fragments
+   *     of the array.
    * @param mode The mode of the metadata. It must be one of the following:
    *    - TILEDB_METADATA_WRITE 
    *    - TILEDB_METADATA_READ 
@@ -146,6 +156,8 @@ class Metadata {
    */
   int init(
       const ArraySchema* array_schema, 
+      const std::vector<std::string>& fragment_names,
+      const std::vector<BookKeeping*>& book_keeping,
       int mode,
       const char** attributes,
       int attribute_num);

core/include/metadata/metadata_iterator.h

@@ -69,6 +69,9 @@ class MetadataIterator {
   /*             ACCESSORS             */
   /* ********************************* */
 
+  /** Return the metadata name. */
+  const std::string& metadata_name() const;
+
   /**
    * Checks if the the iterator has reached its end.
    *
@@ -134,8 +137,10 @@ class MetadataIterator {
  private:
   // PRIVATE ATTRIBUTES
 
-  // TODO
+  /** The array iterator that implements the metadata iterator. */
   ArrayIterator* array_it_;
+  /** The metadata this iterator belongs to. */
+  Metadata* metadata_;
 };
 
 #endif