Skip to content

Commit

Permalink
Fix CI.
Browse files Browse the repository at this point in the history
  • Loading branch information
shaunrd0 committed Jul 23, 2024
1 parent b00936b commit aa494ce
Show file tree
Hide file tree
Showing 8 changed files with 56 additions and 51 deletions.
59 changes: 27 additions & 32 deletions test/src/cpp-integration-rest-schema-evolution.cc
Original file line number Diff line number Diff line change
Expand Up @@ -37,22 +37,24 @@
#include "tiledb/sm/cpp_api/tiledb_experimental"
#include "tiledb/sm/rest/rest_client.h"

#include <climits>
#include <fstream>

using namespace tiledb;

static void create_array(const std::string& array_uri);
static void write_first_fragment(const std::string& array_uri);
static void create_array(const Context& ctx, const std::string& array_uri);
static void write_first_fragment(
const Context& ctx, const std::string& array_uri);
static uint64_t time_travel_destination();
static void add_attr_b(const std::string& array_uri);
static void write_second_fragment(const std::string& array_uri);
static void read_without_time_travel(const std::string& array_uri);
static void read_with_time_travel(const std::string& array_uri, uint64_t when);

void create_array(const std::string& array_uri) {
tiledb::test::VFSTestSetup vfs_test_setup;
tiledb::Context ctx{vfs_test_setup.ctx()};

static void add_attr_b(const Context& ctx, const std::string& array_uri);
static void write_second_fragment(
const Context& ctx, const std::string& array_uri);
static void read_without_time_travel(
const Context& ctx, const std::string& array_uri);
static void read_with_time_travel(
const Context& ctx, const std::string& array_uri, uint64_t when);

void create_array(const Context& ctx, const std::string& array_uri) {
auto obj = tiledb::Object::object(ctx, array_uri);
if (obj.type() != tiledb::Object::Type::Invalid) {
tiledb::Object::remove(ctx, array_uri);
Expand All @@ -73,12 +75,10 @@ void create_array(const std::string& array_uri) {
tiledb::Array::create(array_uri, schema);
}

void write_first_fragment(const std::string& array_uri) {
void write_first_fragment(const Context& ctx, const std::string& array_uri) {
std::vector<int32_t> d_data = {0, 1, 2, 3, 4};
std::vector<int32_t> a_data = {5, 6, 7, 8, 9};

tiledb::test::VFSTestSetup vfs_test_setup;
tiledb::Context ctx{vfs_test_setup.ctx()};
tiledb::Array array(ctx, array_uri, TILEDB_WRITE);
tiledb::Query query(ctx, array, TILEDB_WRITE);
query.set_layout(TILEDB_UNORDERED)
Expand All @@ -101,23 +101,19 @@ uint64_t time_travel_destination() {
return timepoint;
}

void add_attr_b(const std::string& array_uri) {
tiledb::test::VFSTestSetup vfs_test_setup;
tiledb::Context ctx{vfs_test_setup.ctx()};
void add_attr_b(const Context& ctx, const std::string& array_uri) {
auto attr = tiledb::Attribute::create<int32_t>(ctx, "b");

tiledb::ArraySchemaEvolution ase(ctx);
ase.add_attribute(attr);
ase.array_evolve(array_uri);
}

void write_second_fragment(const std::string& array_uri) {
void write_second_fragment(const Context& ctx, const std::string& array_uri) {
std::vector<int32_t> d_data = {5, 6, 7, 8, 9};
std::vector<int32_t> a_data = {10, 11, 12, 13, 14};
std::vector<int32_t> b_data = {15, 16, 17, 18, 19};

tiledb::test::VFSTestSetup vfs_test_setup;
tiledb::Context ctx{vfs_test_setup.ctx()};
tiledb::Array array(ctx, array_uri, TILEDB_WRITE);
tiledb::Query query(ctx, array, TILEDB_WRITE);
query.set_layout(TILEDB_UNORDERED)
Expand All @@ -128,13 +124,12 @@ void write_second_fragment(const std::string& array_uri) {
array.close();
}

void read_without_time_travel(const std::string& array_uri) {
void read_without_time_travel(
const Context& ctx, const std::string& array_uri) {
std::vector<int32_t> d_data(10);
std::vector<int32_t> a_data(10);
std::vector<int32_t> b_data(10);

tiledb::test::VFSTestSetup vfs_test_setup;
tiledb::Context ctx{vfs_test_setup.ctx()};
tiledb::Array array(ctx, array_uri, TILEDB_READ);
tiledb::Query query(ctx, array, TILEDB_READ);
query.set_data_buffer("d", d_data)
Expand All @@ -155,13 +150,12 @@ void read_without_time_travel(const std::string& array_uri) {
}
}

void read_with_time_travel(const std::string& array_uri, uint64_t when) {
void read_with_time_travel(
const Context& ctx, const std::string& array_uri, uint64_t when) {
std::vector<int32_t> d_data(10, INT_MAX);
std::vector<int32_t> a_data(10, INT_MAX);
std::vector<int32_t> b_data(10, INT_MAX);

tiledb::test::VFSTestSetup vfs_test_setup;
tiledb::Context ctx{vfs_test_setup.ctx()};
tiledb::Array array(
ctx,
array_uri,
Expand Down Expand Up @@ -193,17 +187,18 @@ TEST_CASE(
"[time-traveling][array-schema][bug][sc35424][rest]") {
tiledb::test::VFSTestSetup vfs_test_setup;
auto array_uri{vfs_test_setup.array_uri("test_time_traveling_schema")};
auto ctx = vfs_test_setup.ctx();

// Test setup
create_array(array_uri);
write_first_fragment(array_uri);
create_array(ctx, array_uri);
write_first_fragment(ctx, array_uri);
auto timepoint = time_travel_destination();
add_attr_b(array_uri);
write_second_fragment(array_uri);
add_attr_b(ctx, array_uri);
write_second_fragment(ctx, array_uri);

// Check reads with and without time travel.
read_without_time_travel(array_uri);
read_with_time_travel(array_uri, timepoint);
read_without_time_travel(ctx, array_uri);
read_with_time_travel(ctx, array_uri, timepoint);
}

TEST_CASE(
Expand Down
2 changes: 1 addition & 1 deletion test/src/unit-capi-config.cc
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ void check_save_to_file() {
ss << "rest.curl.buffer_size 524288\n";
ss << "rest.curl.verbose false\n";
ss << "rest.http_compressor any\n";
ss << "rest.load_enumerations_on_array_open true\n";
ss << "rest.load_enumerations_on_array_open false\n";
ss << "rest.load_metadata_on_array_open true\n";
ss << "rest.load_non_empty_domain_on_array_open true\n";
ss << "rest.retry_count 25\n";
Expand Down
34 changes: 20 additions & 14 deletions test/src/unit-request-handlers.cc
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,10 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx {
TEST_CASE_METHOD(
HandleLoadArraySchemaRequestFx,
"tiledb_handle_load_array_schema_request - no enumerations",
"[request_handler][load_array_schema]") {
"[request_handler][load_array_schema][default]") {
auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP);

create_array();
REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", "false").ok());
auto schema_response =
call_handler(serialization::LoadArraySchemaRequest(cfg_), stype);
auto schema = std::get<0>(schema_response);
Expand All @@ -148,10 +147,11 @@ TEST_CASE_METHOD(
TEST_CASE_METHOD(
HandleLoadArraySchemaRequestFx,
"tiledb_handle_load_array_schema_request - load enumerations",
"[request_handler][load_array_schema][with-enumerations][default]") {
"[request_handler][load_array_schema][with-enumerations]") {
auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP);

create_array();
REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", "true").ok());
auto schema_response =
call_handler(serialization::LoadArraySchemaRequest(cfg_), stype);
auto schema = std::get<0>(schema_response);
Expand All @@ -173,6 +173,7 @@ TEST_CASE_METHOD(
"tiledb_handle_load_array_schema_request - multiple schemas",
"[request_handler][load_array_schema][schema-evolution]") {
auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP);
std::string load_enums = GENERATE("true", "false");

create_array();

Expand All @@ -181,18 +182,24 @@ TEST_CASE_METHOD(
all_schemas.push_back(schema_add_attribute("c"));
all_schemas.push_back(schema_add_attribute("d"));

REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", load_enums).ok());
auto schema_response =
call_handler(serialization::LoadArraySchemaRequest(cfg_), stype);
auto schema = std::get<0>(schema_response);
REQUIRE(schema->has_enumeration("enmr"));
REQUIRE(schema->get_loaded_enumeration_names().size() == 1);
REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr");
REQUIRE(schema->get_enumeration("enmr") != nullptr);
if (load_enums == "true") {
REQUIRE(schema->has_enumeration("enmr"));
REQUIRE(schema->get_loaded_enumeration_names().size() == 1);
REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr");
REQUIRE(schema->get_enumeration("enmr") != nullptr);
}
// The latest schema should be equal to the last applied evolution.
tiledb::test::schema_equiv(*schema, *all_schemas.back());

// Validate all array schemas returned from the request.
for (int i = 0; const auto& s : std::get<1>(schema_response)) {
// Validate schemas returned from the request in the order they were created.
auto r_all_schemas = std::get<1>(schema_response);
std::map<std::string, shared_ptr<ArraySchema>> resp(
r_all_schemas.begin(), r_all_schemas.end());
for (int i = 0; const auto& s : resp) {
tiledb::test::schema_equiv(*s.second, *all_schemas[i++]);
}
}
Expand Down Expand Up @@ -468,18 +475,17 @@ shared_ptr<ArraySchema> HandleLoadArraySchemaRequestFx::schema_add_attribute(
tiledb::ArraySchemaEvolution ase(ctx);
auto attr = tiledb::Attribute::create<int32_t>(ctx, attr_name);
ase.add_attribute(attr);
auto evolved = ase.ptr()->array_schema_evolution_->evolve_schema(schema_);
// Evolve and update the original schema member variable.
schema_ = ase.ptr()->array_schema_evolution_->evolve_schema(schema_);
// Apply the schema evolution.
Array::evolve_array_schema(
this->ctx_.resources(),
this->uri_,
ase.ptr()->array_schema_evolution_,
this->enc_key_);

// Update the original schema.
schema_ = evolved;
// Return the schema for validation.
return evolved;
// Return the new evolved schema for validation.
return schema_;
}

shared_ptr<ArraySchema> HandleLoadArraySchemaRequestFx::create_schema() {
Expand Down
4 changes: 4 additions & 0 deletions tiledb/api/c_api/config/config_api_external.h
Original file line number Diff line number Diff line change
Expand Up @@ -740,6 +740,10 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT;
* If true, array non empty domain will be loaded and sent to server together
* with the open array <br>
* **Default**: true
* - `rest.load_enumerations_on_array_open` <br>
* If true, enumerations will be loaded and sent to server together with
* the open array.
* **Default**: false
* - `rest.use_refactored_array_open` <br>
* If true, the new, experimental REST routes and APIs for opening an array
* will be used <br>
Expand Down
2 changes: 1 addition & 1 deletion tiledb/sm/array_schema/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ conclude(object_library)
#
commence(object_library array_schema_operations)
this_target_sources(array_schema_operations.cc)
this_target_object_libraries(array_schema generic_tile_io tiledb_crypto)
this_target_object_libraries(array array_schema generic_tile_io tiledb_crypto)
conclude(object_library)

add_test_subdirectory()
2 changes: 1 addition & 1 deletion tiledb/sm/config/config.cc
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ const std::string Config::REST_RETRY_DELAY_FACTOR = "1.25";
const std::string Config::REST_CURL_BUFFER_SIZE = "524288";
const std::string Config::REST_CAPNP_TRAVERSAL_LIMIT = "2147483648";
const std::string Config::REST_CURL_VERBOSE = "false";
const std::string Config::REST_LOAD_ENUMERATIONS_ON_ARRAY_OPEN = "true";
const std::string Config::REST_LOAD_ENUMERATIONS_ON_ARRAY_OPEN = "false";
const std::string Config::REST_LOAD_METADATA_ON_ARRAY_OPEN = "true";
const std::string Config::REST_LOAD_NON_EMPTY_DOMAIN_ON_ARRAY_OPEN = "true";
const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "false";
Expand Down
2 changes: 1 addition & 1 deletion tiledb/sm/cpp_api/config.h
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@ class Config {
* - `rest.load_enumerations_on_array_open` <br>
* If true, enumerations will be loaded and sent to server together with
* the open array.
* **Default**: true
* **Default**: false
* - `rest.load_metadata_on_array_open` <br>
* If true, array metadata will be loaded and sent to server together with
* the open array <br>
Expand Down
2 changes: 1 addition & 1 deletion tiledb/sm/serialization/array_schema.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2154,7 +2154,7 @@ LoadArraySchemaRequest deserialize_load_array_schema_request(
}

void serialize_load_array_schema_response(
const ArraySchema&, SerializationType, Buffer&) {
const Array&, SerializationType, Buffer&) {
throw ArraySchemaSerializationDisabledException();
}

Expand Down

0 comments on commit aa494ce

Please sign in to comment.