diff --git a/client.cpp b/client.cpp index af1802da..58537373 100755 --- a/client.cpp +++ b/client.cpp @@ -256,7 +256,14 @@ get_key_response client::get_key_for_conn(unsigned int command_index, unsigned i iter = obj_iter_type(m_config, command_index); *key_index = m_obj_gen->get_key_index(iter); - m_key_len = snprintf(m_key_buffer, sizeof(m_key_buffer)-1, "%s%llu", m_obj_gen->get_key_prefix(), *key_index); + + if (!m_config->data_import || m_config->generate_keys) { + m_obj_gen->generate_key(*key_index); + } else { + /* For SET command we already read a completes item (see create_set_request()) */ + if (command_index == GET_CMD_IDX) + dynamic_cast(m_obj_gen)->read_next_key(*key_index); + } return available_for_conn; } @@ -277,7 +284,7 @@ bool client::create_arbitrary_request(unsigned int command_index, struct timeval get_key_response res = get_key_for_conn(command_index, conn_id, &key_index); /* If key not available for this connection, we have a bug of sending partial request */ assert(res == available_for_conn); - cmd_size += m_connections[conn_id]->send_arbitrary_command(arg, m_key_buffer, m_key_len); + cmd_size += m_connections[conn_id]->send_arbitrary_command(arg, m_obj_gen->get_key(), m_obj_gen->get_key_len()); } else if (arg->type == data_type) { unsigned int value_len; const char *value = m_obj_gen->get_value(0, &value_len); @@ -313,7 +320,7 @@ bool client::create_set_request(struct timeval& timestamp, unsigned int conn_id) unsigned int value_len; const char *value = m_obj_gen->get_value(key_index, &value_len); - m_connections[conn_id]->send_set_command(×tamp, m_key_buffer, m_key_len, + m_connections[conn_id]->send_set_command(×tamp, m_obj_gen->get_key(), m_obj_gen->get_key_len(), value, value_len, m_obj_gen->get_expiry(), m_config->data_offset); } @@ -328,7 +335,7 @@ bool client::create_get_request(struct timeval& timestamp, unsigned int conn_id) return false; if (res == available_for_conn) { - m_connections[conn_id]->send_get_command(×tamp, m_key_buffer, m_key_len, m_config->data_offset); + m_connections[conn_id]->send_get_command(×tamp, m_obj_gen->get_key(), m_obj_gen->get_key_len(), m_config->data_offset); } return true; @@ -346,7 +353,7 @@ bool client::create_mget_request(struct timeval& timestamp, unsigned int conn_id /* Not supported in cluster mode */ assert(res == available_for_conn); - m_keylist->add_key(m_key_buffer, m_key_len); + m_keylist->add_key(m_obj_gen->get_key(), m_obj_gen->get_key_len()); } m_connections[conn_id]->send_mget_command(×tamp, m_keylist); @@ -378,6 +385,11 @@ void client::create_request(struct timeval timestamp, unsigned int conn_id) // are we set or get? this depends on the ratio else if (m_set_ratio_count < m_config->ratio.a) { + /* Before we can create a SET request, we need to read the next imported item */ + if (m_config->data_import) { + dynamic_cast(m_obj_gen)->read_next_item(); + } + if (!create_set_request(timestamp, conn_id)) return; @@ -482,57 +494,51 @@ unsigned long long int verify_client::get_errors(void) return m_errors; } -void verify_client::create_request(struct timeval timestamp, unsigned int conn_id) -{ - // TODO: Refactor client::create_request so this can be unified. - if (m_set_ratio_count < m_config->ratio.a) { - // Prepare a GET request that will be compared against a previous - // SET request. - data_object *obj = m_obj_gen->get_object(obj_iter_type(m_config, 0)); - unsigned int key_len; - const char *key = obj->get_key(&key_len); +bool verify_client::create_wait_request(struct timeval& timestamp, unsigned int conn_id) { + // Nothing to do + return true; +} + +bool verify_client::create_set_request(struct timeval& timestamp, unsigned int conn_id) { + unsigned long long key_index; + get_key_response res = get_key_for_conn(SET_CMD_IDX, conn_id, &key_index); + if (res == not_available) + return false; + + if (res == available_for_conn) { unsigned int value_len; - const char *value = obj->get_value(&value_len); + const char *value = m_obj_gen->get_value(key_index, &value_len); - m_connections[conn_id]->send_verify_get_command(×tamp, key, key_len, - value, value_len, obj->get_expiry(), + m_connections[conn_id]->send_verify_get_command(×tamp, m_obj_gen->get_key(), m_obj_gen->get_key_len(), + value, value_len, m_config->data_offset); + } - m_set_ratio_count++; - } else if (m_get_ratio_count < m_config->ratio.b) { - // We don't really care about GET operations, all we do here is keep - // the object generator synced. - int iter = obj_iter_type(m_config, 2); - - if (m_config->multi_key_get > 0) { - unsigned int keys_count; + return true; +} - keys_count = m_config->ratio.b - m_get_ratio_count; - if ((int)keys_count > m_config->multi_key_get) - keys_count = m_config->multi_key_get; - m_keylist->clear(); - while (m_keylist->get_keys_count() < keys_count) { - unsigned int keylen; - const char *key = m_obj_gen->get_key(iter, &keylen); +bool verify_client::create_get_request(struct timeval& timestamp, unsigned int conn_id) { + // Just Keep object generator synced + unsigned long long key_index; + get_key_for_conn(GET_CMD_IDX, conn_id, &key_index); - assert(key != NULL); - assert(keylen > 0); + return true; +} - m_keylist->add_key(key, keylen); - } +bool verify_client::create_mget_request(struct timeval& timestamp, unsigned int conn_id) { + // Just Keep object generator synced + unsigned long long key_index; + unsigned int keys_count = m_config->ratio.b - m_get_ratio_count; + if ((int)keys_count > m_config->multi_key_get) + keys_count = m_config->multi_key_get; - m_get_ratio_count += keys_count; - } else { - unsigned int keylen; - m_obj_gen->get_key(iter, &keylen); - m_get_ratio_count++; - } + m_keylist->clear(); + for (unsigned int i = 0; i < keys_count; i++) { - // We don't really send this request, but need to count it to be in sync. - m_reqs_processed++; - } else { - m_get_ratio_count = m_set_ratio_count = 0; + get_key_for_conn(GET_CMD_IDX, conn_id, &key_index); } + + return true; } void verify_client::handle_response(unsigned int conn_id, struct timeval timestamp, diff --git a/client.h b/client.h index cff0519d..13813877 100755 --- a/client.h +++ b/client.h @@ -45,7 +45,6 @@ class client; class client_group; struct benchmark_config; class object_generator; -class data_object; #define SET_CMD_IDX 0 #define GET_CMD_IDX 2 @@ -61,10 +60,6 @@ class client : public connections_manager { bool m_initialized; bool m_end_set; - // key buffer - char m_key_buffer[250]; - int m_key_len; - // test related benchmark_config* m_config; object_generator* m_obj_gen; @@ -93,10 +88,10 @@ class client : public connections_manager { virtual get_key_response get_key_for_conn(unsigned int command_index, unsigned int conn_id, unsigned long long* key_index); virtual bool create_arbitrary_request(unsigned int command_index, struct timeval& timestamp, unsigned int conn_id); - bool create_wait_request(struct timeval& timestamp, unsigned int conn_id); - bool create_set_request(struct timeval& timestamp, unsigned int conn_id); - bool create_get_request(struct timeval& timestamp, unsigned int conn_id); - bool create_mget_request(struct timeval& timestamp, unsigned int conn_id); + virtual bool create_wait_request(struct timeval& timestamp, unsigned int conn_id); + virtual bool create_set_request(struct timeval& timestamp, unsigned int conn_id); + virtual bool create_get_request(struct timeval& timestamp, unsigned int conn_id); + virtual bool create_mget_request(struct timeval& timestamp, unsigned int conn_id); // client manager api's unsigned long long get_reqs_processed() { @@ -185,7 +180,10 @@ class verify_client : public client { unsigned long long int m_errors; virtual bool finished(void); - virtual void create_request(struct timeval timestamp, unsigned int conn_id); + virtual bool create_wait_request(struct timeval& timestamp, unsigned int conn_id); + virtual bool create_set_request(struct timeval& timestamp, unsigned int conn_id); + virtual bool create_get_request(struct timeval& timestamp, unsigned int conn_id); + virtual bool create_mget_request(struct timeval& timestamp, unsigned int conn_id); virtual void handle_response(unsigned int conn_id, struct timeval timestamp, request *request, protocol_response *response); public: diff --git a/cluster_client.cpp b/cluster_client.cpp index ba17ad6e..0cdb9bdb 100644 --- a/cluster_client.cpp +++ b/cluster_client.cpp @@ -312,7 +312,7 @@ get_key_response cluster_client::get_key_for_conn(unsigned int command_index, un // first check if we already have a key in the pool if (!m_key_index_pools[conn_id]->empty()) { *key_index = m_key_index_pools[conn_id]->front(); - m_key_len = snprintf(m_key_buffer, sizeof(m_key_buffer)-1, "%s%llu", m_obj_gen->get_key_prefix(), *key_index); + m_obj_gen->generate_key(*key_index); m_key_index_pools[conn_id]->pop(); return available_for_conn; @@ -321,11 +321,13 @@ get_key_response cluster_client::get_key_for_conn(unsigned int command_index, un // generate key client::get_key_for_conn(command_index, conn_id, key_index); - unsigned int hslot = calc_hslot_crc16_cluster(m_key_buffer, m_key_len); + unsigned int hslot = calc_hslot_crc16_cluster(m_obj_gen->get_key(), m_obj_gen->get_key_len()); // check if the key match for this connection if (m_slot_to_shard[hslot] == conn_id) { - benchmark_debug_log("%s generated key=[%.*s] for itself\n", m_connections[conn_id]->get_readable_id(), m_key_len, m_key_buffer); + benchmark_debug_log("%s generated key=[%.*s] for itself\n", + m_connections[conn_id]->get_readable_id(), + m_obj_gen->get_key_len(), m_obj_gen->get_key()); return available_for_conn; } @@ -347,7 +349,10 @@ get_key_response cluster_client::get_key_for_conn(unsigned int command_index, un return not_available; // store command and key for the other connection - benchmark_debug_log("%s generated key=[%.*s] for %s\n", m_connections[conn_id]->get_readable_id(), m_key_len, m_key_buffer, m_connections[other_conn_id]->get_readable_id()); + benchmark_debug_log("%s generated key=[%.*s] for %s\n", + m_connections[conn_id]->get_readable_id(), + m_obj_gen->get_key_len(), m_obj_gen->get_key(), + m_connections[other_conn_id]->get_readable_id()); key_idx_pool->push(command_index); key_idx_pool->push(*key_index); diff --git a/obj_gen.cpp b/obj_gen.cpp index d5a02b4c..cd1f228f 100644 --- a/obj_gen.cpp +++ b/obj_gen.cpp @@ -382,37 +382,9 @@ unsigned long long object_generator::get_key_index(int iter) return k; } -const char* object_generator::get_key(int iter, unsigned int *len) -{ - unsigned int l; - m_key_index = get_key_index(iter); - - // format key - l = snprintf(m_key_buffer, sizeof(m_key_buffer)-1, - "%s%llu", m_key_prefix, m_key_index); - if (len != NULL) *len = l; - - return m_key_buffer; -} - -data_object* object_generator::get_object(int iter) -{ - // compute key - (void) get_key(iter, NULL); - - // compute value - unsigned int new_size = 0; - get_value(m_key_index, &new_size); - - // compute expiry - unsigned int expiry = get_expiry(); - - // set object - m_object.set_key(m_key_buffer, strlen(m_key_buffer)); - m_object.set_value(m_value_buffer, new_size); - m_object.set_expiry(expiry); - - return &m_object; +void object_generator::generate_key(unsigned long long key_index) { + m_key_len = snprintf(m_key_buffer, sizeof(m_key_buffer)-1, "%s%llu", m_key_prefix, key_index); + m_key = m_key_buffer; } const char* object_generator::get_key_prefix() { @@ -461,67 +433,6 @@ unsigned int object_generator::get_expiry() { /////////////////////////////////////////////////////////////////////////// -data_object::data_object() : - m_key(NULL), m_key_len(0), - m_value(NULL), m_value_len(0), - m_expiry(0) -{ -} - -data_object::~data_object() -{ - clear(); -} - -void data_object::clear(void) -{ - m_key = NULL; - m_key_len = 0; - m_value = NULL; - m_value_len = 0; - m_expiry = 0; -} - -void data_object::set_key(const char* key, unsigned int key_len) -{ - m_key = key; - m_key_len = key_len; -} - -const char* data_object::get_key(unsigned int* key_len) -{ - assert(key_len != NULL); - *key_len = m_key_len; - - return m_key; -} - -void data_object::set_value(const char* value, unsigned int value_len) -{ - m_value = value; - m_value_len = value_len; -} - -const char* data_object::get_value(unsigned int *value_len) -{ - assert(value_len != NULL); - *value_len = m_value_len; - - return m_value; -} - -void data_object::set_expiry(unsigned int expiry) -{ - m_expiry = expiry; -} - -unsigned int data_object::get_expiry(void) -{ - return m_expiry; -} - -/////////////////////////////////////////////////////////////////////////// - imported_keylist::imported_keylist(const char *filename) : m_filename(filename) { @@ -623,18 +534,8 @@ import_object_generator* import_object_generator::clone(void) return new import_object_generator(*this); } -const char* import_object_generator::get_key(int iter, unsigned int *len) -{ - if (m_keys == NULL) { - return object_generator::get_key(iter, len); - } else { - unsigned int k = get_key_index(iter) - 1; - return m_keys->get(k, len); - } -} - -data_object* import_object_generator::get_object(int iter) -{ +void import_object_generator::read_next_item() { + /* Used by SET command to read an item that includes KEY, VALUE, EXPIRE */ memcache_item *i = m_reader.read_item(); if (i == NULL && m_reader.is_eof()) { @@ -648,26 +549,34 @@ data_object* import_object_generator::get_object(int iter) } m_cur_item = i; - m_object.set_value(m_cur_item->get_data(), m_cur_item->get_nbytes() - 2); - if (m_keys != NULL) { - m_object.set_key(m_cur_item->get_key(), m_cur_item->get_nkey()); - } else { - unsigned int tmplen; - const char *tmpkey = object_generator::get_key(iter, &tmplen); - m_object.set_key(tmpkey, tmplen); - } + m_key = m_cur_item->get_key(); + m_key_len = m_cur_item->get_nkey(); +} + +void import_object_generator::read_next_key(unsigned long long key_index) { + /* Used by GET command that needs only a KEY */ + m_key = m_keys->get(key_index-1, (unsigned int *)&m_key_len); +} + +const char* import_object_generator::get_value(unsigned long long key_index, unsigned int *len) { + assert(m_cur_item != NULL); + + *len = m_cur_item->get_nbytes() - 2; + return m_cur_item->get_data(); +} + +unsigned int import_object_generator::get_expiry() { + assert(m_cur_item != NULL); // compute expiry - int expiry = 0; + unsigned int expiry = 0; if (!m_no_expiry) { if (m_expiry_max > 0) { expiry = random_range(m_expiry_min, m_expiry_max); } else { expiry = m_cur_item->get_exptime(); } - m_object.set_expiry(expiry); } - return &m_object; + return expiry; } - diff --git a/obj_gen.h b/obj_gen.h index e1c1cc0b..b2cc3cca 100644 --- a/obj_gen.h +++ b/obj_gen.h @@ -50,26 +50,6 @@ class gaussian_noise: public random_generator { double m_spare; }; -class data_object { -protected: - const char *m_key; - unsigned int m_key_len; - const char *m_value; - unsigned int m_value_len; - unsigned int m_expiry; -public: - data_object(); - ~data_object(); - - void clear(void); - void set_key(const char* key, unsigned int key_len); - const char* get_key(unsigned int* key_len); - void set_value(const char* value, unsigned int value_len); - const char* get_value(unsigned int* value_len); - void set_expiry(unsigned int expiry); - unsigned int get_expiry(void); -}; - #define OBJECT_GENERATOR_KEY_ITERATORS 2 /* number of iterators */ #define OBJECT_GENERATOR_KEY_SET_ITER 1 #define OBJECT_GENERATOR_KEY_GET_ITER 0 @@ -98,12 +78,13 @@ class object_generator { unsigned long long m_key_max; double m_key_stddev; double m_key_median; - data_object m_object; std::vector m_next_key; unsigned long long m_key_index; char m_key_buffer[250]; + const char *m_key; + int m_key_len; char *m_value_buffer; int m_random_fd; gaussian_noise m_random; @@ -132,14 +113,14 @@ class object_generator { void set_key_range(unsigned long long key_min, unsigned long long key_max); void set_key_distribution(double key_stddev, double key_median); void set_random_seed(int seed); - unsigned long long get_key_index(int iter); - virtual const char* get_key(int iter, unsigned int *len); - virtual data_object* get_object(int iter); + void generate_key(unsigned long long key_index); + const char * get_key() { return m_key; } + int get_key_len() { return m_key_len; } const char * get_key_prefix(); - const char* get_value(unsigned long long key_index, unsigned int *len); - unsigned int get_expiry(); + virtual const char* get_value(unsigned long long key_index, unsigned int *len); + virtual unsigned int get_expiry(); }; class imported_keylist; @@ -175,8 +156,11 @@ class import_object_generator : public object_generator { virtual ~import_object_generator(); virtual import_object_generator* clone(void); - virtual const char* get_key(int iter, unsigned int *len); - virtual data_object* get_object(int iter); + void read_next_item(void); + void read_next_key(unsigned long long key_index); + + virtual const char* get_value(unsigned long long key_index, unsigned int *len); + virtual unsigned int get_expiry(); bool open_file(void); }; diff --git a/shard_connection.cpp b/shard_connection.cpp index bcf331ad..2da4f2b0 100644 --- a/shard_connection.cpp +++ b/shard_connection.cpp @@ -611,11 +611,10 @@ void shard_connection::send_mget_command(struct timeval* sent_time, const keylis } void shard_connection::send_verify_get_command(struct timeval* sent_time, const char *key, int key_len, - const char *value, int value_len, int expiry, unsigned int offset) { + const char *value, int value_len, unsigned int offset) { int cmd_size = 0; - benchmark_debug_log("GET key=[%.*s] value_len=%u expiry=%u\n", - key_len, key, value_len, expiry); + benchmark_debug_log("Verify GET key=[%.*s] value_len=%u\n", key_len, key, value_len); cmd_size = m_protocol->write_command_get(key, key_len, offset); push_req(new verify_request(rt_get, cmd_size, sent_time, 1, key, key_len, value, value_len)); diff --git a/shard_connection.h b/shard_connection.h index 9a06f735..fa95030e 100644 --- a/shard_connection.h +++ b/shard_connection.h @@ -98,7 +98,7 @@ class shard_connection { const char *key, int key_len, unsigned int offset); void send_mget_command(struct timeval* sent_time, const keylist* key_list); void send_verify_get_command(struct timeval* sent_time, const char *key, int key_len, - const char *value, int value_len, int expiry, unsigned int offset); + const char *value, int value_len, unsigned int offset); int send_arbitrary_command(const command_arg *arg); int send_arbitrary_command(const command_arg *arg, const char *val, int val_len); void send_arbitrary_command_end(size_t command_index, struct timeval* sent_time, int cmd_size);