Skip to content

Commit

Permalink
Merge pull request #200 from wtsi-npg/devel
Browse files Browse the repository at this point in the history 
merge to master for release 1.1.0
  • Loading branch information
@dozy
dozy authored Jan 8, 2018
2 parents f2f6f86 + f0fdcb7 commit 492e7a7
Show file tree
Hide file tree
Showing 6 changed files with 125 additions and 94 deletions.
54 changes: 29 additions & 25 deletions src/baton-do.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@
#include "config.h"
#include "baton.h"

static int debug_flag = 0;
static int help_flag = 0;
static int silent_flag = 0;
static int unbuffered_flag = 0;
static int unsafe_flag = 0;
static int verbose_flag = 0;
static int version_flag = 0;
static int debug_flag = 0;
static int help_flag = 0;
static int silent_flag = 0;
static int single_server_flag = 0;
static int unbuffered_flag = 0;
static int unsafe_flag = 0;
static int verbose_flag = 0;
static int version_flag = 0;

static size_t default_buffer_size = 1024 * 64 * 16 * 2;

Expand All @@ -44,16 +45,17 @@ int main(int argc, char *argv[]) {
while (1) {
static struct option long_options[] = {
// Flag options
{"debug", no_argument, &debug_flag, 1},
{"help", no_argument, &help_flag, 1},
{"silent", no_argument, &silent_flag, 1},
{"unbuffered", no_argument, &unbuffered_flag, 1},
{"unsafe", no_argument, &unsafe_flag, 1},
{"verbose", no_argument, &verbose_flag, 1},
{"version", no_argument, &version_flag, 1},
{"debug", no_argument, &debug_flag, 1},
{"help", no_argument, &help_flag, 1},
{"silent", no_argument, &silent_flag, 1},
{"single-server", no_argument, &single_server_flag, 1},
{"unbuffered", no_argument, &unbuffered_flag, 1},
{"unsafe", no_argument, &unsafe_flag, 1},
{"verbose", no_argument, &verbose_flag, 1},
{"version", no_argument, &version_flag, 1},
// Indexed options
{"file", required_argument, NULL, 'f'},
{"zone", required_argument, NULL, 'z'},
{"file", required_argument, NULL, 'f'},
{"zone", required_argument, NULL, 'z'},
{0, 0, 0, 0}
};

Expand Down Expand Up @@ -96,14 +98,15 @@ int main(int argc, char *argv[]) {
" Performs remote operations as described in the JSON\n"
" input file.\n"
""
" --file The JSON file describing the operations.\n"
" Optional, defaults to STDIN.\n"
" --silent Silence error messages.\n"
" --unbuffered Flush print operations for each JSON object.\n"
" --file The JSON file describing the operations.\n"
" Optional, defaults to STDIN.\n"
" --silent Silence error messages.\n"
" --single-server Only connect to a single iRODS server\n"
" --unbuffered Flush print operations for each JSON object.\n"

" --verbose Print verbose messages to STDERR.\n"
" --version Print the version number and exit.\n"
" --zone The zone to operate within. Optional.\n";
" --verbose Print verbose messages to STDERR.\n"
" --version Print the version number and exit.\n"
" --zone The zone to operate within. Optional.\n";

if (help_flag) {
printf("%s\n",help);
Expand All @@ -115,8 +118,9 @@ int main(int argc, char *argv[]) {
exit(0);
}

if (unbuffered_flag) flags = flags | FLUSH;
if (unsafe_flag) flags = flags | UNSAFE_RESOLVE;
if (single_server_flag) flags = flags | SINGLE_SERVER;
if (unbuffered_flag) flags = flags | FLUSH;
if (unsafe_flag) flags = flags | UNSAFE_RESOLVE;

if (debug_flag) set_log_threshold(DEBUG);
if (verbose_flag) set_log_threshold(NOTICE);
Expand Down
115 changes: 63 additions & 52 deletions src/baton-put.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@
#include "config.h"
#include "baton.h"

static int checksum_flag = 0;
static int debug_flag = 0;
static int help_flag = 0;
static int silent_flag = 0;
static int unbuffered_flag = 0;
static int unsafe_flag = 0;
static int verbose_flag = 0;
static int version_flag = 0;
static int checksum_flag = 0;
static int debug_flag = 0;
static int help_flag = 0;
static int silent_flag = 0;
static int single_server_flag = 0;
static int unbuffered_flag = 0;
static int unsafe_flag = 0;
static int verbose_flag = 0;
static int version_flag = 0;

static size_t default_buffer_size = 1024 * 64 * 16 * 2;
static size_t max_buffer_size = 1024 * 1024 * 1024;
Expand All @@ -47,17 +48,18 @@ int main(int argc, char *argv[]) {
while (1) {
static struct option long_options[] = {
// Flag options
{"checksum", no_argument, &checksum_flag, 1},
{"debug", no_argument, &debug_flag, 1},
{"help", no_argument, &help_flag, 1},
{"silent", no_argument, &silent_flag, 1},
{"unbuffered", no_argument, &unbuffered_flag, 1},
{"unsafe", no_argument, &unsafe_flag, 1},
{"verbose", no_argument, &verbose_flag, 1},
{"version", no_argument, &version_flag, 1},
{"checksum", no_argument, &checksum_flag, 1},
{"debug", no_argument, &debug_flag, 1},
{"help", no_argument, &help_flag, 1},
{"silent", no_argument, &silent_flag, 1},
{"single-server", no_argument, &single_server_flag, 1},
{"unbuffered", no_argument, &unbuffered_flag, 1},
{"unsafe", no_argument, &unsafe_flag, 1},
{"verbose", no_argument, &verbose_flag, 1},
{"version", no_argument, &version_flag, 1},
// Indexed options
{"file", required_argument, NULL, 'f'},
{"buffer-size", required_argument, NULL, 'b'},
{"file", required_argument, NULL, 'f'},
{"buffer-size", required_argument, NULL, 'b'},
{0, 0, 0, 0}
};

Expand Down Expand Up @@ -88,10 +90,10 @@ int main(int argc, char *argv[]) {
}
}

if (unsafe_flag) flags = flags | UNSAFE_RESOLVE;
if (unbuffered_flag) flags = flags | FLUSH;

if (checksum_flag) flags = flags | CALCULATE_CHECKSUM;
if (checksum_flag) flags = flags | CALCULATE_CHECKSUM;
if (single_server_flag) flags = flags | SINGLE_SERVER;
if (unsafe_flag) flags = flags | UNSAFE_RESOLVE;
if (unbuffered_flag) flags = flags | FLUSH;

const char *help =
"Name\n"
Expand All @@ -107,15 +109,16 @@ int main(int argc, char *argv[]) {
" Puts the contents of files into data objects described in a\n"
" JSON input file.\n"
""
" --buffer-size Set the transfer buffer size.\n"
" --checksum Calculate a checksum on the server side.\n"
" --file The JSON file describing the data objects.\n"
" Optional, defaults to STDIN.\n"
" --silent Silence error messages.\n"
" --unbuffered Flush print operations for each JSON object.\n"
" --unsafe Permit unsafe relative iRODS paths.\n"
" --verbose Print verbose messages to STDERR.\n"
" --version Print the version number and exit.\n";
" --buffer-size Set the transfer buffer size.\n"
" --checksum Calculate a checksum on the server side.\n"
" --file The JSON file describing the data objects.\n"
" Optional, defaults to STDIN.\n"
" --silent Silence error messages.\n"
" --single-server Only connect to a single iRODS server\n"
" --unbuffered Flush print operations for each JSON object.\n"
" --unsafe Permit unsafe relative iRODS paths.\n"
" --verbose Print verbose messages to STDERR.\n"
" --version Print the version number and exit.\n";

if (help_flag) {
printf("%s\n",help);
Expand All @@ -134,33 +137,41 @@ int main(int argc, char *argv[]) {
declare_client_name(argv[0]);
input = maybe_stdin(json_file);

if (buffer_size > max_buffer_size) {
logmsg(WARN, "Requested transfer buffer size %zu exceeds maximum of "
"%zu. Setting buffer size to %zu",
buffer_size, max_buffer_size, max_buffer_size);
buffer_size = max_buffer_size;
}
operation_args_t args = { .flags = flags,
.buffer_size = default_buffer_size,
.zone_name = zone_name };

if (buffer_size % 1024 != 0) {
size_t tmp = ((buffer_size / 1024) + 1) * 1024;
if (tmp > max_buffer_size) {
tmp = max_buffer_size;
}
int status;
if (flags & SINGLE_SERVER) {
logmsg(DEBUG, "Single-server mode, falling back to operation 'write'");

if (tmp > buffer_size) {
buffer_size = tmp;
logmsg(NOTICE, "Rounding transfer buffer size upwards from "
"%zu to %zu", buffer_size, tmp);
if (buffer_size > max_buffer_size) {
logmsg(WARN, "Requested transfer buffer size %zu exceeds maximum of "
"%zu. Setting buffer size to %zu",
buffer_size, max_buffer_size, max_buffer_size);
buffer_size = max_buffer_size;
}
}

logmsg(DEBUG, "Using a transfer buffer size of %zu bytes", buffer_size);
if (buffer_size % 1024 != 0) {
size_t tmp = ((buffer_size / 1024) + 1) * 1024;
if (tmp > max_buffer_size) {
tmp = max_buffer_size;
}

if (tmp > buffer_size) {
buffer_size = tmp;
logmsg(NOTICE, "Rounding transfer buffer size upwards from "
"%zu to %zu", buffer_size, tmp);
}
}

operation_args_t args = { .flags = flags,
.buffer_size = default_buffer_size,
.zone_name = zone_name };
logmsg(DEBUG, "Using a transfer buffer size of %zu bytes", buffer_size);
status = do_operation(input, baton_json_write_op, &args);
}
else {
status = do_operation(input, baton_json_put_op, &args);
}

int status = do_operation(input, baton_json_put_op, &args);
if (input != stdin) fclose(input);

if (status != 0) exit_status = 5;
Expand Down
4 changes: 4 additions & 0 deletions src/json.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,10 @@ int op_save_p(json_t *operation_args) {
return json_is_true(json_object_get(operation_args, JSON_OP_SAVE));
}

int op_single_server_p(json_t *operation_args) {
return json_is_true(json_object_get(operation_args, JSON_OP_SINGLE_SERVER));
}

int op_size_p(json_t *operation_args) {
return json_is_true(json_object_get(operation_args, JSON_OP_SIZE));
}
Expand Down
3 changes: 3 additions & 0 deletions src/json.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@
#define JSON_OP_RECURSE "recurse"
#define JSON_OP_REPLICATE "replicate"
#define JSON_OP_SAVE "save"
#define JSON_OP_SINGLE_SERVER "single-server"
#define JSON_OP_SIZE "size"
#define JSON_OP_TIMESTAMP "timestamp"
#define JSON_OP_PATH "path"
Expand Down Expand Up @@ -275,6 +276,8 @@ int op_replicate_p(json_t *operation_args);

int op_save_p(json_t *operation_args);

int op_single_server_p(json_t *operation_args);

int op_size_p(json_t *operation_args);

int op_timestamp_p(json_t *operation_args);
Expand Down
39 changes: 23 additions & 16 deletions src/operations.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,17 +150,18 @@ json_t *baton_json_dispatch_op(rodsEnv *env, rcComm_t *conn, json_t *envelope,
if (error->code != 0) goto error;

option_flags flags = args_copy.flags;
if (op_acl_p(args)) flags = flags | PRINT_ACL;
if (op_avu_p(args)) flags = flags | PRINT_AVU;
if (op_checksum_p(args)) flags = flags | PRINT_CHECKSUM;
if (op_contents_p(args)) flags = flags | PRINT_CONTENTS;
if (op_replicate_p(args)) flags = flags | PRINT_REPLICATE;
if (op_size_p(args)) flags = flags | PRINT_SIZE;
if (op_timestamp_p(args)) flags = flags | PRINT_TIMESTAMP;
if (op_recurse_p(args)) flags = flags | RECURSIVE;
if (op_force_p(args)) flags = flags | FORCE;
if (op_collection_p(args)) flags = flags | SEARCH_COLLECTIONS;
if (op_object_p(args)) flags = flags | SEARCH_OBJECTS;
if (op_acl_p(args)) flags = flags | PRINT_ACL;
if (op_avu_p(args)) flags = flags | PRINT_AVU;
if (op_checksum_p(args)) flags = flags | PRINT_CHECKSUM;
if (op_contents_p(args)) flags = flags | PRINT_CONTENTS;
if (op_replicate_p(args)) flags = flags | PRINT_REPLICATE;
if (op_size_p(args)) flags = flags | PRINT_SIZE;
if (op_timestamp_p(args)) flags = flags | PRINT_TIMESTAMP;
if (op_recurse_p(args)) flags = flags | RECURSIVE;
if (op_force_p(args)) flags = flags | FORCE;
if (op_collection_p(args)) flags = flags | SEARCH_COLLECTIONS;
if (op_object_p(args)) flags = flags | SEARCH_OBJECTS;
if (op_single_server_p(args)) flags = flags | SINGLE_SERVER;
args_copy.flags = flags;

if (has_operation(args)) {
Expand Down Expand Up @@ -222,7 +223,13 @@ json_t *baton_json_dispatch_op(rodsEnv *env, rcComm_t *conn, json_t *envelope,
}
else if (str_equals(op, JSON_PUT_OP, MAX_STR_LEN)) {
logmsg(DEBUG, "Dispatching to operation '%s'", op);
result = baton_json_put_op(env, conn, target, &args_copy, error);
if (args_copy.flags & SINGLE_SERVER) {
logmsg(DEBUG, "Single-server mode, falling back to operation 'write'");
result = baton_json_write_op(env, conn, target, &args_copy, error);
}
else {
result = baton_json_put_op(env, conn, target, &args_copy, error);
}
}
else if (str_equals(op, JSON_MOVE_OP, MAX_STR_LEN)) {
logmsg(DEBUG, "Dispatching to operation '%s'", op);
Expand Down Expand Up @@ -461,13 +468,13 @@ json_t *baton_json_write_op(rodsEnv *env, rcComm_t *conn, json_t *target,
if (error->code != 0) goto error;

size_t bsize = args->buffer_size;
logmsg(DEBUG, "Using a 'put' buffer size of %zu bytes", bsize);
logmsg(DEBUG, "Using a 'write' buffer size of %zu bytes", bsize);

FILE *in = fopen(file, "r");
if (!in) {
set_baton_error(error, errno,
"Failed to open '%s' for reading: error %d %s",
"README", errno, strerror(errno));
file, errno, strerror(errno));
goto error;
}

Expand All @@ -478,7 +485,7 @@ json_t *baton_json_write_op(rodsEnv *env, rcComm_t *conn, json_t *target,
if (status != 0) {
set_baton_error(error, errno,
"Failed to close '%s': error %d %s",
"README", errno, strerror(errno));
file, errno, strerror(errno));
goto error;
}

Expand Down Expand Up @@ -512,7 +519,7 @@ json_t *baton_json_put_op(rodsEnv *env, rcComm_t *conn, json_t *target,
if (status != 0) {
set_baton_error(error, errno,
"Failed to close '%s': error %d %s",
"README", errno, strerror(errno));
file, errno, strerror(errno));
goto error;
}

Expand Down
4 changes: 3 additions & 1 deletion src/operations.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,9 @@ typedef enum {
/** Flush output */
FLUSH = 1 << 17,
/** Force an operation */
FORCE = 1 << 18
FORCE = 1 << 18,
/** Avoid any operations that contact servers other than rodshost */
SINGLE_SERVER = 1 << 19
} option_flags;

typedef struct operation_args {
Expand Down

0 comments on commit 492e7a7

Please sign in to comment.