Skip to content

Commit

Permalink
nsfs nc secret keys encryption
Browse files Browse the repository at this point in the history 
Signed-off-by: Romy <[email protected]>
(cherry picked from commit 1ed5310)
  • Loading branch information
@romayalon
romayalon committed Apr 10, 2024
1 parent 3588a15 commit e7931fe
Show file tree
Hide file tree
Showing 15 changed files with 640 additions and 39 deletions.
9 changes: 9 additions & 0 deletions config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -822,6 +822,15 @@ config.NSFS_WHITELIST = [];
config.NSFS_HEALTH_ENDPOINT_RETRY_COUNT = 3;
config.NSFS_HEALTH_ENDPOINT_RETRY_DELAY = 10;


/** @type {'file' | 'executable'} */
config.NC_MASTER_KEYS_STORE_TYPE = 'file';
// unless override in config.json, the default will be the config_dir/master_keys.json
config.NC_MASTER_KEYS_FILE_LOCATION = '';
config.NC_MASTER_KEYS_GET_EXECUTABLE = '';
config.NC_MASTER_KEYS_PUT_EXECUTABLE = '';
config.NC_MASTER_KEYS_MANAGER_REFRESH_THRESHOLD = 5 * 60 * 1000; // 5 minutes

//Quota
config.QUOTA_LOW_THRESHOLD = 80;
config.QUOTA_MAX_OBJECTS = Number.MAX_SAFE_INTEGER;
Expand Down
68 changes: 68 additions & 0 deletions docs/dev_guide/NonContainerizedDeveloperCustomizations.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,74 @@ Example:
3. systemctl restart noobaa_nsfs
```

## 19. Set Master Keys Store type-
**Description -** This flag will set the type of the master keys store used by NooBaa.

**Configuration Key -** NC_MASTER_KEYS_STORE_TYPE

**Type -** string

**Default -** 'file'
**Steps -**
```
1. Open /path/to/config_dir/config.json file.
2. Set the config key -
Example:
"NC_MASTER_KEYS_STORE_TYPE": 'executable'
3. systemctl restart noobaa_nsfs
```

## 20. Set Master Keys File Location -
**Description -** This flag will set the location of the master keys file used by NooBaa.

**Configuration Key -** NC_MASTER_KEYS_FILE_LOCATION

**Type -** string

**Default -** '/etc/noobaa.conf.d/master_keys.json'
**Steps -**
```
1. Open /path/to/config_dir/config.json file.
2. Set the config key -
Example:
"NC_MASTER_KEYS_FILE_LOCATION": '/private/tmp/master_keys.json'
3. systemctl restart noobaa_nsfs
```

## 21. Set Master Keys GET executable script -
**Description -** This flag will set the location of the executable script for reading the master keys file used by NooBa.

**Configuration Key -** NC_MASTER_KEYS_GET_EXECUTABLE

**Type -** string

**Default -** undefined
**Steps -**
```
1. Open /path/to/config_dir/config.json file.
2. Set the config key -
Example:
"NC_MASTER_KEYS_GET_EXECUTABLE": '/private/tmp/get_master_keys.sh'
3. systemctl restart noobaa_nsfs
```

## 22. Set Master Keys PUT executable script -
**Description -** This flag will set the location of the executable script for updating the master keys file used by NooBa.

**Configuration Key -** NC_MASTER_KEYS_PUT_EXECUTABLE

**Type -** string

**Default -** undefined
**Steps -**
```
1. Open /path/to/config_dir/config.json file.
2. Set the config key -
Example:
"NC_MASTER_KEYS_PUT_EXECUTABLE": '/private/tmp/put_master_keys.sh'
3. systemctl restart noobaa_nsfs
```

## Config.json example
```
> cat /path/to/config_dir/config.json
Expand Down
40 changes: 27 additions & 13 deletions src/cmd/manage_nsfs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const { print_usage } = require('../manage_nsfs/manage_nsfs_help_utils');
const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES,
LIST_ACCOUNT_FILTERS, LIST_BUCKET_FILTERS, GLACIER_ACTIONS } = require('../manage_nsfs/manage_nsfs_constants');
const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();

function throw_cli_error(error_code, detail, event_arg) {
const error_event = NSFS_CLI_ERROR_EVENT_MAP[error_code.code];
Expand Down Expand Up @@ -200,7 +201,7 @@ async function add_bucket(data) {
const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
const bucket_conf_path = get_config_file_path(buckets_dir_path, data.name);
const exists = await native_fs_utils.is_path_exists(fs_context, bucket_conf_path);
if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name, {bucket: data.name});
if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name, { bucket: data.name });
data._id = mongo_utils.mongoObjectId();
data.owner_account = account_id;
const data_json = JSON.stringify(data);
Expand All @@ -209,7 +210,7 @@ async function add_bucket(data) {
// for validating against the schema we need an object, hence we parse it back to object
nsfs_schema_utils.validate_bucket_schema(JSON.parse(data_json));
await native_fs_utils.create_config_file(fs_context, buckets_dir_path, bucket_conf_path, data_json);
write_stdout_response(ManageCLIResponse.BucketCreated, data_json, {bucket: data.name});
write_stdout_response(ManageCLIResponse.BucketCreated, data_json, { bucket: data.name });
}

/** verify_bucket_owner will check if the bucket_owner has an account
Expand Down Expand Up @@ -419,6 +420,7 @@ async function fetch_existing_account_data(target) {
get_config_file_path(accounts_dir_path, target.name) :
get_symlink_config_file_path(access_keys_dir_path, target.access_keys[0].access_key);
source = await get_config_data(account_path, true);
source.access_keys = await nc_mkm.decrypt_access_keys(source);
} catch (err) {
dbg.log1('NSFS Manage command: Could not find account', target, err);
if (err.code === 'ENOENT') {
Expand Down Expand Up @@ -452,17 +454,20 @@ async function add_account(data) {
throw_cli_error(err_code, event_arg, {account: event_arg});
}
data._id = mongo_utils.mongoObjectId();
data = JSON.stringify(data);
const encrypted_account = await nc_mkm.encrypt_access_keys(data);
data.master_key_id = encrypted_account.master_key_id;
const encrypted_data = JSON.stringify(encrypted_account);
// We take an object that was stringify
// (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
// for validating against the schema we need an object, hence we parse it back to object
nsfs_schema_utils.validate_account_schema(JSON.parse(data));
await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, data);
nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data));
await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, encrypted_data);
await native_fs_utils._create_path(access_keys_dir_path, fs_context, config.BASE_MODE_CONFIG_DIR);
await nb_native().fs.symlink(fs_context, account_config_relative_path, account_config_access_key_path);
write_stdout_response(ManageCLIResponse.AccountCreated, data, {account: event_arg});
write_stdout_response(ManageCLIResponse.AccountCreated, data, { account: event_arg });
}


async function update_account(data) {
await validate_account_args(data, ACTIONS.UPDATE);

Expand All @@ -475,12 +480,14 @@ async function update_account(data) {

if (!update_name && !update_access_key) {
const account_config_path = get_config_file_path(accounts_dir_path, data.name);
data = JSON.stringify(data);
const encrypted_account = await nc_mkm.encrypt_access_keys(data);
data.master_key_id = encrypted_account.master_key_id;
const encrypted_data = JSON.stringify(encrypted_account);
// We take an object that was stringify
// (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
// for validating against the schema we need an object, hence we parse it back to object
nsfs_schema_utils.validate_account_schema(JSON.parse(data));
await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, data);
nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data));
await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, encrypted_data);
write_stdout_response(ManageCLIResponse.AccountUpdated, data);
return;
}
Expand All @@ -499,16 +506,21 @@ async function update_account(data) {
const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists;
throw_cli_error(err_code);
}
data = JSON.stringify(_.omit(data, ['new_name', 'new_access_key']));
data = _.omit(data, ['new_name', 'new_access_key']);
const encrypted_account = await nc_mkm.encrypt_access_keys(data);
data.master_key_id = encrypted_account.master_key_id;
const encrypted_data = JSON.stringify(encrypted_account);
data = JSON.stringify(data);

// We take an object that was stringify
// (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
// for validating against the schema we need an object, hence we parse it back to object
nsfs_schema_utils.validate_account_schema(JSON.parse(data));
nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data));
if (update_name) {
await native_fs_utils.create_config_file(fs_context, accounts_dir_path, new_account_config_path, data);
await native_fs_utils.create_config_file(fs_context, accounts_dir_path, new_account_config_path, encrypted_data);
await native_fs_utils.delete_config_file(fs_context, accounts_dir_path, cur_account_config_path);
} else if (update_access_key) {
await native_fs_utils.update_config_file(fs_context, accounts_dir_path, cur_account_config_path, data);
await native_fs_utils.update_config_file(fs_context, accounts_dir_path, cur_account_config_path, encrypted_data);
}
// TODO: safe_unlink can be better but the current impl causing ELOOP - Too many levels of symbolic links
// need to find a better way for atomic unlinking of symbolic links
Expand Down Expand Up @@ -560,6 +572,7 @@ async function get_account_status(data, show_secrets) {
get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key) :
get_config_file_path(accounts_dir_path, data.name);
const config_data = await get_config_data(account_path, show_secrets);
if (config_data.access_keys) config_data.access_keys = await nc_mkm.decrypt_access_keys(config_data);
write_stdout_response(ManageCLIResponse.AccountStatus, config_data);
} catch (err) {
if (_.isUndefined(data.name)) {
Expand Down Expand Up @@ -656,6 +669,7 @@ async function list_config_files(type, config_path, wide, show_secrets, filters)
if (wide || should_filter) {
const full_path = path.join(config_path, entry.name);
const data = await get_config_data(full_path, show_secrets || should_filter);
if (data.access_keys) data.access_keys = await nc_mkm.decrypt_access_keys(data);
if (should_filter && !filter_list_item(type, data, filters)) return undefined;
// remove secrets on !show_secrets && should filter
return wide ? _.omit(data, show_secrets ? [] : ['access_keys']) : { name: entry.name.slice(0, entry.name.indexOf('.json')) };
Expand Down
Loading

0 comments on commit e7931fe

Please sign in to comment.