Merge pull request #7966 from romayalon/romy-5.15-backports

5.15 | Backports

Author: nimrod-becker

config.js

@@ -751,9 +751,19 @@ config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000;
 // of `manage_nsfs glacier restore`
 config.NSFS_GLACIER_RESTORE_INTERVAL = 15 * 60 * 1000;
 
-// NSFS_GLACIER_EXPIRY_INTERVAL indicates the interval between runs
-// of `manage_nsfs glacier expiry`
-config.NSFS_GLACIER_EXPIRY_INTERVAL = 12 * 60 * 60 * 1000;
+// NSFS_GLACIER_EXPIRY_RUN_TIME must be of the format hh:mm which specifies
+// when NooBaa should allow running glacier expiry process
+// NOTE: This will also be in the same timezone as specified in
+// NSFS_GLACIER_EXPIRY_TZ
+config.NSFS_GLACIER_EXPIRY_RUN_TIME = '03:00';
+
+// NSFS_GLACIER_EXPIRY_RUN_TIME_TOLERANCE_MINS configures the delay
+// tolerance in minutes.
+//
+// eg. If the expiry run time is set to 03:00 and the tolerance is
+// set to be 2 mins then the expiry can trigger till 03:02 (unless
+// already triggered between 03:00 - 03:02
+config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS = 2 * 60;
 
 /** @type {'UTC' | 'LOCAL'} */
 config.NSFS_GLACIER_EXPIRY_TZ = 'LOCAL';
@@ -812,6 +822,15 @@ config.NSFS_WHITELIST = [];
 config.NSFS_HEALTH_ENDPOINT_RETRY_COUNT = 3;
 config.NSFS_HEALTH_ENDPOINT_RETRY_DELAY = 10;
 
+
+/** @type {'file' | 'executable'} */
+config.NC_MASTER_KEYS_STORE_TYPE = 'file';
+// unless override in config.json, the default will be the config_dir/master_keys.json
+config.NC_MASTER_KEYS_FILE_LOCATION = '';
+config.NC_MASTER_KEYS_GET_EXECUTABLE = '';
+config.NC_MASTER_KEYS_PUT_EXECUTABLE = '';
+config.NC_MASTER_KEYS_MANAGER_REFRESH_THRESHOLD = 5 * 60 * 1000; // 5 minutes
+
 //Quota
 config.QUOTA_LOW_THRESHOLD = 80;
 config.QUOTA_MAX_OBJECTS = Number.MAX_SAFE_INTEGER;

docs/dev_guide/NonContainerizedDeveloperCustomizations.md

@@ -337,6 +337,74 @@ Example:
 3. systemctl restart noobaa_nsfs
 ```
 
+## 19. Set Master Keys Store type-
+**Description -** This flag will set the type of the master keys store used by NooBaa.
+
+**Configuration Key -** NC_MASTER_KEYS_STORE_TYPE
+
+**Type -** string
+
+**Default -**  'file'
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"NC_MASTER_KEYS_STORE_TYPE": 'executable'
+3. systemctl restart noobaa_nsfs
+```
+
+## 20. Set Master Keys File Location -
+**Description -** This flag will set the location of the master keys file used by NooBaa.
+
+**Configuration Key -** NC_MASTER_KEYS_FILE_LOCATION
+
+**Type -** string
+
+**Default -**  '/etc/noobaa.conf.d/master_keys.json'
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"NC_MASTER_KEYS_FILE_LOCATION": '/private/tmp/master_keys.json'
+3. systemctl restart noobaa_nsfs
+```
+
+## 21. Set Master Keys GET executable script -
+**Description -** This flag will set the location of the executable script for reading the master keys file used by NooBa.
+
+**Configuration Key -** NC_MASTER_KEYS_GET_EXECUTABLE
+
+**Type -** string
+
+**Default -**  undefined
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"NC_MASTER_KEYS_GET_EXECUTABLE": '/private/tmp/get_master_keys.sh'
+3. systemctl restart noobaa_nsfs
+```
+
+## 22. Set Master Keys PUT executable script -
+**Description -** This flag will set the location of the executable script for updating the master keys file used by NooBa.
+
+**Configuration Key -** NC_MASTER_KEYS_PUT_EXECUTABLE
+
+**Type -** string
+
+**Default -**  undefined
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"NC_MASTER_KEYS_PUT_EXECUTABLE": '/private/tmp/put_master_keys.sh'
+3. systemctl restart noobaa_nsfs
+```
+
 ## Config.json example 
 ```
 > cat /path/to/config_dir/config.json

src/cmd/manage_nsfs.js

@@ -25,6 +25,7 @@ const { print_usage } = require('../manage_nsfs/manage_nsfs_help_utils');
 const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES,
     LIST_ACCOUNT_FILTERS, LIST_BUCKET_FILTERS, GLACIER_ACTIONS } = require('../manage_nsfs/manage_nsfs_constants');
 const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
+const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
 
 function throw_cli_error(error_code, detail, event_arg) {
     const error_event = NSFS_CLI_ERROR_EVENT_MAP[error_code.code];
@@ -200,7 +201,7 @@ async function add_bucket(data) {
     const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
     const bucket_conf_path = get_config_file_path(buckets_dir_path, data.name);
     const exists = await native_fs_utils.is_path_exists(fs_context, bucket_conf_path);
-    if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name, {bucket: data.name});
+    if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, data.name, { bucket: data.name });
     data._id = mongo_utils.mongoObjectId();
     data.owner_account = account_id;
     const data_json = JSON.stringify(data);
@@ -209,7 +210,7 @@ async function add_bucket(data) {
     // for validating against the schema we need an object, hence we parse it back to object
     nsfs_schema_utils.validate_bucket_schema(JSON.parse(data_json));
     await native_fs_utils.create_config_file(fs_context, buckets_dir_path, bucket_conf_path, data_json);
-    write_stdout_response(ManageCLIResponse.BucketCreated, data_json, {bucket: data.name});
+    write_stdout_response(ManageCLIResponse.BucketCreated, data_json, { bucket: data.name });
 }
 
 /** verify_bucket_owner will check if the bucket_owner has an account
@@ -419,6 +420,7 @@ async function fetch_existing_account_data(target) {
             get_config_file_path(accounts_dir_path, target.name) :
             get_symlink_config_file_path(access_keys_dir_path, target.access_keys[0].access_key);
         source = await get_config_data(account_path, true);
+        source.access_keys = await nc_mkm.decrypt_access_keys(source);
     } catch (err) {
         dbg.log1('NSFS Manage command: Could not find account', target, err);
         if (err.code === 'ENOENT') {
@@ -452,17 +454,20 @@ async function add_account(data) {
         throw_cli_error(err_code, event_arg, {account: event_arg});
     }
     data._id = mongo_utils.mongoObjectId();
-    data = JSON.stringify(data);
+    const encrypted_account = await nc_mkm.encrypt_access_keys(data);
+    data.master_key_id = encrypted_account.master_key_id;
+    const encrypted_data = JSON.stringify(encrypted_account);
     // We take an object that was stringify
     // (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
     // for validating against the schema we need an object, hence we parse it back to object
-    nsfs_schema_utils.validate_account_schema(JSON.parse(data));
-    await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, data);
+    nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data));
+    await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, encrypted_data);
     await native_fs_utils._create_path(access_keys_dir_path, fs_context, config.BASE_MODE_CONFIG_DIR);
     await nb_native().fs.symlink(fs_context, account_config_relative_path, account_config_access_key_path);
-    write_stdout_response(ManageCLIResponse.AccountCreated, data, {account: event_arg});
+    write_stdout_response(ManageCLIResponse.AccountCreated, data, { account: event_arg });
 }
 
+
 async function update_account(data) {
     await validate_account_args(data, ACTIONS.UPDATE);
 
@@ -475,12 +480,14 @@ async function update_account(data) {
 
     if (!update_name && !update_access_key) {
         const account_config_path = get_config_file_path(accounts_dir_path, data.name);
-        data = JSON.stringify(data);
+        const encrypted_account = await nc_mkm.encrypt_access_keys(data);
+        data.master_key_id = encrypted_account.master_key_id;
+        const encrypted_data = JSON.stringify(encrypted_account);
         // We take an object that was stringify
         // (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
         // for validating against the schema we need an object, hence we parse it back to object
-        nsfs_schema_utils.validate_account_schema(JSON.parse(data));
-        await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, data);
+        nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data));
+        await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, encrypted_data);
         write_stdout_response(ManageCLIResponse.AccountUpdated, data);
         return;
     }
@@ -499,16 +506,21 @@ async function update_account(data) {
         const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists;
         throw_cli_error(err_code);
     }
-    data = JSON.stringify(_.omit(data, ['new_name', 'new_access_key']));
+    data = _.omit(data, ['new_name', 'new_access_key']);
+    const encrypted_account = await nc_mkm.encrypt_access_keys(data);
+    data.master_key_id = encrypted_account.master_key_id;
+    const encrypted_data = JSON.stringify(encrypted_account);
+    data = JSON.stringify(data);
+
     // We take an object that was stringify
     // (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
     // for validating against the schema we need an object, hence we parse it back to object
-    nsfs_schema_utils.validate_account_schema(JSON.parse(data));
+    nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data));
     if (update_name) {
-        await native_fs_utils.create_config_file(fs_context, accounts_dir_path, new_account_config_path, data);
+        await native_fs_utils.create_config_file(fs_context, accounts_dir_path, new_account_config_path, encrypted_data);
         await native_fs_utils.delete_config_file(fs_context, accounts_dir_path, cur_account_config_path);
     } else if (update_access_key) {
-        await native_fs_utils.update_config_file(fs_context, accounts_dir_path, cur_account_config_path, data);
+        await native_fs_utils.update_config_file(fs_context, accounts_dir_path, cur_account_config_path, encrypted_data);
     }
     // TODO: safe_unlink can be better but the current impl causing ELOOP - Too many levels of symbolic links
     // need to find a better way for atomic unlinking of symbolic links
@@ -560,6 +572,7 @@ async function get_account_status(data, show_secrets) {
             get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key) :
             get_config_file_path(accounts_dir_path, data.name);
         const config_data = await get_config_data(account_path, show_secrets);
+        if (config_data.access_keys) config_data.access_keys = await nc_mkm.decrypt_access_keys(config_data);
         write_stdout_response(ManageCLIResponse.AccountStatus, config_data);
     } catch (err) {
         if (_.isUndefined(data.name)) {
@@ -656,6 +669,7 @@ async function list_config_files(type, config_path, wide, show_secrets, filters)
             if (wide || should_filter) {
                 const full_path = path.join(config_path, entry.name);
                 const data = await get_config_data(full_path, show_secrets || should_filter);
+                if (data.access_keys) data.access_keys = await nc_mkm.decrypt_access_keys(data);
                 if (should_filter && !filter_list_item(type, data, filters)) return undefined;
                 // remove secrets on !show_secrets && should filter
                 return wide ? _.omit(data, show_secrets ? [] : ['access_keys']) : { name: entry.name.slice(0, entry.name.indexOf('.json')) };

src/manage_nsfs/manage_nsfs_glacier.js

@@ -69,12 +69,65 @@ async function process_expiry() {
     const fs_context = native_fs_utils.get_process_fs_context();
 
     await lock_and_run(fs_context, SCAN_LOCK, async () => {
-        if (!(await time_exceeded(fs_context, config.NSFS_GLACIER_EXPIRY_INTERVAL, GlacierBackend.EXPIRY_TIMESTAMP_FILE))) return;
+        const backend = getGlacierBackend();
+        if (
+            await backend.low_free_space() ||
+            await is_desired_time(
+                    fs_context,
+                    new Date(),
+                    config.NSFS_GLACIER_EXPIRY_RUN_TIME,
+                    config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS,
+                    GlacierBackend.EXPIRY_TIMESTAMP_FILE,
+            )
+        ) {
+            await backend.expiry(fs_context);
+            await record_current_time(fs_context, GlacierBackend.EXPIRY_TIMESTAMP_FILE);
+        }
+    });
+}
 
+/**
+ * is_desired_time returns true if the given time matches with
+ * the desired time or if 
+ * @param {nb.NativeFSContext} fs_context 
+ * @param {Date} current
+ * @param {string} desire time in format 'hh:mm'
+ * @param {number} delay_limit_mins
+ * @param {string} timestamp_file 
+ * @returns {Promise<boolean>}
+ */
+async function is_desired_time(fs_context, current, desire, delay_limit_mins, timestamp_file) {
+    const [desired_hour, desired_min] = desire.split(':').map(Number);
+    if (
+        isNaN(desired_hour) ||
+        isNaN(desired_min) ||
+        (desired_hour < 0 || desired_hour >= 24) ||
+        (desired_min < 0 || desired_min >= 60)
+    ) {
+        throw new Error('invalid desired_time - must be hh:mm');
+    }
 
-        await getGlacierBackend().expiry(fs_context);
-        await record_current_time(fs_context, GlacierBackend.EXPIRY_TIMESTAMP_FILE);
-    });
+    const min_time = get_tz_date(desired_hour, desired_min, 0, config.NSFS_GLACIER_EXPIRY_TZ);
+    const max_time = get_tz_date(desired_hour, desired_min + delay_limit_mins, 0, config.NSFS_GLACIER_EXPIRY_TZ);
+
+    if (current >= min_time && current <= max_time) {
+        try {
+            const { data } = await nb_native().fs.readFile(fs_context, path.join(config.NSFS_GLACIER_LOGS_DIR, timestamp_file));
+            const lastrun = new Date(data.toString());
+
+            // Last run should NOT be in this window
+            if (lastrun >= min_time && lastrun <= max_time) return false;
+        } catch (error) {
+            if (error.code === 'ENOENT') return true;
+            console.error('failed to read last run timestamp:', error, 'timestamp_file:', timestamp_file);
+
+            throw error;
+        }
+
+        return true;
+    }
+
+    return false;
 }
 
 /**
@@ -134,6 +187,31 @@ async function run_glacier_operation(fs_context, log_namespace, cb) {
     }
 }
 
+/**
+ * @param {number} hours
+ * @param {number} mins
+ * @param {number} secs
+ * @param {'UTC' | 'LOCAL'} tz
+ * @returns {Date}
+ */
+function get_tz_date(hours, mins, secs, tz) {
+    const date = new Date();
+
+    if (tz === 'UTC') {
+        date.setUTCHours(hours);
+        date.setUTCMinutes(hours);
+        date.setUTCSeconds(secs);
+        date.setUTCMilliseconds(0);
+    } else {
+        date.setHours(hours);
+        date.setMinutes(mins);
+        date.setSeconds(secs);
+        date.setMilliseconds(0);
+    }
+
+    return date;
+}
+
 /**
  * lock_and_run acquires a flock and calls the given callback after
  * acquiring the lock