-
Notifications
You must be signed in to change notification settings - Fork 87
Mongo | Fix mongo_pool issues #9182
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -923,7 +923,7 @@ function get_associated_accounts(pool) { | |
|
||
function find_pool_by_name(req) { | ||
const name = req.rpc_params.name; | ||
const pool = req.system.pools_by_name[name]; | ||
const pool = req.system.pools_by_name?.[name]; | ||
if (!pool) { | ||
throw new RpcError('NO_SUCH_POOL', 'No such pool: ' + name); | ||
} | ||
|
@@ -1301,9 +1301,20 @@ function _is_cloud_pool(pool) { | |
return Boolean(pool.cloud_pool_info); | ||
} | ||
|
||
function _is_optimal_non_default_pool_id(pool) { | ||
return Boolean(pool.name === config.DEFAULT_POOL_NAME); | ||
} | ||
|
||
function get_optimal_non_default_pool_id(system) { | ||
return system.pools_by_name[config.DEFAULT_POOL_NAME]; | ||
} | ||
|
||
async function get_optimal_non_mongo_pool_id() { | ||
naveenpaul1 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
for (const pool of system_store.data.pools) { | ||
|
||
// skip backingstore_pool. | ||
if (_is_optimal_non_default_pool_id(pool)) { | ||
continue; | ||
} | ||
Comment on lines
+1304
to
+1317
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🛠️ Refactor suggestion Skip internal/backingstore pools robustly (structural markers + prefix); fix helper naming Name equality against DEFAULT_POOL_NAME is fragile; prefer explicit backingstore flags and legacy markers, with a DEFAULT_POOL_NAME prefix fallback. Also, the helper name implies an “id” but returns a boolean. Replace the helper and its usage: -function _is_optimal_non_default_pool_id(pool) {
- return Boolean(pool.name === config.DEFAULT_POOL_NAME);
-}
+function _is_backingstore_pool(pool) {
+ if (!pool) return false;
+ // Prefer explicit markers
+ if (pool.hosts_pool_info?.backingstore || pool.cloud_pool_info?.backingstore) return true;
+ // Legacy internal/mongo markers
+ if (pool.resource_type === 'INTERNAL' || pool.mongo_info) return true;
+ // Fallback by prefix (covers suffixed names)
+ const base = config.DEFAULT_POOL_NAME;
+ return !!(base && pool.name?.startsWith(base));
+}
@@
- // skip backingstore_pool.
- if (_is_optimal_non_default_pool_id(pool)) {
+ // skip backingstore/internal/mongo pools.
+ if (_is_backingstore_pool(pool)) {
continue;
} 🤖 Prompt for AI Agents
|
||
const aggr_nodes = await nodes_client.instance().aggregate_nodes_by_pool([pool.name], pool.system._id); | ||
const aggr_hosts = await nodes_client.instance().aggregate_hosts_by_pool([pool.name], pool.system._id); | ||
const { mode = '' } = get_pool_info(pool, aggr_nodes, aggr_hosts); | ||
|
@@ -1418,3 +1429,4 @@ exports.update_issues_report = update_issues_report; | |
exports.update_last_monitoring = update_last_monitoring; | ||
exports.calc_namespace_resource_mode = calc_namespace_resource_mode; | ||
exports.check_deletion_ownership = check_deletion_ownership; | ||
exports.get_optimal_non_default_pool_id = get_optimal_non_default_pool_id; |
Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
@@ -0,0 +1,27 @@ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
/* Copyright (C) 2025 NooBaa */ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
"use strict"; | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
const config = require('../../../../config.js'); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
async function run({ dbg, system_store }) { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
try { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
dbg.log0(`Starting monogo pool delete...`); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
const internal_mongo_pool = `${config.INTERNAL_STORAGE_POOL_NAME}-${system_store.data.systems[0]._id}`; | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
dbg.log0(`Internal mongo pool id is : ${internal_mongo_pool}`); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
const pool_ids = system_store.data.pools.filter(pool => pool.name === internal_mongo_pool); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
if (pool_ids.length > 0) { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
dbg.log0(`Removing default mongo pool: ${pool_ids[0]._id}`); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
await system_store.make_changes({ remove: { pools: [pool_ids[0]._id] }}); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
} else { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
dbg.log0('Removing mongo pool: Could not find the mongo pool...'); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
naveenpaul1 marked this conversation as resolved.
Show resolved
Hide resolved
Comment on lines
+9
to
+17
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🛠️ Refactor suggestion Make removal robust: avoid INTERNAL_STORAGE_POOL_NAME, handle multi-system, detect structurally, remove all matches Relying on config.INTERNAL_STORAGE_POOL_NAME (removed elsewhere), exact name equality, and only the first system is brittle. Detect internal/mongo pools structurally (mongo_info or resource_type === 'INTERNAL') with a safe name-prefix fallback (DEFAULT_POOL_NAME and legacy literal), and remove all matches across systems. - const internal_mongo_pool = `${config.INTERNAL_STORAGE_POOL_NAME}-${system_store.data.systems[0]._id}`;
- dbg.log0(`Internal mongo pool id is : ${internal_mongo_pool}`);
- const pool_ids = system_store.data.pools.filter(pool => pool.name === internal_mongo_pool);
- if (pool_ids.length > 0) {
- dbg.log0(`Removing default mongo pool: ${pool_ids[0]._id}`);
- await system_store.make_changes({ remove: { pools: [pool_ids[0]._id] }});
- } else {
- dbg.log0('Removing mongo pool: Could not find the mongo pool...');
- }
+ const systems = Array.isArray(system_store.data.systems) ? system_store.data.systems : [];
+ const sysIds = new Set(systems.map(s => String(s?._id)).filter(Boolean));
+
+ // Build candidate base names from current config and known legacy prefix
+ const bases = [config?.DEFAULT_POOL_NAME, 'system-internal-storage-pool']
+ .filter(x => typeof x === 'string' && x.length > 0);
+
+ const hasNameMatch = name => {
+ if (!name) return false;
+ if (bases.some(base => name.startsWith(base))) return true;
+ for (const base of bases) for (const id of sysIds) {
+ if (name === `${base}-${id}`) return true;
+ }
+ return false;
+ };
+
+ const pools = Array.isArray(system_store.data.pools) ? system_store.data.pools : [];
+ const pools_to_remove = pools.filter(p =>
+ p?.mongo_info || p?.resource_type === 'INTERNAL' || hasNameMatch(p?.name)
+ );
+ if (pools_to_remove.length > 0) {
+ dbg.log0(`Removing internal mongo pools: ${pools_to_remove.map(p => String(p._id)).join(', ')}`);
+ await system_store.make_changes({
+ remove: { pools: pools_to_remove.map(p => p._id) }
+ });
+ } else {
+ dbg.log0('Removing mongo pool: no internal mongo pools found');
+ } 📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
} catch (err) { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
dbg.error('Got error while removing mongo pool:', err); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
throw err; | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
Comment on lines
+6
to
+22
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💡 Verification agent 🧩 Analysis chainOrder of operations: ensure buckets/tiers are migrated before removal Directly removing pools can leave dangling references (tiers/buckets default resource). Confirm that buckets using the internal pool are migrated first (e.g., via update_all_buckets_default_pool) in the upgrade flow; otherwise removal may break invariants. Run to verify the migration step exists and is invoked by an upgrade script: 🏁 Script executed: #!/bin/bash
# Find the implementation and any invocations of update_all_buckets_default_pool
rg -n "update_all_buckets_default_pool" -A 3
# Inspect upgrade scripts that might call it
fd upgrade_scripts | xargs -I{} rg -n "update_all_buckets_default_pool" {} Length of output: 1179 Ensure buckets are migrated before removing the internal pool We didn’t find any call to • File: async function run({ dbg, system_store }) {
try {
dbg.log0(`Starting monogo pool delete...`);
const internal_mongo_pool = `${config.INTERNAL_STORAGE_POOL_NAME}-${system_store.data.systems[0]._id}`;
+ // Migrate every bucket with the old default pool to a new one
+ dbg.log0(`Migrating buckets off pool: ${internal_mongo_pool}`);
+ await system_store.call('update_all_buckets_default_pool', {
+ pool_name: internal_mongo_pool,
+ // TODO: replace with the target default pool name
+ new_pool_name: config.DEFAULT_STORAGE_POOL_NAME
+ });
dbg.log0(`Internal mongo pool id is : ${internal_mongo_pool}`);
const pool_ids = system_store.data.pools.filter(pool => pool.name === internal_mongo_pool);
if (pool_ids.length > 0) {
dbg.log0(`Removing default mongo pool: ${pool_ids[0]._id}`);
await system_store.make_changes({ remove: { pools: [pool_ids[0]._id] }});
} else {
dbg.log0('Removing mongo pool: Could not find the mongo pool...');
}
} catch (err) {
dbg.error('Got error while removing mongo pool:', err);
throw err;
}
} This ensures no dangling bucket references remain after pool deletion. 📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Plan to do more tests for these scenarios, and the coming PRs will consider this. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @naveenpaul1, understood! That's a sensible approach to handle the bucket migration separately with proper testing. Would you like me to create a GitHub issue to track the bucket migration work for the upcoming PRs? This would help ensure the migration logic is implemented before pools are removed in production deployments. ✏️ Learnings added
🧠 Learnings used
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
module.exports = { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
run, | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
description: 'Noobaa no longer support mongo_pool backingstore, Remove mongo pool', | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
}; |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🛠️ Refactor suggestion
get_optimal_non_default_pool_id may return undefined; add prefix fallback and null-safety
If the default pool name is suffixed, direct lookup fails. Add a safe fallback:
📝 Committable suggestion