Merge pull request #1991 from CVEProject/jd_remove_phone

Add database migrations, remove public org phones, and introduce disabled flag

Author: david-rocca

README.md

@@ -89,14 +89,30 @@ Download MongoDB Compass (MongoDB GUI)
 
 - https://www.mongodb.com/download-center/compass
 
-Create a `cve_dev` database in Compass. The collections will be automatically created when the API starts storing documents.
+Create a `cve_dev` database in Compass.
 
-You can populate the database with test data using:
+You can reset the local development database, run pending database migrations, and populate Cve, Cve-Id-Range, Cve-Id, Org, BaseOrg, User, BaseUser, and Glossary seed data using:
 
 ```sh
 npm run populate:dev
 ```
 
+To populate without automatically applying versioned migrations, use the following command, then run the migrations manually when ready:
+
+```sh
+npm run populate:dev -- --skip-migrations
+npm run db:migrate:dev
+```
+
+If the database only needs versioned data migrations without a local data reset, check the migration status and then apply pending migrations:
+
+```sh
+npm run db:migrate:status
+npm run db:migrate:dev
+```
+
+Migrations are managed by `migrate-mongo` using `migrate-mongo-config.js`. The applied migration records are stored in the `migrations_changelog` collection. Existing `migrate:*` scripts are legacy data sync scripts and are separate from these versioned database migrations.
+
 3. Start the node application
 
 In order to start a dev environment:

api-docs/openapi.json

@@ -3906,7 +3906,7 @@
           "Registry Organization"
         ],
         "summary": "Updates information about the organization specified by short name (accessible to Secretariat or same-organization Admin)",
-        "description": "  <h2>Access Control</h2>  <p>User must belong to an organization with the <b>Secretariat</b> role or be an <b>Admin</b> of the requested organization.</p>  <p>With Joint Approval required for the following fields:</p>  <h2>Expected Behavior</h2>  <b>This endpoint expects a full organization object in the request body.</b>  <p><b>Secretariat:</b> Updates any organization's information</p>  <p><b>Organization Admin:</b> Requests changes to its organization's information</p>  <ul>  <li>short_name</li>  <li>long_name</li>  <li>authority</li>  <li>aliases</li>  <li>oversees</li>  <li>top_level_root</li>  <li>charter_or_scope</li>  <li>product_list</li>  <li>disclosure_policy</li>  <li>contact_info.websites</li>  <li>contact_info.emails</li>  <li>contact_info.phone</li>  <li>partner_role_type</li>  <li>partner_country</li>  <li>advisory_locations</li>  <li>advisory_location_require_credentials</li>  <li>vulnerability_advisory_location_for_web_scraping</li>  <li>industry</li>  <li>tl_root_start_date</li>  <li>is_cna_discussion_list</li>  </ul>",
+        "description": "  <h2>Access Control</h2>  <p>User must belong to an organization with the <b>Secretariat</b> role or be an <b>Admin</b> of the requested organization.</p>  <p>With Joint Approval required for the following fields:</p>  <h2>Expected Behavior</h2>  <b>This endpoint expects a full organization object in the request body.</b>  <p><b>Secretariat:</b> Updates any organization's information</p>  <p><b>Organization Admin:</b> Requests changes to its organization's information</p>  <ul>  <li>short_name</li>  <li>long_name</li>  <li>authority</li>  <li>aliases</li>  <li>oversees</li>  <li>top_level_root</li>  <li>is_top_level_root</li>  <li>is_last_resort</li>  <li>charter_or_scope</li>  <li>product_list</li>  <li>disclosure_policy</li>  <li>contact_info.websites</li>  <li>contact_info.emails</li>  <li>partner_role_type</li>  <li>partner_country</li>  <li>advisory_locations</li>  <li>advisory_location_require_credentials</li>  <li>vulnerability_advisory_location_for_web_scraping</li>  <li>industry</li>  <li>tl_root_start_date</li>  <li>is_cna_discussion_list</li>  </ul>",
         "operationId": "registryOrgUpdateSingle",
         "parameters": [
           {
@@ -8193,4 +8193,4 @@
       }
     }
   }
-}
+}

datadump/pre-population/orgs.json

@@ -323,11 +323,6 @@
         }
     },
     {
-        "authority": {
-            "active_roles": [
-                "CNA"
-            ]
-        },
         "name": "Chase, May and Jones",
         "short_name": "sister_20",
         "time": {
@@ -338,4 +333,4 @@
             "id_quota": 1408
         }
     }
-]
+]

docker/README.md

@@ -76,34 +76,34 @@ If you do not require special configuration to access the internet, you can safe
 
 ### Pre-load Data
 
-Populate mongoDB with test data included in `datadump/pre-population/`
+Populate MongoDB with local development seed data included in `datadump/pre-population/`. This resets the local development database, runs pending database migrations, and populates Cve, Cve-Id-Range, Cve-Id, Org, BaseOrg, User, BaseUser, and Glossary.
 
-Run the command below using `populate:dev` or `populate:int` depending on your environment:
+Run the command below for development environments:
 ```
-docker-compose exec cveawg npm run populate:int
+docker-compose exec cveawg npm run populate:dev
 ```
 
 You should see the following:
 ```
 > cve-services@0.0.3 populate:dev /app
 > NODE_ENV=development node-dev src/scripts/populate.js
 
-2022-06-07 19:58:32 [info]: "Using NODE_ENV 'development' and app environment 'development'"
-2022-06-07 19:58:32 [info]: "Using dbName = cve_dev"
-2022-06-07 19:58:32 [info]: "Will try to connect to database cve_dev at docdb:27017"
 2022-06-07 19:58:32 [info]: "Successfully connected to database!"
-Are you sure you wish to pre-populate the database for the development environment? Doing so will drop the Cve, Cve-Id-Range, Cve-Id, User, Org collection(s) in the cve_dev database. (y/n) y
+Are you sure you wish to pre-populate the database for the development environment? Doing so will drop and rebuild the database, populate the Org, BaseOrg, User, BaseUser, Glossary collection(s), and run migrations in the cve_dev database. (y/n) y
 
+2022-06-07 19:58:37 [info]: "Dropping cve_dev database before population..."
+2022-06-07 19:58:37 [info]: "Successfully dropped cve_dev database."
 2022-06-07 19:58:37 [info]: "Populating Org collection..."
 2022-06-07 19:58:37 [info]: "Org populated!"
 2022-06-07 19:58:37 [info]: "Populating User collection..."
 2022-06-07 19:58:38 [info]: "User populated!"
-2022-06-07 19:58:38 [info]: "Populating Cve-Id-Range collection..."
-2022-06-07 19:58:38 [info]: "Populating Cve collection..."
-2022-06-07 19:58:38 [info]: "Populating Cve-Id collection..."
-2022-06-07 19:58:38 [info]: "Cve-Id-Range populated!"
-2022-06-07 19:58:38 [info]: "Cve populated!"
-2022-06-07 19:58:39 [info]: "Cve-Id populated!"
+2022-06-07 19:58:38 [info]: "Populating BaseOrg collection..."
+2022-06-07 19:58:38 [info]: "BaseOrg populated!"
+2022-06-07 19:58:38 [info]: "Populating BaseUser collection..."
+2022-06-07 19:58:38 [info]: "BaseUser populated!"
+2022-06-07 19:58:38 [info]: "Populating Glossary collection..."
+2022-06-07 19:58:39 [info]: "Glossary populated!"
+2022-06-07 19:58:39 [info]: "Successfully ran database migrations."
 2022-06-07 19:58:39 [info]: "Successfully populated the database!"
 
 ```
@@ -112,14 +112,6 @@ Are you sure you wish to pre-populate the database for the development environme
 
 The API token key is generated or stored differently depending on the value of the `NODE_ENV` environment variable.
 
-#### Integration
-
-For `integration` Node environments, the API key will be generate and saved to the `user-secret.txt` file when the database is populated.
-
-Display the key with:
-
-`docker-compose exec cveawg grep admin2 user-secret.txt`
-
 #### Development
 
 In `development` environments, the API is the value of the `LOCAL_KEY` variable in the `.docker-env` file.
@@ -189,7 +181,7 @@ docker compose -f docker-compose.mongo-cluster.yml up -d docdb docdb-read-1 docd
 Use this connection string from the host, including MongoDB Compass:
 
 ```text
-mongodb://localhost:27017,localhost:27018,localhost:27019/cve_test?replicaSet=rs0&readPreference=secondaryPreferred&retryWrites=false
+mongodb://127.0.0.1:27017,127.0.0.1:27018,127.0.0.1:27019/cve_test?replicaSet=rs0&readPreference=secondaryPreferred&retryWrites=false
 ```
 
 To run the same local replica topology with Mongo 8, override the image:
@@ -199,6 +191,8 @@ cd docker/
 MONGO_IMAGE=mongo:8.0 docker compose -f docker-compose.mongo-cluster.yml up -d --force-recreate docdb docdb-read-1 docdb-read-2 mongo-init
 ```
 
+The Mongo-only compose file overrides `GLIBC_TUNABLES` for compatibility with Docker Desktop Linux kernels 6.19 through 7.0.13. MongoDB's default TCMalloc configuration refuses to start on those kernel versions. The override can be removed after all development environments use kernel 7.0.14 or newer.
+
 If you already created the local volumes with Mongo 5, Mongo 8 may fail to start against those files. For a fresh Mongo 8 local cluster, remove the Mongo-only volumes first. This deletes local Mongo data for this compose file:
 
 ```bash

docker/docker-compose.mongo-cluster.yml

@@ -2,15 +2,18 @@ services:
   docdb:
     image: ${MONGO_IMAGE:-mongo:5.0}
     container_name: mongo
+    environment:
+      # MongoDB's default rseq setting is incompatible with Docker kernels 6.19 through 7.0.13.
+      GLIBC_TUNABLES: glibc.pthread.rseq=1
     ports:
-      - "27017:27017"
-      - "27018:27018"
-      - "27019:27019"
+      - "127.0.0.1:27017:27017"
+      - "127.0.0.1:27018:27018"
+      - "127.0.0.1:27019:27019"
     volumes:
       - docdb-host-data:/data/db
     command: ["mongod", "--replSet", "rs0", "--bind_ip_all", "--port", "27017"]
     healthcheck:
-      test: ["CMD-SHELL", "mongosh --quiet --port 27017 --eval 'db.adminCommand({ ping: 1 }).ok' || exit 1"]
+      test: ["CMD-SHELL", "mongosh --quiet --host 127.0.0.1 --port 27017 --eval 'db.adminCommand({ ping: 1 }).ok' || exit 1"]
       interval: 10s
       timeout: 10s
       retries: 12
@@ -19,6 +22,8 @@ services:
   docdb-read-1:
     image: ${MONGO_IMAGE:-mongo:5.0}
     container_name: mongo-read-1
+    environment:
+      GLIBC_TUNABLES: glibc.pthread.rseq=1
     network_mode: "service:docdb"
     depends_on:
       docdb:
@@ -27,7 +32,7 @@ services:
       - docdb-host-read-1-data:/data/db
     command: ["mongod", "--replSet", "rs0", "--bind_ip_all", "--port", "27018"]
     healthcheck:
-      test: ["CMD-SHELL", "mongosh --quiet --port 27018 --eval 'db.adminCommand({ ping: 1 }).ok' || exit 1"]
+      test: ["CMD-SHELL", "mongosh --quiet --host 127.0.0.1 --port 27018 --eval 'db.adminCommand({ ping: 1 }).ok' || exit 1"]
       interval: 10s
       timeout: 10s
       retries: 12
@@ -36,6 +41,8 @@ services:
   docdb-read-2:
     image: ${MONGO_IMAGE:-mongo:5.0}
     container_name: mongo-read-2
+    environment:
+      GLIBC_TUNABLES: glibc.pthread.rseq=1
     network_mode: "service:docdb"
     depends_on:
       docdb:
@@ -44,7 +51,7 @@ services:
       - docdb-host-read-2-data:/data/db
     command: ["mongod", "--replSet", "rs0", "--bind_ip_all", "--port", "27019"]
     healthcheck:
-      test: ["CMD-SHELL", "mongosh --quiet --port 27019 --eval 'db.adminCommand({ ping: 1 }).ok' || exit 1"]
+      test: ["CMD-SHELL", "mongosh --quiet --host 127.0.0.1 --port 27019 --eval 'db.adminCommand({ ping: 1 }).ok' || exit 1"]
       interval: 10s
       timeout: 10s
       retries: 12
@@ -62,13 +69,13 @@ services:
         condition: service_healthy
     command: >
       sh -c "
-        mongosh --host localhost --port 27017 --eval '
+        mongosh --host 127.0.0.1 --port 27017 --eval '
           const desiredConfig = {
             _id: \"rs0\",
             members: [
-              { _id: 0, host: \"localhost:27017\", priority: 2 },
-              { _id: 1, host: \"localhost:27018\", priority: 1 },
-              { _id: 2, host: \"localhost:27019\", priority: 1 }
+              { _id: 0, host: \"127.0.0.1:27017\", priority: 2 },
+              { _id: 1, host: \"127.0.0.1:27018\", priority: 1 },
+              { _id: 2, host: \"127.0.0.1:27019\", priority: 1 }
             ]
           };
 
@@ -86,7 +93,7 @@ services:
             rs.status();
             const currentConfig = rs.conf();
             if (configMatches(currentConfig)) {
-              print(\"Replica set already initialized with the expected localhost members.\");
+              print(\"Replica set already initialized with the expected IPv4 members.\");
             } else {
               print(\"Updating replica set members...\");
               rs.reconfig({ ...desiredConfig, version: currentConfig.version + 1 });

migrate-mongo-config.js

@@ -0,0 +1,77 @@
+require('dotenv').config()
+
+const config = require('config')
+
+const appEnv = process.env.NODE_ENV || 'development'
+
+function getConfiguredValue (key) {
+  const configPath = `${appEnv}.${key}`
+  return config.has(configPath) ? config.get(configPath) : null
+}
+
+function getMongoConnectionString () {
+  if (process.env.MONGO_CONN_STRING) {
+    return process.env.MONGO_CONN_STRING
+  }
+
+  const dbUser = process.env.MONGO_USER || getConfiguredValue('username')
+  const dbPassword = process.env.MONGO_PASSWORD || getConfiguredValue('password')
+  const dbHost = process.env.MONGO_HOST || getConfiguredValue('host')
+  const dbPort = process.env.MONGO_PORT || getConfiguredValue('port')
+  const dbName = getMongoDatabaseName()
+  const dbLoginPrepend = dbUser && dbPassword ? `${dbUser}:${dbPassword}@` : ''
+
+  if (process.env.useAWS) {
+    return `mongodb://${dbLoginPrepend}${dbHost}:${dbPort}/${dbName}?replicaSet=rs0&readPreference=secondaryPreferred&retryWrites=false`
+  }
+
+  return `mongodb://${dbLoginPrepend}${dbHost}:${dbPort}/${dbName}`
+}
+
+function getDatabaseNameFromUri () {
+  if (!process.env.MONGO_CONN_STRING) {
+    return null
+  }
+
+  try {
+    const uri = new URL(process.env.MONGO_CONN_STRING)
+    const databaseName = uri.pathname.replace(/^\//, '')
+    return databaseName || null
+  } catch (err) {
+    return null
+  }
+}
+
+function getMongoDatabaseName () {
+  return process.env.MONGO_DB_NAME || getDatabaseNameFromUri() || getConfiguredValue('database')
+}
+
+function getMongoConnectionOptions () {
+  if (process.env.useAWS) {
+    return {
+      authMechanism: 'SCRAM-SHA-1',
+      tls: false
+    }
+  }
+
+  return {
+    tls: false
+  }
+}
+
+const migrationConfig = {
+  mongodb: {
+    url: getMongoConnectionString(),
+    databaseName: getMongoDatabaseName(),
+    options: getMongoConnectionOptions()
+  },
+  migrationsDir: 'migrations',
+  changelogCollectionName: 'migrations_changelog',
+  lockCollectionName: 'migrations_changelog_lock',
+  lockTtl: 0,
+  migrationFileExtension: '.js',
+  useFileHash: false,
+  moduleSystem: 'commonjs'
+}
+
+module.exports = migrationConfig