perf(NODE-6246): Use buffer pool for ObjectId to significantly improve memory usage

src/objectid.ts

@@ -4,6 +4,30 @@ import { type InspectFn, defaultInspect } from './parser/utils';
 import { ByteUtils } from './utils/byte_utils';
 import { NumberUtils } from './utils/number_utils';
 
+let currentPool: Uint8Array | null = null;
+let poolSize = 1000; // Default: Hold 1000 ObjectId buffers in a pool
+let currentPoolOffset = 0;
+
+/**
+ * Retrieves a ObjectId pool and offset. This function may create a new ObjectId buffer pool and reset the pool offset
+ * @internal
+ */
+function getPool(): [Uint8Array, number] {
+  if (!currentPool || currentPoolOffset + 12 > currentPool.byteLength) {
+    currentPool = ByteUtils.allocateUnsafe(poolSize * 12);
+    currentPoolOffset = 0;
+  }
+  return [currentPool, currentPoolOffset];
+}
+
+/**
+ * Increments the pool offset by 12 bytes
+ * @internal
+ */
+function incrementPool(): void {
+  currentPoolOffset += 12;
+}
+
 // Regular expression that checks for hex value
 const checkForHexRegExp = new RegExp('^[0-9a-fA-F]{24}$');
 
@@ -37,8 +61,26 @@ export class ObjectId extends BSONValue {
 
   static cacheHexString: boolean;
 
-  /** ObjectId Bytes @internal */
-  private buffer!: Uint8Array;
+  /**
+   * The size of the current ObjectId buffer pool.
+   */
+  static get poolSize(): number {
+    return poolSize;
+  }
+
+  static set poolSize(size: number) {
+    const iSize = Math.ceil(size); // Ensure new pool size is an integer
+    if (iSize <= 0) {
+      throw new BSONError('poolSize must be a positive integer greater than 0');
+    }
+    poolSize = iSize;
+  }
+
+  /** ObjectId buffer pool pointer @internal */
+  private pool: Uint8Array;
+  /** Buffer pool offset @internal */
+  private offset: number;
+
   /** ObjectId hexString cache @internal */
   private __id?: string;
 
@@ -73,6 +115,13 @@ export class ObjectId extends BSONValue {
    * @param inputId - A 12 byte binary Buffer.
    */
   constructor(inputId: Uint8Array);
+  /**
+   * Create ObjectId from a large binary Buffer. Only 12 bytes starting from the offset are used.
+   * @internal
+   * @param inputId - A 12 byte binary Buffer.
+   * @param inputIndex - The offset to start reading the inputId buffer.
+   */
+  constructor(inputId: Uint8Array, inputIndex?: number);
   /** To generate a new ObjectId, use ObjectId() with no argument. */
   constructor();
   /**
@@ -86,7 +135,10 @@ export class ObjectId extends BSONValue {
    *
    * @param inputId - An input value to create a new ObjectId from.
    */
-  constructor(inputId?: string | number | ObjectId | ObjectIdLike | Uint8Array) {
+  constructor(
+    inputId?: string | number | ObjectId | ObjectIdLike | Uint8Array,
+    inputIndex?: number
+  ) {
     super();
     // workingId is set based on type of input and whether valid id exists for the input
     let workingId;
@@ -103,17 +155,28 @@ export class ObjectId extends BSONValue {
       workingId = inputId;
     }
 
+    const [pool, offset] = getPool();
+
     // The following cases use workingId to construct an ObjectId
     if (workingId == null || typeof workingId === 'number') {
       // The most common use case (blank id, new objectId instance)
       // Generate a new id
-      this.buffer = ObjectId.generate(typeof workingId === 'number' ? workingId : undefined);
-    } else if (ArrayBuffer.isView(workingId) && workingId.byteLength === 12) {
-      // If intstanceof matches we can escape calling ensure buffer in Node.js environments
-      this.buffer = ByteUtils.toLocalBufferType(workingId);
+      ObjectId.generate(typeof workingId === 'number' ? workingId : undefined, pool, offset);
+    } else if (ArrayBuffer.isView(workingId)) {
+      if (workingId.byteLength === 12) {
+        inputIndex = 0;
+      } else if (
+        typeof inputIndex !== 'number' ||
+        inputIndex < 0 ||
+        workingId.byteLength < inputIndex + 12 ||
+        isNaN(inputIndex)
+      ) {
+        throw new BSONError('Buffer length must be 12 or a valid offset must be specified');
+      }
+      for (let i = 0; i < 12; i++) pool[offset + i] = workingId[inputIndex + i];
     } else if (typeof workingId === 'string') {
       if (workingId.length === 24 && checkForHexRegExp.test(workingId)) {
-        this.buffer = ByteUtils.fromHex(workingId);
+        pool.set(ByteUtils.fromHex(workingId), offset);
       } else {
         throw new BSONError(
           'input must be a 24 character hex string, 12 byte Uint8Array, or an integer'
@@ -124,20 +187,32 @@ export class ObjectId extends BSONValue {
     }
     // If we are caching the hex string
     if (ObjectId.cacheHexString) {
-      this.__id = ByteUtils.toHex(this.id);
+      this.__id = ByteUtils.toHex(pool, offset, offset + 12);
     }
+    // Increment pool offset once we have completed initialization
+    this.pool = pool;
+    this.offset = offset;
+    incrementPool();
+  }
+
+  /** ObjectId bytes @internal */
+  get buffer(): Uint8Array {
+    return this.id;
   }
 
   /**
    * The ObjectId bytes
    * @readonly
    */
   get id(): Uint8Array {
-    return this.buffer;
+    return this.pool.subarray(this.offset, this.offset + 12);
   }
 
   set id(value: Uint8Array) {
-    this.buffer = value;
+    if (value.byteLength !== 12) {
+      throw new BSONError('input must be a 12 byte Uint8Array');
+    }
+    this.pool.set(value, this.offset);
     if (ObjectId.cacheHexString) {
       this.__id = ByteUtils.toHex(value);
     }
@@ -149,7 +224,7 @@ export class ObjectId extends BSONValue {
       return this.__id;
     }
 
-    const hexString = ByteUtils.toHex(this.id);
+    const hexString = ByteUtils.toHex(this.pool, this.offset, this.offset + 12);
 
     if (ObjectId.cacheHexString && !this.__id) {
       this.__id = hexString;
@@ -170,34 +245,38 @@ export class ObjectId extends BSONValue {
    * Generate a 12 byte id buffer used in ObjectId's
    *
    * @param time - pass in a second based timestamp.
+   * @param buffer - Optionally pass in a buffer instance.
+   * @param offset - Optionally pass in a buffer offset.
    */
-  static generate(time?: number): Uint8Array {
+  static generate(time?: number, buffer?: Uint8Array, offset: number = 0): Uint8Array {
     if ('number' !== typeof time) {
       time = Math.floor(Date.now() / 1000);
     }
 
     const inc = ObjectId.getInc();
-    const buffer = ByteUtils.allocateUnsafe(12);
+    if (!buffer) {
+      buffer = ByteUtils.allocateUnsafe(12);
+    }
 
     // 4-byte timestamp
-    NumberUtils.setInt32BE(buffer, 0, time);
+    NumberUtils.setInt32BE(buffer, offset, time);
 
     // set PROCESS_UNIQUE if yet not initialized
     if (PROCESS_UNIQUE === null) {
       PROCESS_UNIQUE = ByteUtils.randomBytes(5);
     }
 
     // 5-byte process unique
-    buffer[4] = PROCESS_UNIQUE[0];
-    buffer[5] = PROCESS_UNIQUE[1];
-    buffer[6] = PROCESS_UNIQUE[2];
-    buffer[7] = PROCESS_UNIQUE[3];
-    buffer[8] = PROCESS_UNIQUE[4];
+    buffer[offset + 4] = PROCESS_UNIQUE[0];
+    buffer[offset + 5] = PROCESS_UNIQUE[1];
+    buffer[offset + 6] = PROCESS_UNIQUE[2];
+    buffer[offset + 7] = PROCESS_UNIQUE[3];
+    buffer[offset + 8] = PROCESS_UNIQUE[4];
 
     // 3-byte counter
-    buffer[11] = inc & 0xff;
-    buffer[10] = (inc >> 8) & 0xff;
-    buffer[9] = (inc >> 16) & 0xff;
+    buffer[offset + 11] = inc & 0xff;
+    buffer[offset + 10] = (inc >> 8) & 0xff;
+    buffer[offset + 9] = (inc >> 16) & 0xff;
 
     return buffer;
   }
@@ -239,9 +318,16 @@ export class ObjectId extends BSONValue {
     }
 
     if (ObjectId.is(otherId)) {
-      return (
-        this.buffer[11] === otherId.buffer[11] && ByteUtils.equals(this.buffer, otherId.buffer)
-      );
+      if (otherId.pool && typeof otherId.offset === 'number') {
+        for (let i = 11; i >= 0; i--) {
+          if (this.pool[this.offset + i] !== otherId.pool[otherId.offset + i]) {
+            return false;
+          }
+        }
+        return true;
+      }
+      // If otherId does not have pool and offset, fallback to buffer comparison for compatibility
+      return ByteUtils.equals(this.buffer, otherId.buffer);
     }
 
     if (typeof otherId === 'string') {
@@ -260,7 +346,7 @@ export class ObjectId extends BSONValue {
   /** Returns the generation date (accurate up to the second) that this ID was generated. */
   getTimestamp(): Date {
     const timestamp = new Date();
-    const time = NumberUtils.getUint32BE(this.buffer, 0);
+    const time = NumberUtils.getUint32BE(this.pool, this.offset);
     timestamp.setTime(Math.floor(time) * 1000);
     return timestamp;
   }
@@ -272,18 +358,20 @@ export class ObjectId extends BSONValue {
 
   /** @internal */
   serializeInto(uint8array: Uint8Array, index: number): 12 {
-    uint8array[index] = this.buffer[0];
-    uint8array[index + 1] = this.buffer[1];
-    uint8array[index + 2] = this.buffer[2];
-    uint8array[index + 3] = this.buffer[3];
-    uint8array[index + 4] = this.buffer[4];
-    uint8array[index + 5] = this.buffer[5];
-    uint8array[index + 6] = this.buffer[6];
-    uint8array[index + 7] = this.buffer[7];
-    uint8array[index + 8] = this.buffer[8];
-    uint8array[index + 9] = this.buffer[9];
-    uint8array[index + 10] = this.buffer[10];
-    uint8array[index + 11] = this.buffer[11];
+    const pool = this.pool;
+    const offset = this.offset;
+    uint8array[index] = pool[offset];
+    uint8array[index + 1] = pool[offset + 1];
+    uint8array[index + 2] = pool[offset + 2];
+    uint8array[index + 3] = pool[offset + 3];
+    uint8array[index + 4] = pool[offset + 4];
+    uint8array[index + 5] = pool[offset + 5];
+    uint8array[index + 6] = pool[offset + 6];
+    uint8array[index + 7] = pool[offset + 7];
+    uint8array[index + 8] = pool[offset + 8];
+    uint8array[index + 9] = pool[offset + 9];
+    uint8array[index + 10] = pool[offset + 10];
+    uint8array[index + 11] = pool[offset + 11];
     return 12;
   }
 
@@ -293,7 +381,7 @@ export class ObjectId extends BSONValue {
    * @param time - an integer number representing a number of seconds.
    */
   static createFromTime(time: number): ObjectId {
-    const buffer = ByteUtils.allocate(12);
+    const buffer = ByteUtils.allocateUnsafe(12);
     for (let i = 11; i >= 4; i--) buffer[i] = 0;
     // Encode time into first 4 bytes
     NumberUtils.setInt32BE(buffer, 0, time);

src/parser/deserializer.ts

@@ -263,9 +263,7 @@ function deserializeObject(
       value = ByteUtils.toUTF8(buffer, index, index + stringSize - 1, shouldValidateKey);
       index = index + stringSize;
     } else if (elementType === constants.BSON_DATA_OID) {
-      const oid = ByteUtils.allocateUnsafe(12);
-      for (let i = 0; i < 12; i++) oid[i] = buffer[index + i];
-      value = new ObjectId(oid);
+      value = new ObjectId(buffer, index);
       index = index + 12;
     } else if (elementType === constants.BSON_DATA_INT && promoteValues === false) {
       value = new Int32(NumberUtils.getInt32LE(buffer, index));

src/utils/byte_utils.ts

@@ -30,7 +30,7 @@ export type ByteUtils = {
   /** Create a Uint8Array from a hex string */
   fromHex: (hex: string) => Uint8Array;
   /** Create a lowercase hex string from bytes */
-  toHex: (buffer: Uint8Array) => string;
+  toHex: (buffer: Uint8Array, start?: number, end?: number) => string;
   /** Create a string from utf8 code units, fatal=true will throw an error if UTF-8 bytes are invalid, fatal=false will insert replacement characters */
   toUTF8: (buffer: Uint8Array, start: number, end: number, fatal: boolean) => string;
   /** Get the utf8 code unit count from a string if it were to be transformed to utf8 */

src/utils/node_byte_utils.ts

@@ -124,8 +124,8 @@ export const nodeJsByteUtils = {
     return Buffer.from(hex, 'hex');
   },
 
-  toHex(buffer: Uint8Array): string {
-    return nodeJsByteUtils.toLocalBufferType(buffer).toString('hex');
+  toHex(buffer: Uint8Array, start?: number, end?: number): string {
+    return nodeJsByteUtils.toLocalBufferType(buffer).toString('hex', start, end);
   },
 
   toUTF8(buffer: Uint8Array, start: number, end: number, fatal: boolean): string {

src/utils/web_byte_utils.ts

@@ -170,8 +170,10 @@ export const webByteUtils = {
     return Uint8Array.from(buffer);
   },
 
-  toHex(uint8array: Uint8Array): string {
-    return Array.from(uint8array, byte => byte.toString(16).padStart(2, '0')).join('');
+  toHex(uint8array: Uint8Array, start?: number, end?: number): string {
+    return Array.from(uint8array.subarray(start, end), byte =>
+      byte.toString(16).padStart(2, '0')
+    ).join('');
   },
 
   toUTF8(uint8array: Uint8Array, start: number, end: number, fatal: boolean): string {

test/node/bson_test.js

@@ -20,7 +20,11 @@ const MaxKey = BSON.MaxKey;
 const BSONError = BSON.BSONError;
 const { BinaryParser } = require('./tools/binary_parser');
 const vm = require('vm');
-const { assertBuffersEqual, isBufferOrUint8Array } = require('./tools/utils');
+const {
+  assertBuffersEqual,
+  isBufferOrUint8Array,
+  assertDeepEqualsWithObjectId
+} = require('./tools/utils');
 const { inspect } = require('util');
 
 /**
@@ -707,7 +711,7 @@ describe('BSON', function () {
     expect(serialized_data).to.deep.equal(serialized_data2);
 
     var doc2 = b.deserialize(serialized_data);
-    expect(doc).to.deep.equal(doc2);
+    assertDeepEqualsWithObjectId(doc, doc2);
     expect(doc2.dbref.oid.toHexString()).to.deep.equal(oid.toHexString());
     done();
   });
@@ -1001,7 +1005,7 @@ describe('BSON', function () {
 
     var deserialized_data = BSON.deserialize(serialized_data);
     expect(doc.b).to.deep.equal(deserialized_data.b);
-    expect(doc).to.deep.equal(deserialized_data);
+    assertDeepEqualsWithObjectId(doc, deserialized_data);
     done();
   });
 
@@ -1213,7 +1217,7 @@ describe('BSON', function () {
 
     var doc2 = BSON.deserialize(serialized_data);
 
-    expect(doc).to.deep.equal(doc2);
+    assertDeepEqualsWithObjectId(doc, doc2);
     done();
   });
 

test/node/bson_type_classes.test.ts

@@ -18,6 +18,7 @@ import {
   UUID,
   BSONValue
 } from '../register-bson';
+import { assertDeepEqualsWithObjectId } from './tools/utils';
 import * as vm from 'node:vm';
 
 const BSONTypeClasses = [
@@ -128,7 +129,13 @@ describe('BSON Type classes common interfaces', () => {
           ctx.ObjectId = ObjectId;
         }
         vm.runInNewContext(`module.exports.result = ${bsonValue.inspect()}`, ctx);
-        expect(ctx.module.exports.result).to.deep.equal(bsonValue);
+
+        if (ctx.ObjectId) {
+          // Since ObjectId uses a pool we expect offset to be different
+          assertDeepEqualsWithObjectId(ctx.module.exports.result, bsonValue);
+        } else {
+          expect(ctx.module.exports.result).to.deep.equal(bsonValue);
+        }
       });
     }
   });