fix: support profiles and shard split thresholds in mfs

Adds IPIP-499 profiles and shard split thresholds to MFS operations

Author: achingbrain

packages/mfs/package.json

@@ -53,7 +53,7 @@
     "interface-datastore": "^9.0.2",
     "ipfs-unixfs": "^12.0.0",
     "ipfs-unixfs-exporter": "^15.0.2",
-    "ipfs-unixfs-importer": "^16.0.1",
+    "ipfs-unixfs-importer": "^16.1.4",
     "it-map": "^3.1.4",
     "multiformats": "^13.4.1"
   },

packages/mfs/test/rm.spec.ts

@@ -105,7 +105,8 @@ describe('rm', () => {
       content: Uint8Array.from([0, 1, 2, 3, 4])
     }], blockstore, {
       wrapWithDirectory: true,
-      shardSplitThresholdBytes
+      shardSplitThresholdBytes,
+      shardSplitStrategy: 'links-bytes'
     }))
 
     if (importResult == null) {
@@ -122,19 +123,22 @@ describe('rm', () => {
 
     // create the same shard with unixfs command
     await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), `${dirPath}/file-1.txt`, {
-      shardSplitThresholdBytes
+      shardSplitThresholdBytes,
+      shardSplitStrategy: 'links-bytes'
     })
 
     await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'directory')
 
     await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), `${dirPath}/file-2.txt`, {
-      shardSplitThresholdBytes
+      shardSplitThresholdBytes,
+      shardSplitStrategy: 'links-bytes'
     })
 
     await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
 
     await fs.rm(`${dirPath}/file-2.txt`, {
-      shardSplitThresholdBytes
+      shardSplitThresholdBytes,
+      shardSplitStrategy: 'links-bytes'
     })
 
     const dirStats = await fs.stat(dirPath)
@@ -151,7 +155,8 @@ describe('rm', () => {
       content: Uint8Array.from([0, 1, 2, 3, 4])
     }], blockstore, {
       wrapWithDirectory: true,
-      shardSplitThresholdBytes
+      shardSplitThresholdBytes,
+      shardSplitStrategy: 'links-bytes'
     }))
 
     if (importResult == null) {
@@ -168,19 +173,22 @@ describe('rm', () => {
 
     // create the same shard with unixfs command
     await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), `${dirPath}/file-1.txt`, {
-      shardSplitThresholdBytes
+      shardSplitThresholdBytes,
+      shardSplitStrategy: 'links-bytes'
     })
 
     await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
 
     await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), `${dirPath}/file-2.txt`, {
-      shardSplitThresholdBytes
+      shardSplitThresholdBytes,
+      shardSplitStrategy: 'links-bytes'
     })
 
     await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory')
 
     await fs.rm(`${dirPath}/file-2.txt`, {
-      shardSplitThresholdBytes
+      shardSplitThresholdBytes,
+      shardSplitStrategy: 'links-bytes'
     })
 
     const dirStats = await fs.stat(dirPath)
@@ -204,7 +212,8 @@ describe('rm', () => {
 
     // remove the file that caused the sub-shard to be created and the CID should be the same as the importer
     await fs.rm(`${dirPath}/${fileName}`, {
-      shardSplitThresholdBytes: 1
+      shardSplitThresholdBytes: 1,
+      shardSplitStrategy: 'links-bytes'
     })
 
     // should still be a shard
@@ -231,7 +240,8 @@ describe('rm', () => {
 
     // remove the file that caused the sub-shard to be created and the CID should be the same as the importer
     await fs.rm(`${dirPath}/${fileName}`, {
-      shardSplitThresholdBytes: 1
+      shardSplitThresholdBytes: 1,
+      shardSplitStrategy: 'links-bytes'
     })
 
     // should still be a shard

packages/unixfs/package.json

@@ -73,11 +73,10 @@
     "@libp2p/logger": "^6.0.5",
     "@libp2p/utils": "^7.0.5",
     "@multiformats/murmur3": "^2.1.8",
-    "hamt-sharding": "^3.0.6",
     "interface-blockstore": "^6.0.1",
     "ipfs-unixfs": "^12.0.0",
     "ipfs-unixfs-exporter": "^15.0.2",
-    "ipfs-unixfs-importer": "^16.0.1",
+    "ipfs-unixfs-importer": "^16.1.4",
     "it-all": "^3.0.9",
     "it-first": "^3.0.9",
     "it-glob": "^3.0.4",

packages/unixfs/src/commands/utils/add-link.ts

@@ -7,7 +7,6 @@ import { sha256 } from 'multiformats/hashes/sha2'
 // @ts-expect-error no types
 import SparseArray from 'sparse-array'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
-import { DEFAULT_SHARD_SPLIT_THRESHOLD_BYTES } from '../../constants.ts'
 import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from '../../errors.js'
 import { wrapHash } from './consumable-hash.js'
 import { hamtBucketBits, hamtHashFn } from './hamt-constants.js'
@@ -22,8 +21,7 @@ import type { Directory } from './cid-to-directory.js'
 import type { GetStore, PutStore } from '../../unixfs.js'
 import type { PBNode, PBLink } from '@ipld/dag-pb'
 import type { AbortOptions } from '@libp2p/interface'
-import type { ImportResult } from 'ipfs-unixfs-importer'
-import type { Version } from 'multiformats/cid'
+import type { ImporterOptions, ImportResult } from 'ipfs-unixfs-importer'
 
 const log = logger('helia:unixfs:components:utils:add-link')
 
@@ -32,10 +30,8 @@ export interface AddLinkResult {
   cid: CID
 }
 
-export interface AddLinkOptions extends AbortOptions {
+export interface AddLinkOptions extends AbortOptions, Pick<ImporterOptions, 'profile' | 'shardSplitThresholdBytes' | 'shardSplitStrategy' | 'shardFanoutBits' | 'cidVersion'> {
   allowOverwriting?: boolean
-  shardSplitThresholdBytes?: number
-  cidVersion?: Version
 }
 
 export async function addLink (parent: Directory, child: Required<PBLink>, blockstore: GetStore & PutStore, options: AddLinkOptions): Promise<AddLinkResult> {
@@ -55,7 +51,7 @@ export async function addLink (parent: Directory, child: Required<PBLink>, block
 
   const result = await addToDirectory(parent, child, blockstore, options)
 
-  if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes ?? DEFAULT_SHARD_SPLIT_THRESHOLD_BYTES, options)) {
+  if (await isOverShardThreshold(result.node, blockstore, options)) {
     log('converting directory to sharded directory')
 
     const converted = await convertToShardedDirectory(result, blockstore)