fix(unixfs): align HAMT sharding threshold with JS implementation

HAMT sharding threshold comparison was historically implemented as
`>` in JS and `>=` in Go:

- JS: https://github.com/ipfs/helia/blob/005c2a7/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts#L31
- Go: https://github.com/ipfs/boxo/blob/319662c/ipld/unixfs/io/directory.go#L438

This inconsistency meant a directory exactly at the 256 KiB threshold
would stay basic in JS but convert to HAMT in Go, producing different
CIDs for the same input.

This commit changes Go to use `>` (matching JS), so a directory exactly
at the threshold now stays as a basic (flat) directory. This aligns
cross-implementation behavior for CID determinism per IPIP-499.

Also adds SizeEstimationMode to MkdirOpts so MFS directories respect
the configured estimation mode instead of always using the global default.

Author: lidel

CHANGELOG.md

@@ -33,6 +33,7 @@ The following emojis are used to highlight certain changes:
 
 ### Fixed
 
+- 🛠 `ipld/unixfs/io`: fixed HAMT sharding threshold comparison to use `>` instead of `>=`. A directory exactly at the threshold now stays as a basic (flat) directory, aligning behavior with code documentation and the JS implementation. This is a theoretical breaking change, but unlikely to impact real-world users as it requires a directory to be exactly at the threshold boundary. If you depend on the old behavior, adjust `HAMTShardingSize` to be 1 byte lower. See [IPIP-499](https://github.com/ipfs/specs/pull/499).
 - `bitswap/network`: Fixed goroutine leak that could cause bitswap to stop serving blocks after extended uptime. The root cause is `stream.Close()` blocking indefinitely when remote peers are unresponsive during multistream handshake ([go-libp2p#3448](https://github.com/libp2p/go-libp2p/pull/3448)). This PR ([#1083](https://github.com/ipfs/boxo/pull/1083)) adds a localized fix specific to bitswap's `SendMessage` by setting a read deadline before closing streams.
 
 ### Security

ipld/unixfs/io/directory.go

@@ -26,6 +26,9 @@ var log = logging.Logger("unixfs")
 // The size is not the *exact* block size of the encoded BasicDirectory but just
 // the estimated size based byte length of links name and CID (BasicDirectory's
 // ProtoNode doesn't use the Data field so this estimate is pretty accurate).
+//
+// Threshold behavior: directory converts to HAMT when estimatedSize > HAMTShardingSize.
+// A directory exactly at the threshold stays basic (threshold value is NOT included).
 var HAMTShardingSize = int(256 * units.KiB)
 
 // DefaultShardWidth is the default value used for hamt sharding width.
@@ -325,7 +328,8 @@ func NewBasicDirectory(dserv ipld.DAGService, opts ...DirectoryOption) (*BasicDi
 		return nil, err
 	}
 
-	// Scan node links (if any) to restore estimated size.
+	// Initialize all size tracking fields from the node.
+	// This must be done after node is created and options applied.
 	basicDir.computeEstimatedSizeAndTotalLinks()
 
 	return basicDir, nil
@@ -338,7 +342,7 @@ func NewBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *Bas
 	basicDir.node = node
 	basicDir.dserv = dserv
 
-	// Scan node links (if any) to restore estimated size.
+	// Initialize all size tracking fields from the node.
 	basicDir.computeEstimatedSizeAndTotalLinks()
 
 	return basicDir
@@ -487,8 +491,12 @@ func (d *BasicDirectory) GetSizeEstimationMode() SizeEstimationMode {
 	return HAMTSizeEstimation // fall back to global
 }
 
+// computeEstimatedSizeAndTotalLinks initializes all size tracking fields from the current node.
+// This includes estimatedSize (for links-based estimation), totalLinks count,
+// and cachedBlockSize (for block-based estimation when that mode is active).
 func (d *BasicDirectory) computeEstimatedSizeAndTotalLinks() {
 	d.estimatedSize = 0
+	d.totalLinks = 0
 	// err is just breaking the iteration and we always return nil
 	_ = d.ForEachLink(context.TODO(), func(l *ipld.Link) error {
 		d.addToEstimatedSize(l.Name, l.Cid)
@@ -497,6 +505,12 @@ func (d *BasicDirectory) computeEstimatedSizeAndTotalLinks() {
 	})
 	// ForEachLink will never fail traversing the BasicDirectory
 	// and neither the inner callback `addToEstimatedSize`.
+
+	// Initialize cachedBlockSize for block-based estimation.
+	// Check both instance-specific mode and global mode.
+	if d.GetSizeEstimationMode() == SizeEstimationBlock && d.node != nil {
+		d.cachedBlockSize, _ = calculateBlockSize(d.node)
+	}
 }
 
 func (d *BasicDirectory) addToEstimatedSize(name string, linkCid cid.Cid) {
@@ -568,7 +582,9 @@ func (d *BasicDirectory) needsToSwitchByLinkSize(name string, nodeToAdd ipld.Nod
 		operationSizeChange += linksize.LinkSizeFunction(name, nodeToAdd.Cid())
 	}
 
-	switchShardingSize := d.estimatedSize+operationSizeChange >= HAMTShardingSize
+	// Switch to HAMT when size exceeds threshold (> not >=).
+	// Directory exactly at threshold stays basic.
+	switchShardingSize := d.estimatedSize+operationSizeChange > HAMTShardingSize
 	switchMaxLinks := d.checkMaxLinksExceeded(nodeToAdd, entryToRemove)
 	return switchShardingSize || switchMaxLinks, nil
 }
@@ -610,7 +626,9 @@ func (d *BasicDirectory) needsToSwitchByBlockSize(name string, nodeToAdd ipld.No
 		return false, err
 	}
 
-	switchShardingSize := exactSize >= HAMTShardingSize
+	// Switch to HAMT when size exceeds threshold (> not >=).
+	// Directory exactly at threshold stays basic.
+	switchShardingSize := exactSize > HAMTShardingSize
 	switchMaxLinks := d.checkMaxLinksExceeded(nodeToAdd, entryToRemove)
 	return switchShardingSize || switchMaxLinks, nil
 }
@@ -1041,9 +1059,9 @@ func (d *HAMTDirectory) sizeBelowThreshold(ctx context.Context, sizeChange int)
 		}
 
 		partialSize += d.linkSizeFor(linkResult.Link)
-		if partialSize+sizeChange >= HAMTShardingSize {
-			// We have already fetched enough shards to assert we are above the
-			// threshold, so no need to keep fetching.
+		// Check if size exceeds threshold (> not >=, matching upgrade logic).
+		// Early exit: no need to enumerate more links once we know we're above.
+		if partialSize+sizeChange > HAMTShardingSize {
 			below = false
 			break
 		}

ipld/unixfs/io/directory_test.go

@@ -990,3 +990,4 @@ func TestHAMTDirectorySizeEstimationMode(t *testing.T) {
 		assert.Equal(t, SizeEstimationBlock, hamtDir.GetSizeEstimationMode())
 	})
 }
+

ipld/unixfs/io/profile_test.go

@@ -130,18 +130,19 @@ func TestProfileHAMTThresholdBehavior(t *testing.T) {
 			assert.False(t, isHAMT, "should remain BasicDirectory when below threshold (200 < 300)")
 		})
 
-		t.Run("at threshold switches to HAMTDirectory", func(t *testing.T) {
+		t.Run("at threshold stays BasicDirectory", func(t *testing.T) {
 			dir, err := NewDirectory(ds)
 			require.NoError(t, err)
 
 			// Add 3 entries = 300 bytes, exactly at threshold
+			// With > comparison, at threshold stays basic
 			for i := range 3 {
 				err = dir.AddChild(ctx, fmt.Sprintf("e%d", i), child)
 				require.NoError(t, err)
 			}
 
 			_, isHAMT := dir.(*DynamicDirectory).Directory.(*HAMTDirectory)
-			assert.True(t, isHAMT, "should switch to HAMTDirectory when at threshold (300 >= 300)")
+			assert.False(t, isHAMT, "should stay BasicDirectory when exactly at threshold (300 == 300)")
 		})
 
 		t.Run("above threshold switches to HAMTDirectory", func(t *testing.T) {
@@ -270,4 +271,79 @@ func TestProfileHAMTThresholdBehavior(t *testing.T) {
 		assert.Greater(t, actualBlockSize, linksEstimate,
 			"links-based estimation should underestimate actual block size")
 	})
+
+	t.Run("SizeEstimationBlock exact threshold boundary", func(t *testing.T) {
+		saveAndRestoreGlobals(t)
+
+		// Test that the HAMT switch happens exactly when size > threshold (not >=)
+		HAMTSizeEstimation = SizeEstimationBlock
+
+		ds := mdtest.Mock()
+		ctx := t.Context()
+		child := ft.EmptyDirNode()
+		require.NoError(t, ds.Add(ctx, child))
+
+		// First, determine the exact block size after adding entries
+		testDir, err := NewDirectory(ds, WithSizeEstimationMode(SizeEstimationBlock))
+		require.NoError(t, err)
+
+		// Add entries and track sizes at each step
+		var sizes []int
+		for i := 0; i < 10; i++ {
+			err = testDir.AddChild(ctx, fmt.Sprintf("entry%d", i), child)
+			require.NoError(t, err)
+
+			node, err := testDir.GetNode()
+			require.NoError(t, err)
+			pn, ok := node.(*mdag.ProtoNode)
+			require.True(t, ok)
+			size, err := calculateBlockSize(pn)
+			require.NoError(t, err)
+			sizes = append(sizes, size)
+		}
+
+		// Set threshold to exactly the size after 5 entries
+		// (entry0..entry4 = 5 entries)
+		exactThreshold := sizes[4]
+		t.Logf("threshold set to exactly %d bytes (size after 5 entries)", exactThreshold)
+
+		t.Run("at exact threshold stays BasicDirectory", func(t *testing.T) {
+			HAMTShardingSize = exactThreshold
+
+			dir, err := NewDirectory(ds, WithSizeEstimationMode(SizeEstimationBlock))
+			require.NoError(t, err)
+
+			// Add 5 entries to reach exactly threshold size
+			for i := 0; i < 5; i++ {
+				err = dir.AddChild(ctx, fmt.Sprintf("entry%d", i), child)
+				require.NoError(t, err)
+			}
+
+			node, err := dir.GetNode()
+			require.NoError(t, err)
+			pn := node.(*mdag.ProtoNode)
+			actualSize, _ := calculateBlockSize(pn)
+			t.Logf("actual size: %d, threshold: %d", actualSize, exactThreshold)
+
+			_, isHAMT := dir.(*DynamicDirectory).Directory.(*HAMTDirectory)
+			assert.False(t, isHAMT, "should stay BasicDirectory when size equals threshold (%d == %d)", actualSize, exactThreshold)
+		})
+
+		t.Run("one byte over threshold switches to HAMTDirectory", func(t *testing.T) {
+			// Set threshold to size[4] - 1 so that size[4] is > threshold
+			HAMTShardingSize = sizes[4] - 1
+
+			dir, err := NewDirectory(ds, WithSizeEstimationMode(SizeEstimationBlock))
+			require.NoError(t, err)
+
+			// Add 5 entries - last one should trigger HAMT
+			for i := 0; i < 5; i++ {
+				err = dir.AddChild(ctx, fmt.Sprintf("entry%d", i), child)
+				require.NoError(t, err)
+			}
+
+			_, isHAMT := dir.(*DynamicDirectory).Directory.(*HAMTDirectory)
+			assert.True(t, isHAMT, "should switch to HAMTDirectory when size exceeds threshold (%d > %d)", sizes[4], HAMTShardingSize)
+		})
+	})
 }

mfs/dir.go

@@ -89,12 +89,16 @@ func NewDirectory(ctx context.Context, name string, node ipld.Node, parent paren
 // The directory is added to the DAGService. To create a new MFS
 // root use NewEmptyRootFolder instead.
 func NewEmptyDirectory(ctx context.Context, name string, parent parent, dserv ipld.DAGService, prov provider.MultihashProvider, opts MkdirOpts) (*Directory, error) {
-	db, err := uio.NewDirectory(dserv,
+	dirOpts := []uio.DirectoryOption{
 		uio.WithMaxLinks(opts.MaxLinks),
 		uio.WithMaxHAMTFanout(opts.MaxHAMTFanout),
 		uio.WithStat(opts.Mode, opts.ModTime),
 		uio.WithCidBuilder(opts.CidBuilder),
-	)
+	}
+	if opts.SizeEstimationMode != nil {
+		dirOpts = append(dirOpts, uio.WithSizeEstimationMode(*opts.SizeEstimationMode))
+	}
+	db, err := uio.NewDirectory(dserv, dirOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -580,6 +584,7 @@ func (d *Directory) setNodeData(data []byte, links []*ipld.Link) error {
 	// We need to carry our desired settings.
 	db.SetMaxLinks(d.unixfsDir.GetMaxLinks())
 	db.SetMaxHAMTFanout(d.unixfsDir.GetMaxHAMTFanout())
+	db.SetSizeEstimationMode(d.unixfsDir.GetSizeEstimationMode())
 	d.unixfsDir = db
 
 	return nil

mfs/ops.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	uio "github.com/ipfs/boxo/ipld/unixfs/io"
 	cid "github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
 )
@@ -120,13 +121,14 @@ func PutNode(r *Root, path string, nd ipld.Node) error {
 
 // MkdirOpts is used by Mkdir
 type MkdirOpts struct {
-	Mkparents     bool
-	Flush         bool
-	CidBuilder    cid.Builder
-	Mode          os.FileMode
-	ModTime       time.Time
-	MaxLinks      int
-	MaxHAMTFanout int
+	Mkparents          bool
+	Flush              bool
+	CidBuilder         cid.Builder
+	Mode               os.FileMode
+	ModTime            time.Time
+	MaxLinks           int
+	MaxHAMTFanout      int
+	SizeEstimationMode *uio.SizeEstimationMode
 }
 
 // Mkdir creates a directory at 'path' under the directory 'd', creating
@@ -157,8 +159,9 @@ func Mkdir(r *Root, pth string, opts MkdirOpts) error {
 
 	// opts to make the parents leave MkParents and Flush as false.
 	parentsOpts := MkdirOpts{
-		MaxLinks:      opts.MaxLinks,
-		MaxHAMTFanout: opts.MaxHAMTFanout,
+		MaxLinks:           opts.MaxLinks,
+		MaxHAMTFanout:      opts.MaxHAMTFanout,
+		SizeEstimationMode: opts.SizeEstimationMode,
 	}
 
 	for i, d := range parts[:len(parts)-1] {