feat(miner): make precommit HAMT & AMT CIDs nullable

Author: rvagg

builtin/v16/migration/miner.go

@@ -0,0 +1,106 @@
+package migration
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+
+	miner15 "github.com/filecoin-project/go-state-types/builtin/v15/miner"
+	miner16 "github.com/filecoin-project/go-state-types/builtin/v16/miner"
+	"github.com/filecoin-project/go-state-types/migration"
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"golang.org/x/xerrors"
+)
+
+type minerMigrator struct {
+	OutCodeCID                      cid.Cid
+	EmptyPreCommittedSectorsHamtCid cid.Cid
+	EmptyPrecommitCleanUpAmtCid     cid.Cid
+}
+
+func newMinerMigrator(
+	_ context.Context,
+	_ cbor.IpldStore,
+	outCode,
+	emptyPreCommittedSectorsHamtCid,
+	emptyPrecommitCleanUpAmtCid cid.Cid,
+) (*minerMigrator, error) {
+	return &minerMigrator{
+		OutCodeCID:                      outCode,
+		EmptyPreCommittedSectorsHamtCid: emptyPreCommittedSectorsHamtCid,
+		EmptyPrecommitCleanUpAmtCid:     emptyPrecommitCleanUpAmtCid,
+	}, nil
+}
+
+// TOOD: remove these, they're just for debugging
+var minerCount, withEmptyPrecommitCleanUpAmtCidCount, withEmptyPreCommittedSectorsHamtCidCount, migratedCount uint64
+
+func (m *minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, in migration.ActorMigrationInput) (result *migration.ActorMigrationResult, err error) {
+	var inState miner15.State
+	if err := store.Get(ctx, in.Head, &inState); err != nil {
+		return nil, xerrors.Errorf("failed to load miner state for %s: %w", in.Address, err)
+	}
+
+	newHead := in.Head
+
+	var epccuacc, epcshcc uint64
+	var preCommittedSectors, preCommittedSectorsCleanUp *cid.Cid
+	if !m.EmptyPrecommitCleanUpAmtCid.Equals(inState.PreCommittedSectorsCleanUp) {
+		preCommittedSectorsCleanUp = &inState.PreCommittedSectorsCleanUp
+	} else {
+		epccuacc = atomic.AddUint64(&withEmptyPrecommitCleanUpAmtCidCount, 1)
+	}
+	if !m.EmptyPreCommittedSectorsHamtCid.Equals(inState.PreCommittedSectors) {
+		preCommittedSectors = &inState.PreCommittedSectors
+	} else {
+		epcshcc = atomic.AddUint64(&withEmptyPreCommittedSectorsHamtCidCount, 1)
+	}
+
+	// we only need to update the state if one of these need to be nullable, otherwise the existing
+	// state, with a valid CID, will decode correctly even though they're now pointer fields
+	if preCommittedSectors == nil || preCommittedSectorsCleanUp == nil {
+		outState := miner16.State{
+			Info:                       inState.Info,
+			PreCommitDeposits:          inState.PreCommitDeposits,
+			LockedFunds:                inState.LockedFunds,
+			VestingFunds:               inState.VestingFunds,
+			FeeDebt:                    inState.FeeDebt,
+			InitialPledge:              inState.InitialPledge,
+			PreCommittedSectors:        preCommittedSectors,
+			PreCommittedSectorsCleanUp: preCommittedSectorsCleanUp,
+			AllocatedSectors:           inState.AllocatedSectors,
+			Sectors:                    inState.Sectors,
+			ProvingPeriodStart:         inState.ProvingPeriodStart,
+			CurrentDeadline:            inState.CurrentDeadline,
+			Deadlines:                  inState.Deadlines,
+			EarlyTerminations:          inState.EarlyTerminations,
+			DeadlineCronActive:         inState.DeadlineCronActive,
+		}
+
+		if newHead, err = store.Put(ctx, &outState); err != nil {
+			return nil, xerrors.Errorf("failed to put new state: %w", err)
+		}
+
+		atomic.AddUint64(&migratedCount, 1)
+	}
+
+	if nc := atomic.AddUint64(&minerCount, 1); nc%10000 == 0 {
+		fmt.Printf("Checked %d miners, %d with empty PreCommittedSectors, %d with empty PreCommittedSectorsCleanUp, %d migrated\n", nc, epcshcc, epccuacc, atomic.LoadUint64(&migratedCount))
+	}
+
+	return &migration.ActorMigrationResult{
+		NewCodeCID: m.MigratedCodeCID(),
+		NewHead:    newHead,
+	}, nil
+}
+
+func (m *minerMigrator) MigratedCodeCID() cid.Cid {
+	return m.OutCodeCID
+}
+
+func (m *minerMigrator) Deferred() bool {
+	return false
+}
+
+var _ migration.ActorMigration = (*minerMigrator)(nil)

builtin/v16/migration/top.go

@@ -3,9 +3,10 @@ package migration
 import (
 	"context"
 
-	adt16 "github.com/filecoin-project/go-state-types/builtin/v16/util/adt"
+	"github.com/filecoin-project/go-state-types/builtin/v16/util/adt"
 
 	system15 "github.com/filecoin-project/go-state-types/builtin/v13/system"
+	miner16 "github.com/filecoin-project/go-state-types/builtin/v16/miner"
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/builtin"
@@ -24,7 +25,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID
 		return cid.Undef, xerrors.Errorf("invalid migration config with %d workers", cfg.MaxWorkers)
 	}
 
-	adtStore := adt16.WrapStore(ctx, store)
+	adtStore := adt.WrapStore(ctx, store)
 
 	// Load input and output state trees
 	actorsIn, err := builtin.LoadTree(adtStore, actorsRootIn)
@@ -67,14 +68,22 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID
 	// Set of prior version code CIDs for actors to defer during iteration, for explicit migration afterwards.
 	deferredCodeIDs := make(map[cid.Cid]struct{})
 
+	miner15Cid := cid.Undef
+
 	for _, oldEntry := range oldManifestData.Entries {
 		newCodeCID, ok := newManifest.Get(oldEntry.Name)
 		if !ok {
 			return cid.Undef, xerrors.Errorf("code cid for %s actor not found in new manifest", oldEntry.Name)
 		}
+		if oldEntry.Name == manifest.MinerKey {
+			miner15Cid = oldEntry.Code
+		}
 		migrations[oldEntry.Code] = migration.CachedMigration(cache, migration.CodeMigrator{OutCodeCID: newCodeCID})
 	}
 
+	if miner15Cid == cid.Undef {
+		return cid.Undef, xerrors.Errorf("could not find miner actor in old manifest")
+	}
 	// migrations that migrate both code and state, override entries in `migrations`
 
 	// The System Actor
@@ -90,6 +99,35 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID
 		return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations)+len(deferredCodeIDs), len(oldManifestData.Entries))
 	}
 
+	var emptyPreCommittedSectorsHamtCid cid.Cid
+	if hamt, err := adt.MakeEmptyMap(adtStore, builtin.DefaultHamtBitwidth); err != nil {
+		return cid.Undef, xerrors.Errorf("failed to create empty precommit clean up amount array: %w", err)
+	} else {
+		if emptyPreCommittedSectorsHamtCid, err = hamt.Root(); err != nil {
+			return cid.Undef, xerrors.Errorf("failed to get root of empty precommit clean up amount array: %w", err)
+		}
+	}
+	var emptyPrecommitCleanUpAmtCid cid.Cid
+	if amt, err := adt.MakeEmptyArray(adtStore, miner16.PrecommitCleanUpAmtBitwidth); err != nil {
+		return cid.Undef, xerrors.Errorf("failed to create empty precommit clean up amount array: %w", err)
+	} else {
+		if emptyPrecommitCleanUpAmtCid, err = amt.Root(); err != nil {
+			return cid.Undef, xerrors.Errorf("failed to get root of empty precommit clean up amount array: %w", err)
+		}
+	}
+
+	miner16Cid, ok := newManifest.Get(manifest.MinerKey)
+	if !ok {
+		return cid.Undef, xerrors.Errorf("code cid for miner actor not found in new manifest")
+	}
+
+	minerMig, err := newMinerMigrator(ctx, store, miner16Cid, emptyPreCommittedSectorsHamtCid, emptyPrecommitCleanUpAmtCid)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to create miner migrator: %w", err)
+	}
+
+	migrations[miner15Cid] = migration.CachedMigration(cache, minerMig)
+
 	actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("failed to run migration: %w", err)

builtin/v16/miner/cbor_gen.go

@@ -778,54 +778,58 @@ func CheckPreCommits(st *State, store adt.Store, allocatedSectorsMap map[uint64]
 
 	// invert pre-commit clean up queue into a lookup by sector number
 	cleanUpEpochs := make(map[uint64]abi.ChainEpoch)
-	if cleanUpQ, err := util.LoadBitfieldQueue(store, st.PreCommittedSectorsCleanUp, st.QuantSpecEveryDeadline(), PrecommitCleanUpAmtBitwidth); err != nil {
-		acc.Addf("error loading pre-commit clean up queue: %v", err)
-	} else {
-		err = cleanUpQ.ForEach(func(epoch abi.ChainEpoch, bf bitfield.BitField) error {
-			quantized := quant.QuantizeUp(epoch)
-			acc.Require(quantized == epoch, "precommit expiration %d is not quantized", epoch)
-			if err = bf.ForEach(func(secNum uint64) error {
-				cleanUpEpochs[secNum] = epoch
+	if st.PreCommittedSectorsCleanUp != nil {
+		if cleanUpQ, err := util.LoadBitfieldQueue(store, *st.PreCommittedSectorsCleanUp, st.QuantSpecEveryDeadline(), PrecommitCleanUpAmtBitwidth); err != nil {
+			acc.Addf("error loading pre-commit clean up queue: %v", err)
+		} else {
+			err = cleanUpQ.ForEach(func(epoch abi.ChainEpoch, bf bitfield.BitField) error {
+				quantized := quant.QuantizeUp(epoch)
+				acc.Require(quantized == epoch, "precommit expiration %d is not quantized", epoch)
+				if err = bf.ForEach(func(secNum uint64) error {
+					cleanUpEpochs[secNum] = epoch
+					return nil
+				}); err != nil {
+					acc.Addf("error iteration pre-commit expiration bitfield: %v", err)
+				}
 				return nil
-			}); err != nil {
-				acc.Addf("error iteration pre-commit expiration bitfield: %v", err)
-			}
-			return nil
-		})
-		acc.RequireNoError(err, "error iterating pre-commit clean up queue")
+			})
+			acc.RequireNoError(err, "error iterating pre-commit clean up queue")
+		}
 	}
 
 	precommitTotal := big.Zero()
-	if precommitted, err := adt.AsMap(store, st.PreCommittedSectors, builtin.DefaultHamtBitwidth); err != nil {
-		acc.Addf("error loading precommitted sectors: %v", err)
-	} else {
-		var precommit SectorPreCommitOnChainInfo
-		err = precommitted.ForEach(&precommit, func(key string) error {
-			secNum, err := abi.ParseUIntKey(key)
-			if err != nil {
-				acc.Addf("error parsing pre-commit key as uint: %v", err)
-				return nil
-			}
-
-			allocated := false
-			if allocatedSectorsMap != nil {
-				allocated = allocatedSectorsMap[secNum]
-			} else {
-				allocated, err = allocatedSectorsBf.IsSet(secNum)
+	if st.PreCommittedSectors != nil {
+		if precommitted, err := adt.AsMap(store, *st.PreCommittedSectors, builtin.DefaultHamtBitwidth); err != nil {
+			acc.Addf("error loading precommitted sectors: %v", err)
+		} else {
+			var precommit SectorPreCommitOnChainInfo
+			err = precommitted.ForEach(&precommit, func(key string) error {
+				secNum, err := abi.ParseUIntKey(key)
 				if err != nil {
-					acc.Addf("error checking allocated sectors: %v", err)
+					acc.Addf("error parsing pre-commit key as uint: %v", err)
 					return nil
 				}
-			}
-			acc.Require(allocated, "pre-committed sector number has not been allocated %d", secNum)
 
-			_, found := cleanUpEpochs[secNum]
-			acc.Require(found, "no clean up epoch for pre-commit at %d", precommit.PreCommitEpoch)
+				allocated := false
+				if allocatedSectorsMap != nil {
+					allocated = allocatedSectorsMap[secNum]
+				} else {
+					allocated, err = allocatedSectorsBf.IsSet(secNum)
+					if err != nil {
+						acc.Addf("error checking allocated sectors: %v", err)
+						return nil
+					}
+				}
+				acc.Require(allocated, "pre-committed sector number has not been allocated %d", secNum)
 
-			precommitTotal = big.Add(precommitTotal, precommit.PreCommitDeposit)
-			return nil
-		})
-		acc.RequireNoError(err, "error iterating pre-committed sectors")
+				_, found := cleanUpEpochs[secNum]
+				acc.Require(found, "no clean up epoch for pre-commit at %d", precommit.PreCommitEpoch)
+
+				precommitTotal = big.Add(precommitTotal, precommit.PreCommitDeposit)
+				return nil
+			})
+			acc.RequireNoError(err, "error iterating pre-committed sectors")
+		}
 	}
 
 	acc.Require(st.PreCommitDeposits.Equals(precommitTotal),

builtin/v16/miner/invariants.go

@@ -35,10 +35,10 @@ type State struct {
 	InitialPledge abi.TokenAmount // Sum of initial pledge requirements of all active sectors
 
 	// Sectors that have been pre-committed but not yet proven.
-	PreCommittedSectors cid.Cid // Map, HAMT[SectorNumber]SectorPreCommitOnChainInfo
+	PreCommittedSectors *cid.Cid // Map, HAMT[SectorNumber]SectorPreCommitOnChainInfo
 
 	// PreCommittedSectorsCleanUp maintains the state required to cleanup expired PreCommittedSectors.
-	PreCommittedSectorsCleanUp cid.Cid // BitFieldQueue (AMT[Epoch]*BitField)
+	PreCommittedSectorsCleanUp *cid.Cid // BitFieldQueue (AMT[Epoch]*BitField)
 
 	// Allocated sector IDs. Sector IDs can never be reused once allocated.
 	AllocatedSectors cid.Cid // BitField
@@ -217,7 +217,11 @@ func (st *State) QuantSpecForDeadline(dlIdx uint64) builtin.QuantSpec {
 }
 
 func (st *State) GetPrecommittedSector(store adt.Store, sectorNo abi.SectorNumber) (*SectorPreCommitOnChainInfo, bool, error) {
-	precommitted, err := adt.AsMap(store, st.PreCommittedSectors, builtin.DefaultHamtBitwidth)
+	if st.PreCommittedSectors == nil {
+		return nil, false, nil
+	}
+
+	precommitted, err := adt.AsMap(store, *st.PreCommittedSectors, builtin.DefaultHamtBitwidth)
 	if err != nil {
 		return nil, false, err
 	}