storage: fix tomstoned objects revival not working

Delete tombstone after revival. Note that deletion occurs only on the specific
node where the object is located.

Closes #3486.

Signed-off-by: Andrey Butusov <[email protected]>

Author: End-rey

CHANGELOG.md

@@ -7,6 +7,7 @@ Changelog for NeoFS Node
 
 ### Fixed
 - Send on closed channel panic in node's new epoch handler (#3529)
+- Tomstoned objects revival not working (#3542)
 
 ### Changed
 - Stream payload without buffering to reduce memory usage in CLI `Get/Put` operations (#3535)

pkg/local_object_storage/engine/revive_test.go

@@ -0,0 +1,47 @@
+package engine
+
+import (
+	"testing"
+
+	"github.com/nspcc-dev/neofs-node/pkg/core/object"
+	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
+	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
+	cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
+	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStorageEngine_ReviveObject(t *testing.T) {
+	e := testNewEngineWithShardNum(t, 1)
+	defer func() { _ = e.Close() }()
+
+	cnr := cidtest.ID()
+
+	obj := generateObjectWithCID(cnr)
+	require.NoError(t, e.Put(obj, nil))
+	addr := object.AddressOf(obj)
+
+	ts := generateObjectWithCID(cnr)
+	ts.SetType(objectSDK.TypeTombstone)
+	addAttribute(ts, objectSDK.AttributeAssociatedObject, obj.GetID().EncodeToString())
+	addAttribute(ts, objectSDK.AttributeExpirationEpoch, "0")
+	tsAddr := object.AddressOf(ts)
+
+	require.NoError(t, e.Put(ts, nil))
+
+	_, err := e.Get(addr)
+	require.ErrorIs(t, err, apistatus.ErrObjectAlreadyRemoved)
+
+	rs, err := e.ReviveObject(addr)
+	require.NoError(t, err)
+	require.Equal(t, meta.ReviveStatusGraveyard, rs.Shards[0].Status.StatusType())
+	require.Equal(t, tsAddr, rs.Shards[0].Status.TombstoneAddress())
+
+	got, err := e.Get(addr)
+	require.NoError(t, err)
+	require.Equal(t, obj, got)
+
+	// tombstone metadata should be dropped, so getting TS must fail with not found
+	_, err = e.Get(tsAddr)
+	require.ErrorIs(t, err, apistatus.ErrObjectNotFound)
+}

pkg/local_object_storage/metabase/put_test.go

@@ -167,9 +167,28 @@ func TestDB_Put_ObjectWithTombstone(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, meta.ReviveStatusGarbage, rs.StatusType())
 	require.Equal(t, "successful revival from garbage bucket", rs.Message())
+	require.Equal(t, tsAddr, rs.TombstoneAddress())
 
 	t.Run("after revival", func(t *testing.T) {
-		t.Skip("https://github.com/nspcc-dev/neofs-node/issues/3486")
+		// tombstone is still there, but the garbage was cleared
+		t.Run("get garbage", func(t *testing.T) {
+			gObjs, _, err := db.GetGarbage(100)
+			require.NoError(t, err)
+			require.NotContains(t, gObjs, addr)
+		})
+		t.Run("iterate garbage", func(t *testing.T) {
+			var collected []oid.Address
+			err := db.IterateOverGarbage(func(gObj meta.GarbageObject) error {
+				collected = append(collected, gObj.Address())
+				return nil
+			}, nil)
+			require.NoError(t, err)
+			require.NotContains(t, collected, addr)
+		})
+
+		_, err = db.Delete([]oid.Address{tsAddr})
+		require.NoError(t, err)
+
 		assertObjectAvailability(t, db, addr, obj)
 	})
 }

pkg/local_object_storage/metabase/revive.go

@@ -1,6 +1,7 @@
 package meta
 
 import (
+	"bytes"
 	"fmt"
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/util/logicerr"
@@ -31,6 +32,9 @@ const (
 type ReviveStatus struct {
 	statusType reviveStatusType
 	message    string
+	// tombstoneAddr holds the address of the tombstone used to inhume the object (if any).
+	// It is set only when revival happened from a graveyard.
+	tombstoneAddr oid.Address
 }
 
 // Message returns message of status.
@@ -43,6 +47,11 @@ func (s *ReviveStatus) StatusType() reviveStatusType {
 	return s.statusType
 }
 
+// TombstoneAddress returns the tombstone address.
+func (s *ReviveStatus) TombstoneAddress() oid.Address {
+	return s.tombstoneAddr
+}
+
 func (s *ReviveStatus) setStatusGraveyard(tomb string) {
 	s.statusType = ReviveStatusGraveyard
 	s.message = fmt.Sprintf("successful revival from graveyard, tomb: %s", tomb)
@@ -73,6 +82,7 @@ func (db *DB) ReviveObject(addr oid.Address) (res ReviveStatus, err error) {
 	}
 
 	currEpoch := db.epochState.CurrentEpoch()
+	cnr := addr.Container()
 
 	err = db.boltDB.Update(func(tx *bbolt.Tx) error {
 		garbageObjectsBKT := tx.Bucket(garbageObjectsBucketName)
@@ -101,6 +111,7 @@ func (db *DB) ReviveObject(addr oid.Address) (res ReviveStatus, err error) {
 				return err
 			}
 			res.setStatusGraveyard(tombAddress.EncodeToString())
+			res.tombstoneAddr = tombAddress
 		} else {
 			val = garbageContainersBKT.Get(targetKey[:cidSize])
 			if val != nil {
@@ -116,6 +127,20 @@ func (db *DB) ReviveObject(addr oid.Address) (res ReviveStatus, err error) {
 				// nor was marked with GC mark
 				return ErrObjectWasNotRemoved
 			}
+
+			metaBucket := tx.Bucket(metaBucketKey(cnr))
+			var metaCursor *bbolt.Cursor
+			if metaBucket != nil {
+				metaCursor = metaBucket.Cursor()
+			}
+
+			tombID, err := getTombstoneByAssociatedObject(metaCursor, addr.Object())
+			if err != nil {
+				return fmt.Errorf("iterate covered by tombstones: %w", err)
+			}
+			if !tombID.IsZero() {
+				res.tombstoneAddr = oid.NewAddress(cnr, tombID)
+			}
 		}
 
 		if err := garbageObjectsBKT.Delete(targetKey); err != nil {
@@ -126,7 +151,7 @@ func (db *DB) ReviveObject(addr oid.Address) (res ReviveStatus, err error) {
 			// if object is stored, and it is regular object then update bucket
 			// with container size estimations
 			if obj.Type() == object.TypeRegular {
-				if err := changeContainerInfo(tx, addr.Container(), int(obj.PayloadSize()), 1); err != nil {
+				if err := changeContainerInfo(tx, cnr, int(obj.PayloadSize()), 1); err != nil {
 					return err
 				}
 			}
@@ -145,3 +170,42 @@ func (db *DB) ReviveObject(addr oid.Address) (res ReviveStatus, err error) {
 
 	return
 }
+
+// getTombstoneByAssociatedObject iterates meta index to find tombstone object
+// associated with the provided object ID. If found, returns its ID, otherwise
+// returns zero ID and nil error.
+func getTombstoneByAssociatedObject(metaCursor *bbolt.Cursor, idObj oid.ID) (oid.ID, error) {
+	var id oid.ID
+	if metaCursor == nil {
+		return id, fmt.Errorf("nil meta cursor")
+	}
+
+	var (
+		typString        = object.TypeTombstone.String()
+		idStr            = idObj.EncodeToString()
+		accPrefix        = make([]byte, 1+len(object.AttributeAssociatedObject)+1+len(idStr)+1)
+		typeKey          = make([]byte, metaIDTypePrefixSize+len(typString))
+		expirationPrefix = make([]byte, attrIDFixedLen+len(object.AttributeExpirationEpoch))
+	)
+
+	expirationPrefix[0] = metaPrefixIDAttr
+	copy(expirationPrefix[1+oid.Size:], object.AttributeExpirationEpoch)
+
+	accPrefix[0] = metaPrefixAttrIDPlain
+	copy(accPrefix[1:], object.AttributeAssociatedObject)
+	copy(accPrefix[1+len(object.AttributeAssociatedObject)+1:], idStr)
+
+	fillIDTypePrefix(typeKey)
+	copy(typeKey[metaIDTypePrefixSize:], typString)
+
+	for k, _ := metaCursor.Seek(accPrefix); bytes.HasPrefix(k, accPrefix); k, _ = metaCursor.Next() {
+		mainObj := k[len(accPrefix):]
+		copy(typeKey[1:], mainObj)
+
+		if metaCursor.Bucket().Get(typeKey) != nil {
+			return id, id.Decode(mainObj)
+		}
+	}
+
+	return id, nil
+}

pkg/local_object_storage/metabase/revive_test.go

@@ -41,10 +41,15 @@ func TestDB_ReviveObject(t *testing.T) {
 		res, err := db.ReviveObject(object.AddressOf(raw))
 		require.NoError(t, err)
 		require.Equal(t, meta.ReviveStatusGraveyard, res.StatusType())
+		require.NotNil(t, res.TombstoneAddress())
 
 		exists, err = metaExists(db, object.AddressOf(raw))
 		require.NoError(t, err)
 		require.True(t, exists)
+
+		exists, err = metaExists(db, tombstoneID)
+		require.NoError(t, err)
+		require.False(t, exists)
 	})
 
 	t.Run("from GC", func(t *testing.T) {

pkg/local_object_storage/shard/put_test.go

@@ -224,6 +224,47 @@ func TestDB_Put_Tombstone(t *testing.T) {
 
 	tombAddr := oid.NewAddress(tomb.GetContainerID(), tomb.GetID())
 
+	t.Run("revive after tombstone", func(t *testing.T) {
+		sh, fs := newShardWithFSTree(t)
+
+		require.NoError(t, sh.Put(&obj, nil))
+		require.NoError(t, sh.Put(&tomb, nil))
+
+		exist, err := sh.Exists(objAddr, false)
+		require.Error(t, err, apistatus.ErrObjectAlreadyRemoved)
+		require.False(t, exist)
+
+		_, err = sh.Get(objAddr, false)
+		require.ErrorIs(t, err, apistatus.ErrObjectAlreadyRemoved)
+
+		rs, err := sh.ReviveObject(objAddr)
+		require.NoError(t, err)
+		require.Equal(t, meta.ReviveStatusGarbage, rs.StatusType())
+		require.Equal(t, tombAddr, rs.TombstoneAddress())
+
+		exist, err = sh.Exists(objAddr, false)
+		require.NoError(t, err)
+		require.True(t, exist)
+
+		got, err := sh.Get(objAddr, false)
+		require.NoError(t, err)
+		require.Equal(t, &obj, got)
+
+		exist, err = sh.Exists(tombAddr, false)
+		require.NoError(t, err)
+		require.False(t, exist)
+
+		_, err = sh.Get(tombAddr, false)
+		require.ErrorIs(t, err, apistatus.ErrObjectNotFound)
+
+		exist, err = fs.Exists(tombAddr)
+		require.NoError(t, err)
+		require.False(t, exist)
+
+		_, err = fs.Get(tombAddr)
+		require.ErrorIs(t, err, apistatus.ErrObjectNotFound)
+	})
+
 	for _, tc := range []struct {
 		name         string
 		preset       func(*testing.T, *shard.Shard)

pkg/local_object_storage/shard/revive.go

@@ -3,6 +3,7 @@ package shard
 import (
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
+	"go.uber.org/zap"
 )
 
 // ReviveObject try to revive object in Shard, by remove records from graveyard and garbage.
@@ -17,5 +18,16 @@ func (s *Shard) ReviveObject(addr oid.Address) (meta.ReviveStatus, error) {
 	} else if s.GetMode().NoMetabase() {
 		return meta.ReviveStatus{}, ErrDegradedMode
 	}
-	return s.metaBase.ReviveObject(addr)
+
+	st, err := s.metaBase.ReviveObject(addr)
+	if err == nil {
+		if ts := st.TombstoneAddress(); !ts.Object().IsZero() {
+			if delErr := s.deleteObjs([]oid.Address{ts}); delErr != nil {
+				s.log.Debug("tombstone delete after revive failed", zap.Stringer("tombstone", ts), zap.Error(delErr))
+			} else {
+				s.log.Debug("tombstone deleted after revive", zap.Stringer("tombstone", ts))
+			}
+		}
+	}
+	return st, err
 }