p2p/nodedb: bench sync thresholds (#17405)

benchmarks are unreliable if run in multi-cores (because even 1 core
overloading disk write io). So:
, must use `-cpu=1`: 
```
go test -bench=BenchmarkNodeDBGeometry/1mb_2sec -run=BenchmarkNodeDBGeometry -cpu=1 -count=10 -benchtime=5s ./p2p/enode
goos: linux
goarch: amd64
pkg: github.com/erigontech/erigon/p2p/enode
cpu: AMD EPYC 4344P 8-Core Processor
BenchmarkNodeDBGeometry/1mb_2sec         	  304875	     24948 ns/op	         2.000 ms_worst
BenchmarkNodeDBGeometry/1mb_2sec         	  271047	     27810 ns/op	         3.000 ms_worst
BenchmarkNodeDBGeometry/1mb_2sec         	  246031	     29828 ns/op	         6.000 ms_worst
BenchmarkNodeDBGeometry/1mb_2sec         	  230364	     31389 ns/op	         9.000 ms_worst
BenchmarkNodeDBGeometry/1mb_2sec         	  205081	     34422 ns/op	        11.00 ms_worst
BenchmarkNodeDBGeometry/1mb_2sec         	  199292	     35899 ns/op	        13.00 ms_worst
BenchmarkNodeDBGeometry/1mb_2sec         	  189828	     40628 ns/op	        20.00 ms_worst
BenchmarkNodeDBGeometry/1mb_2sec         	  148378	     47022 ns/op	        17.00 ms_worst
BenchmarkNodeDBGeometry/1mb_2sec         	  141898	     48844 ns/op	        18.00 ms_worst
PASS
```

```
go test -bench=BenchmarkNodeDBGeometry/5mb_2sec -run=BenchmarkNodeDBGeometry -cpu=1 -count=10 -benchtime=5s ./p2p/enode
goos: linux
goarch: amd64
pkg: github.com/erigontech/erigon/p2p/enode
cpu: AMD EPYC 4344P 8-Core Processor
BenchmarkNodeDBGeometry/5mb_2sec         	  267589	     30181 ns/op	         3.000 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  252652	     31581 ns/op	         5.000 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  222480	     34548 ns/op	         8.000 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  203396	     37386 ns/op	         9.000 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  193489	     39445 ns/op	        12.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  177519	     40932 ns/op	        11.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  170472	     42475 ns/op	        13.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  169920	     43283 ns/op	        15.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  168820	     42521 ns/op	        14.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  171238	     43919 ns/op	        15.00 ms_worst
```


```
go test -bench=BenchmarkNodeDBGeometry/5mb_2sec -run=BenchmarkNodeDBGeometry -cpu=1 -count=10 -benchtime=5s ./p2p/enode
goos: linux
goarch: amd64
pkg: github.com/erigontech/erigon/p2p/enode
cpu: AMD EPYC 4344P 8-Core Processor
BenchmarkNodeDBGeometry/5mb_2sec         	  271959	     29516 ns/op	         3.000 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  268170	     30549 ns/op	         5.000 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  227325	     34291 ns/op	         7.000 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  202426	     37019 ns/op	        10.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  192224	     39110 ns/op	        11.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  181965	     40802 ns/op	        13.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  171835	     42273 ns/op	        14.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  164223	     42940 ns/op	        15.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  168981	     42261 ns/op	        14.00 ms_worst
BenchmarkNodeDBGeometry/5mb_2sec         	  167252	     43589 ns/op	        15.00 ms_worst
```


```
 go test -bench=BenchmarkNodeDBGeometry/10mb_2sec -run=BenchmarkNodeDBGeometry -cpu=1 -count=10 -benchtime=5s ./p2p/enode
goos: linux
goarch: amd64
pkg: github.com/erigontech/erigon/p2p/enode
cpu: AMD EPYC 4344P 8-Core Processor
BenchmarkNodeDBGeometry/10mb_2sec         	  241651	     31061 ns/op	         4.000 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  229740	     32291 ns/op	         5.000 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  211956	     33151 ns/op	         7.000 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  207318	     34564 ns/op	         8.000 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  190208	     36950 ns/op	         8.000 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  181152	     38898 ns/op	         9.000 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  171712	     40510 ns/op	         9.000 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  167190	     41969 ns/op	        10.00 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  164137	     40918 ns/op	         9.000 ms_worst
BenchmarkNodeDBGeometry/10mb_2sec         	  166485	     41738 ns/op	        10.00 ms_worst
```

```
 go test -bench=BenchmarkNodeDBGeometry/20mb_2sec -run=BenchmarkNodeDBGeometry -cpu=1 -count=10 -benchtime=5s ./p2p/enode
goos: linux
goarch: amd64
pkg: github.com/erigontech/erigon/p2p/enode
cpu: AMD EPYC 4344P 8-Core Processor
BenchmarkNodeDBGeometry/20mb_2sec         	  241070	     31248 ns/op	         8.000 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  201418	     36015 ns/op	        11.00 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  182701	     39571 ns/op	        11.00 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  166308	     43206 ns/op	        14.00 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  154442	     46006 ns/op	        16.00 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  146713	     47762 ns/op	        16.00 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  138229	     48429 ns/op	        18.00 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  126934	     49735 ns/op	        20.00 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  122917	     49225 ns/op	        21.00 ms_worst
BenchmarkNodeDBGeometry/20mb_2sec         	  131707	     48877 ns/op	        19.00 ms_worst
```

Also PageSize 16kb -> 4kb: improving 30% throughput

---------

Co-authored-by: alex <[email protected]>

Author: AskAlexSharov

db/kv/kv_interface.go

@@ -125,6 +125,7 @@ type RoDB interface {
 
 	// CHandle pointer to the underlying C environment handle, if applicable (e.g. *C.MDBX_env)
 	CHandle() unsafe.Pointer
+	Path() string
 }
 
 type RwDB interface {

db/kv/mdbx/kv_mdbx.go

@@ -172,27 +172,6 @@ func (opts MdbxOpts) InMem(tb testing.TB, tmpDir string) MdbxOpts {
 	return opts
 }
 
-var pathDbMap = map[string]kv.RoDB{}
-var pathDbMapLock sync.Mutex
-
-func addToPathDbMap(path string, db kv.RoDB) {
-	pathDbMapLock.Lock()
-	defer pathDbMapLock.Unlock()
-	pathDbMap[path] = db
-}
-
-func removeFromPathDbMap(path string) {
-	pathDbMapLock.Lock()
-	defer pathDbMapLock.Unlock()
-	delete(pathDbMap, path)
-}
-
-func PathDbMap() map[string]kv.RoDB {
-	pathDbMapLock.Lock()
-	defer pathDbMapLock.Unlock()
-	return maps.Clone(pathDbMap)
-}
-
 var ErrDBDoesNotExists = errors.New("can't create database - because opening in `Accede` mode. probably another (main) process can create it")
 
 func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) {
@@ -432,7 +411,6 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) {
 
 	}
 	db.path = opts.path
-	addToPathDbMap(opts.path, db)
 	if dbg.MdbxLockInRam() && opts.label == dbcfg.ChainDB {
 		log.Info("[dbg] locking db in mem", "label", opts.label)
 		if err := db.View(ctx, func(tx kv.Tx) error { return tx.(*MdbxTx).LockDBInRam() }); err != nil {
@@ -581,7 +559,6 @@ func (db *MdbxKV) Close() {
 			db.log.Warn("failed to remove in-mem db file", "err", err)
 		}
 	}
-	removeFromPathDbMap(db.path)
 }
 
 func (db *MdbxKV) BeginRo(ctx context.Context) (txn kv.Tx, err error) {
@@ -1064,18 +1041,16 @@ func (tx *MdbxTx) Rollback() {
 	if tx.tx == nil {
 		return
 	}
-	defer func() {
-		tx.tx = nil
-		tx.db.trackTxEnd()
-		if tx.readOnly {
-			tx.db.roTxsLimiter.Release(1)
-		} else {
-			runtime.UnlockOSThread()
-		}
-		tx.db.leakDetector.Del(tx.traceID)
-	}()
 	tx.closeCursors()
 	tx.tx.Abort()
+	tx.tx = nil
+	tx.db.trackTxEnd()
+	if tx.readOnly {
+		tx.db.roTxsLimiter.Release(1)
+	} else {
+		runtime.UnlockOSThread()
+	}
+	tx.db.leakDetector.Del(tx.traceID)
 }
 
 func (tx *MdbxTx) SpaceDirty() (uint64, uint64, error) {

db/kv/mdbx/kv_mdbx_temporary.go

@@ -928,6 +928,17 @@ func TestDB_BatchTime(t *testing.T) {
 	}
 }
 
+func BenchmarkDB_BeginRO(b *testing.B) {
+	_db := BaseCaseDBForBenchmark(b)
+	db := _db.(*MdbxKV)
+
+	b.ResetTimer()
+	for i := 1; i <= b.N; i++ {
+		tx, _ := db.BeginRo(context.Background())
+		tx.Rollback()
+	}
+}
+
 func BenchmarkDB_Get(b *testing.B) {
 	_db := BaseCaseDBForBenchmark(b)
 	table := "Table"

db/kv/mdbx/kv_mdbx_test.go

@@ -131,6 +131,7 @@ func NewRemote(v gointerfaces.Version, logger log.Logger, remoteKV remoteproto.K
 }
 
 func (db *DB) PageSize() datasize.ByteSize { panic("not implemented") }
+func (db *DB) Path() string                { panic("not implemented") }
 func (db *DB) ReadOnly() bool              { return true }
 func (db *DB) AllTables() kv.TableCfg      { return db.buckets }
 

db/kv/remotedb/kv_remote.go

@@ -28,7 +28,6 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/erigontech/erigon/db/kv"
-	"github.com/erigontech/erigon/db/kv/mdbx"
 	"github.com/erigontech/erigon/node/paths"
 )
 
@@ -136,7 +135,8 @@ func SetupDbAccess(ctx *cli.Context, metricsMux *http.ServeMux) {
 
 func writeDbList(w http.ResponseWriter, dataDir string) {
 	w.Header().Set("Content-Type", "application/json")
-	m := mdbx.PathDbMap()
+	//m := mdbx.PathDbMap()
+	m := map[string]kv.RoDB{}
 	dbs := make([]string, 0, len(m))
 	for path := range m {
 		dbs = append(dbs, strings.ReplaceAll(strings.TrimPrefix(path, dataDir)[1:], "\\", "/"))
@@ -146,7 +146,8 @@ func writeDbList(w http.ResponseWriter, dataDir string) {
 }
 
 func writeDbTables(w http.ResponseWriter, r *http.Request, dataDir string, dbname string) {
-	m := mdbx.PathDbMap()
+	//m := mdbx.PathDbMap()
+	m := map[string]kv.RoDB{}
 	db, ok := m[filepath.Join(dataDir, dbname)]
 	if !ok {
 		http.Error(w, fmt.Sprintf(`"%s" is not in the list of allowed dbs`, dbname), http.StatusNotFound)
@@ -199,7 +200,8 @@ func writeDbTables(w http.ResponseWriter, r *http.Request, dataDir string, dbnam
 }
 
 func writeDbRead(w http.ResponseWriter, r *http.Request, dataDir string, dbname string, table string, key []byte, offset int64, limit int64) {
-	m := mdbx.PathDbMap()
+	//m := mdbx.PathDbMap()
+	m := map[string]kv.RoDB{}
 	db, ok := m[filepath.Join(dataDir, dbname)]
 	if !ok {
 		fmt.Fprintf(w, "ERROR: path %s is not in the list of allowed paths", dbname)

diagnostics/db.go

@@ -28,7 +28,6 @@ import (
 	"github.com/erigontech/erigon/common"
 	"github.com/erigontech/erigon/common/dbg"
 	"github.com/erigontech/erigon/common/log/v3"
-	"github.com/erigontech/erigon/diagnostics/diaglib"
 )
 
 var ErrorUnsupportedPlatform = errors.New("unsupported platform")
@@ -85,13 +84,6 @@ func LogMemStats(ctx context.Context, logger log.Logger) {
 			l := v.Fields()
 			l = append(l, "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys))
 
-			diaglib.Send(diaglib.MemoryStats{
-				Alloc:       m.Alloc,
-				Sys:         m.Sys,
-				OtherFields: v.Fields(),
-				Timestamp:   time.Now(),
-			})
-
 			logger.Info("[mem] memory stats", l...)
 			UpdatePrometheusVirtualMemStats(vm)
 		}

diagnostics/mem/common.go

@@ -61,14 +61,13 @@ const (
 	// Use localItemKey to create those keys.
 	dbLocalSeq = "seq"
 )
-
 const (
 	dbNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped.
 	dbCleanupCycle   = time.Hour      // Time period for running the expiration task.
 	dbVersion        = 10
 
-	dbSyncBytesThreshold = 5 * datasize.MB // see BenchmarkSyncPeriodDefault
-	dbSyncPeriod         = 2 * time.Second
+	dbSyncBytesThreshold = 5 * datasize.MB // see BenchmarkNodeDBGeometry
+	dbSyncPeriod         = 2 * time.Second // see BenchmarkNodeDBGeometry
 )
 
 var (
@@ -108,6 +107,7 @@ func newMemoryDB(ctx context.Context, logger log.Logger, tmpDir string) (*DB, er
 	db, err := mdbx.New(dbcfg.SentryDB, logger).
 		InMem(nil, tmpDir).
 		WithTableCfg(bucketsConfig).
+		PageSize(4 * datasize.KB).
 		MapSize(1 * datasize.GB).
 		Open(ctx)
 	if err != nil {
@@ -123,12 +123,14 @@ func newMemoryDB(ctx context.Context, logger log.Logger, tmpDir string) (*DB, er
 // newPersistentDB creates/opens a persistent node database,
 // also flushing its contents in case of a version mismatch.
 func newPersistentDB(ctx context.Context, logger log.Logger, path string) (*DB, error) {
+	// see `BenchmarkNodeDBGeometry`
 	db, err := mdbx.New(dbcfg.SentryDB, logger).
 		Path(path).
 		WithTableCfg(bucketsConfig).
+		PageSize(4 * datasize.KB).
 		MapSize(8 * datasize.GB).
-		GrowthStep(16 * datasize.MB).
-		Flags(func(f uint) uint { return f&^mdbx1.Durable | mdbx1.SafeNoSync }).
+		GrowthStep(2 * datasize.MB).
+		Flags(func(f uint) uint { return f&^mdbx1.Durable | mdbx1.SafeNoSync | mdbx1.WriteMap }).
 		SyncBytes(dbSyncBytesThreshold).
 		SyncPeriod(dbSyncPeriod).
 		DirtySpace(uint64(32 * datasize.MB)).