Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion db.go
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,8 @@ func (db *DB) close() (err error) {
select {
case db.flushChan <- db.mt:
db.imm = append(db.imm, db.mt) // Flusher will attempt to remove this from s.imm.
db.mt = nil // Will segfault if we try writing!
db.mt.releaseWAL()
db.mt = nil // Will segfault if we try writing!
db.opt.Debugf("pushed to flush chan\n")
return true
default:
Expand Down Expand Up @@ -1058,6 +1059,7 @@ func (db *DB) ensureRoomForWrite() error {
db.mt.sl.MemSize(), len(db.flushChan))
// We manage to push this task. Let's modify imm.
db.imm = append(db.imm, db.mt)
db.mt.releaseWAL()
db.mt, err = db.newMemTable()
if err != nil {
return y.Wrapf(err, "cannot create new mem table")
Expand Down Expand Up @@ -1882,6 +1884,7 @@ func (db *DB) DropPrefix(prefixes ...[]byte) error {
defer db.lock.Unlock()

db.imm = append(db.imm, db.mt)
db.mt.releaseWAL()
for _, memtable := range db.imm {
if memtable.sl.Empty() {
memtable.DecrRef()
Expand Down
24 changes: 24 additions & 0 deletions memtable.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ func (db *DB) openMemTables(opt Options) error {
}
// These should no longer be written to. So, make them part of the imm.
db.imm = append(db.imm, mt)
// WAL data has been replayed into the skiplist — release the mmap.
mt.releaseWAL()
}
if len(fids) != 0 {
db.nextMemFid = fids[len(fids)-1]
Expand Down Expand Up @@ -222,6 +224,28 @@ func (mt *memTable) DecrRef() {
mt.sl.DecrRef()
}

// releaseWAL unmaps the WAL mmap for an immutable memtable. After the WAL has been replayed
// into the skiplist (via UpdateSkipList), the mmap'd region is no longer needed — all reads
// go through the skiplist. On 32-bit systems this frees ~128 MB of virtual address space per
// memtable, which is critical given the ~3 GB user-space limit.
func (mt *memTable) releaseWAL() {
if mt.wal == nil || mt.wal.Fd == nil {
return
}
path := mt.wal.Fd.Name()
if mt.wal.Data != nil {
_ = z.Munmap(mt.wal.Data)
mt.wal.Data = nil
}
_ = mt.wal.Fd.Close()
mt.wal.Fd = nil
// Replace the OnClose callback: we've already unmapped the data and
// closed the fd, so just remove the file when the skiplist is freed.
mt.sl.OnClose = func() {
_ = os.Remove(path)
}
}

func (mt *memTable) replayFunction(opt Options) func(Entry, valuePointer) error {
first := true
return func(e Entry, _ valuePointer) error { // Function for replaying.
Expand Down
Loading