From 33c6cde16d8df899a5ad7765e555547d16c5cedd Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 15:56:36 -0400 Subject: [PATCH 01/13] fix(seidb): fail closed on live memiavl WAL reads Use an immutable WAL view for read-only replay so digest tooling never repairs or copies the live changelog. Return actionable retry errors when a point-in-time view cannot safely reach the requested version. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 85 +++++- sei-db/state_db/sc/memiavl/db_test.go | 130 +++++++++ .../seidb/operations/evm_logical_digest.go | 32 ++- .../cmd/seidb/operations/memiavl_open_test.go | 61 +++++ sei-db/wal/changelog.go | 19 ++ sei-db/wal/readonly.go | 248 ++++++++++++++++++ sei-db/wal/readonly_test.go | 240 +++++++++++++++++ 7 files changed, 800 insertions(+), 15 deletions(-) create mode 100644 sei-db/tools/cmd/seidb/operations/memiavl_open_test.go create mode 100644 sei-db/wal/readonly.go create mode 100644 sei-db/wal/readonly_test.go diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index ddd18f675c..feb1fee485 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -27,7 +27,21 @@ import ( const LockFileName = "LOCK" -var errReadOnly = errors.New("db is read-only") +var ( + errReadOnly = errors.New("db is read-only") + + // ErrReadOnlyWALCorrupt means a read-only open observed an incomplete, + // corrupt, or concurrently recovering changelog. The source WAL is left + // untouched; callers can retry after the writer finishes its current WAL + // operation. + ErrReadOnlyWALCorrupt = errors.New("read-only changelog is incomplete or corrupt") + + // ErrReadOnlyWALUnavailable means the immutable WAL view cannot replay + // every version from the selected snapshot through the requested target. + // The live writer may have pruned or advanced the changelog while the + // reader opened it; callers can retry against a new point-in-time view. + ErrReadOnlyWALUnavailable = errors.New("read-only changelog cannot reach the requested version") +) // DB implements DB-like functionalities on top of MultiTree: // - async snapshot rewriting @@ -158,9 +172,26 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { ) }() var ( - err error - fileLock FileLock + err error + fileLock FileLock + mtree *MultiTree + streamHandler wal.ChangelogWAL ) + defer func() { + if _err == nil { + return + } + if streamHandler != nil { + _ = streamHandler.Close() + } + if mtree != nil { + _ = mtree.Close() + } + if fileLock != nil { + _ = fileLock.Unlock() + _ = fileLock.Destroy() + } + }() if err := opts.Validate(); err != nil { return nil, fmt.Errorf("invalid commit store options: %w", err) } @@ -194,19 +225,28 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { } path := filepath.Join(opts.Dir, snapshot) - mtree, err := LoadMultiTree(context.Background(), path, opts) + mtree, err = LoadMultiTree(context.Background(), path, opts) if err != nil { return nil, err } // Snapshot mmap files are loaded with MADV_RANDOM in OpenSnapshot(). - // MemIAVL owns changelog lifecycle: always open the WAL here. - // Even in read-only mode we may need WAL replay to reconstruct non-snapshot versions. - streamHandler, err := wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ - WriteBufferSize: opts.AsyncCommitBuffer, - }) + // MemIAVL owns changelog lifecycle: always open the WAL here. Read-only + // callers still need replay to reconstruct non-snapshot versions, but they + // must not use the writable opener: it repairs a torn tail by truncating it + // and completes interrupted WAL truncations by renaming or removing files. + if opts.ReadOnly { + streamHandler, err = wal.OpenReadOnlyChangelogWAL(utils.GetChangelogPath(opts.Dir)) + } else { + streamHandler, err = wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ + WriteBufferSize: opts.AsyncCommitBuffer, + }) + } if err != nil { + if opts.ReadOnly && errors.Is(err, wal.ErrCorrupt) { + return nil, fmt.Errorf("%w; source WAL was not modified: %w", ErrReadOnlyWALCorrupt, err) + } return nil, fmt.Errorf("failed to open changelog WAL: %w", err) } @@ -221,6 +261,23 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { if !walHasEntries { walIndexDelta = mtree.WorkingCommitInfo().Version - 1 } + if opts.ReadOnly && walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { + firstIndex, firstErr := streamHandler.FirstOffset() + if firstErr != nil { + return nil, fmt.Errorf("read changelog first offset: %w", firstErr) + } + if firstIndex > math.MaxInt64 { + return nil, fmt.Errorf("%w: first WAL offset %d overflows int64", ErrReadOnlyWALUnavailable, firstIndex) + } + firstVersion := int64(firstIndex) + walIndexDelta + firstNeeded := utils.NextVersion(mtree.Version(), mtree.initialVersion.Load()) + if firstVersion > firstNeeded { + snapshotVersion := mtree.Version() + return nil, fmt.Errorf("%w: selected snapshot version %d needs changelog version %d, "+ + "but the immutable WAL view starts at version %d", + ErrReadOnlyWALUnavailable, snapshotVersion, firstNeeded, firstVersion) + } + } // Replay WAL to catch up to target version (if WAL has entries) if walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { @@ -230,6 +287,11 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { } logger.Info("finished replay and caught up to target version", "version", targetVersion) } + if opts.ReadOnly && targetVersion > 0 && mtree.Version() != targetVersion { + reached := mtree.Version() + return nil, fmt.Errorf("%w: requested %d, reached %d", + ErrReadOnlyWALUnavailable, targetVersion, reached) + } if opts.LoadForOverwriting && targetVersion > 0 { currentSnapshot, err := os.Readlink(currentPath(opts.Dir)) @@ -296,6 +358,10 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { snapshotWriterPool: workerPool, opts: opts, } + // The DB owns these resources from this point forward. + mtree = nil + streamHandler = nil + fileLock = nil // Apply initial stores on a fresh DB (version 0) so they get persisted to WAL. // This creates the trees and populates pendingLogEntry, which will be written @@ -307,6 +373,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { upgrades = append(upgrades, &proto.TreeNameUpgrade{Name: name}) } if err := db.ApplyUpgrades(upgrades); err != nil { + _ = db.Close() return nil, fmt.Errorf("failed to apply initial stores: %w", err) } } diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index 3cddd00f77..d804189902 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "runtime/debug" + "sort" "strconv" "sync" "testing" @@ -1116,3 +1117,132 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) { require.NoError(t, err) require.Equal(t, "snapshot-1", target) } + +func TestReadOnlyOpenRejectsTornWALWithoutRepair(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) + require.NoError(t, err) + for i := 0; i < 3; i++ { + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[i], + }})) + _, err := db.Commit() + require.NoError(t, err) + } + require.NoError(t, db.Close()) + + segment := lastMemiAVLWALSegment(t, dir) + file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) + require.NoError(t, err) + _, err = file.Write([]byte{0x10}) + require.NoError(t, err) + require.NoError(t, file.Close()) + before, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + + _, err = OpenDB(0, Options{Dir: dir, ReadOnly: true}) + require.ErrorIs(t, err, ErrReadOnlyWALCorrupt) + after, readErr := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, readErr) + require.Equal(t, before, after, "read-only open must leave a torn live tail untouched") + + repaired, err := OpenDB(0, Options{Dir: dir}) + require.NoError(t, err, "the writable owner must retain the existing tail-repair behavior") + require.Equal(t, int64(3), repaired.Version()) + require.NoError(t, repaired.Close()) +} + +func TestReadOnlyOpenRejectsWALGap(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) + require.NoError(t, err) + for i := 0; i < 3; i++ { + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[i], + }})) + _, err := db.Commit() + require.NoError(t, err) + } + require.NoError(t, db.GetWAL().TruncateBefore(2)) + require.NoError(t, db.Close()) + + _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true}) + require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) + require.Contains(t, err.Error(), "needs changelog version 1") + require.Contains(t, err.Error(), "starts at version 2") +} + +func TestReadOnlyOpenRejectsShortWAL(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) + require.NoError(t, err) + for i := 0; i < 3; i++ { + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[i], + }})) + _, err := db.Commit() + require.NoError(t, err) + } + require.NoError(t, db.GetWAL().TruncateAfter(2)) + require.NoError(t, db.Close()) + + _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true}) + require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) + require.Contains(t, err.Error(), "requested 3, reached 2") +} + +func TestOpenDBFailureReleasesFileLock(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) + require.NoError(t, err) + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[0], + }})) + _, err = db.Commit() + require.NoError(t, err) + require.NoError(t, db.Close()) + + require.NoError(t, os.Remove(currentPath(dir))) + _, err = OpenDB(1, Options{Dir: dir, LoadForOverwriting: true}) + require.ErrorContains(t, err, "fail to read current version") + + lock, err := LockFile(filepath.Join(dir, LockFileName)) + require.NoError(t, err, "failed OpenDB must release its exclusive lock") + require.NoError(t, lock.Unlock()) + require.NoError(t, lock.Destroy()) +} + +func lastMemiAVLWALSegment(t *testing.T, dir string) string { + t.Helper() + entries, err := os.ReadDir(utils.GetChangelogPath(dir)) + require.NoError(t, err) + var names []string + for _, entry := range entries { + if !entry.IsDir() && len(entry.Name()) == 20 { + names = append(names, entry.Name()) + } + } + require.NotEmpty(t, names) + sort.Strings(names) + return filepath.Join(utils.GetChangelogPath(dir), names[len(names)-1]) +} diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 79ecd140c6..49a3693907 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -81,12 +81,14 @@ const ( // that exact height (or --height 0 for the current symlink). This is the // preferred mode whenever the target height lines up with an existing // snapshot boundary. -// - replay (SLOW): opens a read-only DB, replays the changelog up to -// --height, then walks the in-memory/mmap tree. Roughly an order of -// magnitude slower than snapshot (changelog replay + per-leaf tree walk -// instead of a sequential file read). Use it only when no snapshot exists -// at the target height — e.g. nodes whose snapshot rewrite lags the tip, so -// an arbitrary comparison height has no snapshot- on disk. +// - replay (SLOW): opens a non-mutating read-only WAL view, replays the +// changelog up to --height, then walks the in-memory/mmap tree. Roughly an +// order of magnitude slower than snapshot (changelog replay + per-leaf tree +// walk instead of a sequential file read). If a live writer leaves a torn +// tail in view, replay fails and asks the operator to rerun instead of +// repairing the source WAL. Use it only when no snapshot exists at the +// target height — e.g. nodes whose snapshot rewrite lags the tip, so an +// arbitrary comparison height has no snapshot- on disk. // // The flatkv side is always a pebble WAL-replay-to-height and is fast // regardless. So when comparing across nodes, pick a height that is an existing @@ -1111,8 +1113,26 @@ func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) ZeroCopy: true, }) if err != nil { + if errors.Is(err, memiavl.ErrReadOnlyWALCorrupt) { + return nil, fmt.Errorf("memiavl changelog tail is incomplete, corrupt, or changing; "+ + "live WAL was not modified; rerun the command, and if the error persists after stopping seid, "+ + "repair the WAL offline: %w", err) + } + if errors.Is(err, memiavl.ErrReadOnlyWALUnavailable) { + return nil, fmt.Errorf("the immutable memiavl changelog view could not reach height %d; "+ + "live WAL was not modified; rerun the command: %w", height, err) + } return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } + if height > 0 && db.Version() != height { + versionErr := fmt.Errorf("memiavl replay version mismatch: requested %d, reached %d; "+ + "the live changelog did not provide a complete path to the target; rerun the command", + height, db.Version()) + if closeErr := db.Close(); closeErr != nil { + return nil, errors.Join(versionErr, fmt.Errorf("close memiavl read-only replay: %w", closeErr)) + } + return nil, versionErr + } return db, nil } diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go new file mode 100644 index 0000000000..ad06579c35 --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -0,0 +1,61 @@ +package operations + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +func TestOpenMemiAVLReplayReadOnlyReportsRetryWithoutRepair(t *testing.T) { + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}}, + }})) + _, err := store.Commit() + require.NoError(t, err) + require.NoError(t, store.Close()) + + dbDir := utils.GetCosmosSCStorePath(homeDir) + segment := lastOperationsMemiAVLWALSegment(t, dbDir) + file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) + require.NoError(t, err) + _, err = file.Write([]byte{0x10}) + require.NoError(t, err) + require.NoError(t, file.Close()) + before, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + + _, err = openMemiAVLReplayReadOnly(dbDir, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "live WAL was not modified") + require.Contains(t, err.Error(), "rerun the command") + + after, readErr := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, readErr) + require.Equal(t, before, after) +} + +func lastOperationsMemiAVLWALSegment(t *testing.T, dbDir string) string { + t.Helper() + changelogDir := utils.GetChangelogPath(dbDir) + entries, err := os.ReadDir(changelogDir) + require.NoError(t, err) + var names []string + for _, entry := range entries { + if !entry.IsDir() && len(entry.Name()) == 20 { + names = append(names, entry.Name()) + } + } + require.NotEmpty(t, names) + sort.Strings(names) + return filepath.Join(changelogDir, names[len(names)-1]) +} diff --git a/sei-db/wal/changelog.go b/sei-db/wal/changelog.go index fadb054f31..c3071b9679 100644 --- a/sei-db/wal/changelog.go +++ b/sei-db/wal/changelog.go @@ -26,6 +26,25 @@ func NewChangelogWAL(dir string, config Config) (ChangelogWAL, error) { ) } +// OpenReadOnlyChangelogWAL opens an immutable point-in-time view of the +// changelog segment files. It never creates, truncates, removes, or renames WAL +// files. A torn tail or an in-progress recovery marker returns ErrCorrupt so +// callers can fail and retry after the writer moves on. +func OpenReadOnlyChangelogWAL(dir string) (ChangelogWAL, error) { + readOnly, err := openReadOnlyWAL( + dir, + func(data []byte) (proto.ChangelogEntry, error) { + var entry proto.ChangelogEntry + err := entry.Unmarshal(data) + return entry, err + }, + ) + if err != nil { + return nil, err + } + return readOnly, nil +} + // FindFirstOffsetAfterVersion returns the first WAL offset whose entry version is // strictly greater than targetVersion. If no such entry exists, it returns // lastOffset+1. Changelog versions are monotonic, but empty blocks can advance diff --git a/sei-db/wal/readonly.go b/sei-db/wal/readonly.go new file mode 100644 index 0000000000..884ca77394 --- /dev/null +++ b/sei-db/wal/readonly.go @@ -0,0 +1,248 @@ +package wal + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + + tidwallwal "github.com/tidwall/wal" +) + +var ( + // ErrReadOnly is returned when a caller tries to mutate a read-only WAL. + ErrReadOnly = errors.New("WAL is read-only") + + // ErrCorrupt identifies a malformed or unstable WAL view. It aliases the + // underlying tidwall sentinel so callers do not need to import the storage + // implementation only to decide whether a read-only open can be retried. + ErrCorrupt = tidwallwal.ErrCorrupt +) + +type readOnlySegment struct { + name string + index uint64 +} + +type readOnlyEntry struct { + file *os.File + dataOffset int64 + size int +} + +// readOnlyWAL is an immutable view of the plain segment files present when it +// opens. It does not use tidwall/wal.Open because that function creates files, +// opens the tail for writing, and completes interrupted truncations by removing +// and renaming segment files. +type readOnlyWAL[T any] struct { + unmarshal UnmarshalFn[T] + files []*os.File + entries []readOnlyEntry + firstOffset uint64 + closed atomic.Bool + closeOnce sync.Once + closeErr error +} + +func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[T], error) { + segments, err := listReadOnlySegments(dir) + if err != nil { + return nil, err + } + + log := &readOnlyWAL[T]{unmarshal: unmarshal} + cleanup := func(err error) (*readOnlyWAL[T], error) { + _ = log.Close() + return nil, err + } + + var nextIndex uint64 + for i, segment := range segments { + if i > 0 && segment.index != nextIndex { + return cleanup(fmt.Errorf("%w: segment %s starts at index %d, expected %d", + ErrCorrupt, segment.name, segment.index, nextIndex)) + } + + path := filepath.Join(dir, segment.name) + file, err := os.Open(filepath.Clean(path)) + if err != nil { + return cleanup(fmt.Errorf("open WAL segment %s: %w", path, err)) + } + log.files = append(log.files, file) + + info, err := file.Stat() + if err != nil { + return cleanup(fmt.Errorf("stat WAL segment %s: %w", path, err)) + } + if !info.Mode().IsRegular() { + return cleanup(fmt.Errorf("%w: WAL segment %s is not a regular file", ErrCorrupt, path)) + } + + data, err := io.ReadAll(io.NewSectionReader(file, 0, info.Size())) + if err != nil { + return cleanup(fmt.Errorf("read WAL segment %s: %w", path, err)) + } + if int64(len(data)) != info.Size() { + return cleanup(fmt.Errorf("%w: WAL segment %s changed while it was read", + ErrCorrupt, path)) + } + + entries, err := indexReadOnlySegment(file, data) + if err != nil { + return cleanup(fmt.Errorf("index WAL segment %s: %w", path, err)) + } + if len(entries) == 0 && i != len(segments)-1 { + return cleanup(fmt.Errorf("%w: non-tail WAL segment %s is empty", ErrCorrupt, path)) + } + if len(log.entries) == 0 && len(entries) > 0 { + log.firstOffset = segment.index + } + log.entries = append(log.entries, entries...) + nextIndex = segment.index + uint64(len(entries)) + } + return log, nil +} + +func listReadOnlySegments(dir string) ([]readOnlySegment, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read WAL directory %s: %w", dir, err) + } + + segments := make([]readOnlySegment, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { + return nil, fmt.Errorf("%w: WAL recovery marker %s is present; retry after the writer finishes", + ErrCorrupt, name) + } + if len(name) != 20 { + continue + } + index, err := strconv.ParseUint(name, 10, 64) + if err != nil || index == 0 { + continue + } + segments = append(segments, readOnlySegment{name: name, index: index}) + } + sort.Slice(segments, func(i, j int) bool { + return segments[i].index < segments[j].index + }) + return segments, nil +} + +func indexReadOnlySegment(file *os.File, data []byte) ([]readOnlyEntry, error) { + entries := make([]readOnlyEntry, 0) + for pos := 0; pos < len(data); { + recordLen, err := loadNextBinaryEntry(data[pos:]) + if err != nil { + return nil, err + } + size, prefixLen := binary.Uvarint(data[pos:]) + entrySize := int(size) //nolint:gosec // loadNextBinaryEntry rejects sizes above math.MaxInt32. + entries = append(entries, readOnlyEntry{ + file: file, + dataOffset: int64(pos + prefixLen), + size: entrySize, + }) + pos += recordLen + } + return entries, nil +} + +func (log *readOnlyWAL[T]) Write(T) error { + return ErrReadOnly +} + +func (log *readOnlyWAL[T]) TruncateBefore(uint64) error { + return ErrReadOnly +} + +func (log *readOnlyWAL[T]) TruncateAfter(uint64) error { + return ErrReadOnly +} + +func (log *readOnlyWAL[T]) TruncateAll() error { + return ErrReadOnly +} + +func (log *readOnlyWAL[T]) FirstOffset() (uint64, error) { + if log.closed.Load() { + return 0, os.ErrClosed + } + if len(log.entries) == 0 { + return 0, nil + } + return log.firstOffset, nil +} + +func (log *readOnlyWAL[T]) LastOffset() (uint64, error) { + if log.closed.Load() { + return 0, os.ErrClosed + } + if len(log.entries) == 0 { + return 0, nil + } + return log.firstOffset + uint64(len(log.entries)) - 1, nil +} + +func (log *readOnlyWAL[T]) ReadAt(index uint64) (T, error) { + var zero T + if log.closed.Load() { + return zero, os.ErrClosed + } + if index < log.firstOffset || index-log.firstOffset >= uint64(len(log.entries)) { + return zero, fmt.Errorf("read WAL offset %d: out of range", index) + } + + entry := log.entries[index-log.firstOffset] + data := make([]byte, entry.size) + if _, err := entry.file.ReadAt(data, entry.dataOffset); err != nil { + return zero, fmt.Errorf("read WAL offset %d: %w", index, err) + } + value, err := log.unmarshal(data) + if err != nil { + return zero, fmt.Errorf("unmarshal WAL offset %d: %w", index, err) + } + return value, nil +} + +func (log *readOnlyWAL[T]) Replay(start, end uint64, processFn func(index uint64, entry T) error) error { + if end < start { + return nil + } + for index := start; index <= end; index++ { + entry, err := log.ReadAt(index) + if err != nil { + return err + } + if err := processFn(index, entry); err != nil { + return fmt.Errorf("process WAL offset %d: %w", index, err) + } + } + return nil +} + +func (log *readOnlyWAL[T]) Close() error { + log.closeOnce.Do(func() { + log.closed.Store(true) + var errs []error + for _, file := range log.files { + if err := file.Close(); err != nil { + errs = append(errs, err) + } + } + log.closeErr = errors.Join(errs...) + }) + return log.closeErr +} diff --git a/sei-db/wal/readonly_test.go b/sei-db/wal/readonly_test.go new file mode 100644 index 0000000000..9fb0ebc3c0 --- /dev/null +++ b/sei-db/wal/readonly_test.go @@ -0,0 +1,240 @@ +package wal + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +func TestOpenReadOnlyChangelogWALReplaysWithoutMutation(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + writeReadOnlyTestData(t, writable) + require.NoError(t, writable.Close()) + + before := snapshotWALFiles(t, dir) + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + + first, err := readOnly.FirstOffset() + require.NoError(t, err) + require.Equal(t, uint64(1), first) + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Equal(t, uint64(3), last) + + var names []string + require.NoError(t, readOnly.Replay(first, last, func(_ uint64, entry proto.ChangelogEntry) error { + names = append(names, entry.Changesets[0].Name) + return nil + })) + require.Equal(t, []string{"test", "test", "test"}, names) + require.Equal(t, before, snapshotWALFiles(t, dir)) + + require.ErrorIs(t, readOnly.Write(proto.ChangelogEntry{}), ErrReadOnly) + require.ErrorIs(t, readOnly.TruncateBefore(2), ErrReadOnly) + require.ErrorIs(t, readOnly.TruncateAfter(2), ErrReadOnly) + require.NoError(t, readOnly.Close()) + require.NoError(t, readOnly.Close()) +} + +func TestOpenReadOnlyChangelogWALRejectsTornTailWithoutRepair(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + writeReadOnlyTestData(t, writable) + require.NoError(t, writable.Close()) + + segment := lastPlainWALSegment(t, dir) + file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) + require.NoError(t, err) + _, err = file.Write([]byte{0x10}) // declares a 16-byte record whose payload has not arrived + require.NoError(t, err) + require.NoError(t, file.Close()) + before := snapshotWALFiles(t, dir) + + _, err = OpenReadOnlyChangelogWAL(dir) + require.ErrorIs(t, err, ErrCorrupt) + require.Equal(t, before, snapshotWALFiles(t, dir), "read-only open must not repair the source tail") +} + +func TestOpenReadOnlyChangelogWALKeepsPointInTimeView(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + require.NoError(t, writable.Write(proto.ChangelogEntry{Version: 1})) + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, readOnly.Close()) }) + + require.NoError(t, writable.Write(proto.ChangelogEntry{Version: 2})) + require.NoError(t, writable.Close()) + + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Equal(t, uint64(1), last) + _, err = readOnly.ReadAt(2) + require.Error(t, err) +} + +func TestOpenReadOnlyChangelogWALRejectsRecoveryMarkers(t *testing.T) { + for _, suffix := range []string{".START", ".END"} { + t.Run(suffix, func(t *testing.T) { + dir := t.TempDir() + marker := filepath.Join(dir, "00000000000000000001"+suffix) + require.NoError(t, os.WriteFile(marker, nil, 0o600)) + before := snapshotWALFiles(t, dir) + + _, err := OpenReadOnlyChangelogWAL(dir) + require.ErrorIs(t, err, ErrCorrupt) + require.Equal(t, before, snapshotWALFiles(t, dir), + "read-only open must not complete writable WAL recovery") + }) + } +} + +func TestOpenReadOnlyChangelogWALEmptyDirectory(t *testing.T) { + dir := t.TempDir() + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + first, err := readOnly.FirstOffset() + require.NoError(t, err) + require.Zero(t, first) + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Zero(t, last) + require.NoError(t, readOnly.Close()) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, "read-only open must not create an initial segment") +} + +func TestOpenReadOnlyChangelogWALDoesNotCreateMissingDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "missing") + + _, err := OpenReadOnlyChangelogWAL(dir) + require.Error(t, err) + require.NoDirExists(t, dir) +} + +func TestOpenReadOnlyChangelogWALConcurrentWriter(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + + const versions = 500 + done := make(chan struct{}) + writerErr := make(chan error, 1) + go func() { + var runErr error + for version := 1; version <= versions && runErr == nil; version++ { + runErr = writable.Write(proto.ChangelogEntry{Version: int64(version)}) + if runErr != nil || version <= 20 || version%10 != 0 { + continue + } + first, err := writable.FirstOffset() + if err != nil { + runErr = err + break + } + keepFrom := uint64(version - 20) + if keepFrom > first { + runErr = writable.TruncateBefore(keepFrom) + } + } + if closeErr := writable.Close(); runErr == nil { + runErr = closeErr + } + writerErr <- runErr + close(done) + }() + +reading: + for { + select { + case <-done: + break reading + default: + } + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + if err != nil { + continue // source changed while opening; fail-closed and retry is valid + } + first, err := readOnly.FirstOffset() + require.NoError(t, err) + last, err := readOnly.LastOffset() + require.NoError(t, err) + if first > 0 { + require.GreaterOrEqual(t, last, first) + entry, err := readOnly.ReadAt(last) + require.NoError(t, err) + require.Equal(t, int64(last), entry.Version) + } + require.NoError(t, readOnly.Close()) + } + require.NoError(t, <-writerErr) + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + defer func() { require.NoError(t, readOnly.Close()) }() + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Equal(t, uint64(versions), last) + entry, err := readOnly.ReadAt(last) + require.NoError(t, err) + require.Equal(t, int64(versions), entry.Version) +} + +func lastPlainWALSegment(t *testing.T, dir string) string { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + var names []string + for _, entry := range entries { + if !entry.IsDir() && len(entry.Name()) == 20 { + names = append(names, entry.Name()) + } + } + require.NotEmpty(t, names) + sort.Strings(names) + return filepath.Join(dir, names[len(names)-1]) +} + +func snapshotWALFiles(t *testing.T, dir string) map[string][]byte { + t.Helper() + files := make(map[string][]byte) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + for _, entry := range entries { + if entry.IsDir() { + continue + } + data, err := os.ReadFile(filepath.Clean(filepath.Join(dir, entry.Name()))) + require.NoError(t, err) + files[entry.Name()] = data + } + return files +} + +func writeReadOnlyTestData(t *testing.T, changelog ChangelogWAL) { + t.Helper() + for i, changeset := range ChangeSets { + require.NoError(t, changelog.Write(proto.ChangelogEntry{ + Version: int64(i + 1), + Changesets: []*proto.NamedChangeSet{{ + Name: "test", + Changeset: changeset, + }}, + })) + } +} From e8d17789d3e217fb80b46b6673685e09002beab2 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 16:03:53 -0400 Subject: [PATCH 02/13] fix(wal): classify vanished segments as retryable Treat a segment removed between directory listing and open as WAL churn so read-only callers return the actionable retry path. Co-authored-by: Cursor --- sei-db/wal/readonly.go | 4 ++++ sei-db/wal/readonly_test.go | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/sei-db/wal/readonly.go b/sei-db/wal/readonly.go index 884ca77394..8cc2d3ed01 100644 --- a/sei-db/wal/readonly.go +++ b/sei-db/wal/readonly.go @@ -73,6 +73,10 @@ func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[ path := filepath.Join(dir, segment.name) file, err := os.Open(filepath.Clean(path)) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return cleanup(fmt.Errorf("%w: WAL segment %s disappeared while opening: %w", + ErrCorrupt, path, err)) + } return cleanup(fmt.Errorf("open WAL segment %s: %w", path, err)) } log.files = append(log.files, file) diff --git a/sei-db/wal/readonly_test.go b/sei-db/wal/readonly_test.go index 9fb0ebc3c0..ebace3693a 100644 --- a/sei-db/wal/readonly_test.go +++ b/sei-db/wal/readonly_test.go @@ -126,6 +126,16 @@ func TestOpenReadOnlyChangelogWALDoesNotCreateMissingDirectory(t *testing.T) { require.NoDirExists(t, dir) } +func TestOpenReadOnlyChangelogWALClassifiesVanishedSegmentAsCorrupt(t *testing.T) { + dir := t.TempDir() + segment := filepath.Join(dir, "00000000000000000001") + require.NoError(t, os.Symlink(filepath.Join(dir, "vanished"), segment)) + + _, err := OpenReadOnlyChangelogWAL(dir) + require.ErrorIs(t, err, ErrCorrupt) + require.ErrorIs(t, err, os.ErrNotExist) +} + func TestOpenReadOnlyChangelogWALConcurrentWriter(t *testing.T) { dir := t.TempDir() writable, err := NewChangelogWAL(dir, Config{}) From 5879f4ef474cfe421bf0665514e36b929e8ca021 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 16:31:25 -0400 Subject: [PATCH 03/13] fix(wal): stabilize live read-only views Use the complete tail prefix during concurrent writes, treat a missing changelog as empty, and classify deferred read failures through the retryable WAL error path. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 14 +++- sei-db/state_db/sc/memiavl/db_test.go | 79 ++++++++++++++++++- .../seidb/operations/evm_logical_digest.go | 9 --- .../cmd/seidb/operations/memiavl_open_test.go | 6 +- sei-db/wal/changelog.go | 4 +- sei-db/wal/readonly.go | 27 ++++++- sei-db/wal/readonly_test.go | 65 +++++++++++++-- 7 files changed, 173 insertions(+), 31 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index feb1fee485..302c5eb8b7 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -236,6 +236,12 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { // callers still need replay to reconstruct non-snapshot versions, but they // must not use the writable opener: it repairs a torn tail by truncating it // and completes interrupted WAL truncations by renaming or removing files. + classifyReadOnlyWALError := func(err error) error { + if opts.ReadOnly && errors.Is(err, wal.ErrCorrupt) { + return fmt.Errorf("%w; source WAL was not modified: %w", ErrReadOnlyWALCorrupt, err) + } + return err + } if opts.ReadOnly { streamHandler, err = wal.OpenReadOnlyChangelogWAL(utils.GetChangelogPath(opts.Dir)) } else { @@ -244,8 +250,9 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { }) } if err != nil { - if opts.ReadOnly && errors.Is(err, wal.ErrCorrupt) { - return nil, fmt.Errorf("%w; source WAL was not modified: %w", ErrReadOnlyWALCorrupt, err) + err = classifyReadOnlyWALError(err) + if errors.Is(err, ErrReadOnlyWALCorrupt) { + return nil, err } return nil, fmt.Errorf("failed to open changelog WAL: %w", err) } @@ -255,6 +262,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { var walHasEntries bool walIndexDelta, walHasEntries, err = computeWALIndexDelta(streamHandler) if err != nil { + err = classifyReadOnlyWALError(err) return nil, fmt.Errorf("failed to compute WAL index delta: %w", err) } // If WAL is empty, set delta so first WAL entry aligns with NextVersion(). @@ -283,7 +291,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { if walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { logger.Info("Start catching up and replaying the MemIAVL changelog file") if err := mtree.Catchup(context.Background(), streamHandler, walIndexDelta, targetVersion); err != nil { - return nil, err + return nil, classifyReadOnlyWALError(err) } logger.Info("finished replay and caught up to target version", "version", targetVersion) } diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index d804189902..752505dafb 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -2,6 +2,7 @@ package memiavl import ( "context" + "encoding/binary" "encoding/hex" "os" "path/filepath" @@ -1118,7 +1119,7 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) { require.Equal(t, "snapshot-1", target) } -func TestReadOnlyOpenRejectsTornWALWithoutRepair(t *testing.T) { +func TestReadOnlyOpenUsesCompleteWALPrefixWithoutRepair(t *testing.T) { dir := t.TempDir() db, err := OpenDB(0, Options{ Dir: dir, @@ -1145,12 +1146,17 @@ func TestReadOnlyOpenRejectsTornWALWithoutRepair(t *testing.T) { before, err := os.ReadFile(filepath.Clean(segment)) require.NoError(t, err) - _, err = OpenDB(0, Options{Dir: dir, ReadOnly: true}) - require.ErrorIs(t, err, ErrReadOnlyWALCorrupt) + readOnly, err := OpenDB(0, Options{Dir: dir, ReadOnly: true}) + require.NoError(t, err) + require.Equal(t, int64(3), readOnly.Version()) + require.NoError(t, readOnly.Close()) after, readErr := os.ReadFile(filepath.Clean(segment)) require.NoError(t, readErr) require.Equal(t, before, after, "read-only open must leave a torn live tail untouched") + _, err = OpenDB(4, Options{Dir: dir, ReadOnly: true}) + require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) + repaired, err := OpenDB(0, Options{Dir: dir}) require.NoError(t, err, "the writable owner must retain the existing tail-repair behavior") require.Equal(t, int64(3), repaired.Version()) @@ -1206,6 +1212,73 @@ func TestReadOnlyOpenRejectsShortWAL(t *testing.T) { require.Contains(t, err.Error(), "requested 3, reached 2") } +func TestReadOnlyOpenClassifiesReplayDecodeFailure(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) + require.NoError(t, err) + for i := 0; i < 2; i++ { + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[i], + }})) + _, err = db.Commit() + require.NoError(t, err) + } + require.NoError(t, db.Close()) + + segment := lastMemiAVLWALSegment(t, dir) + data, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + firstSize, firstPrefixLen := binary.Uvarint(data) + require.Positive(t, firstPrefixLen) + require.LessOrEqual(t, firstSize, uint64(len(data)-firstPrefixLen)) + secondPos := firstPrefixLen + int(firstSize) //nolint:gosec // bounded by the segment length above. + secondSize, secondPrefixLen := binary.Uvarint(data[secondPos:]) + require.Positive(t, secondPrefixLen) + require.LessOrEqual(t, secondSize, uint64(len(data)-secondPos-secondPrefixLen)) + secondDataStart := secondPos + secondPrefixLen + secondDataEnd := secondDataStart + int(secondSize) //nolint:gosec // bounded by the segment length above. + for i := secondDataStart; i < secondDataEnd; i++ { + data[i] = 0xff + } + require.NoError(t, os.WriteFile(filepath.Clean(segment), data, 0o600)) + + _, err = OpenDB(2, Options{Dir: dir, ReadOnly: true}) + require.ErrorIs(t, err, ErrReadOnlyWALCorrupt) + after, readErr := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, readErr) + require.Equal(t, data, after, "read-only replay must not repair an unparseable entry") +} + +func TestReadOnlyOpenTreatsMissingWALAsEmpty(t *testing.T) { + dir := t.TempDir() + db, err := OpenDB(0, Options{ + Dir: dir, + CreateIfMissing: true, + InitialStores: []string{"test"}, + }) + require.NoError(t, err) + require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: "test", + Changeset: ChangeSets[0], + }})) + _, err = db.Commit() + require.NoError(t, err) + require.NoError(t, db.RewriteSnapshot(context.Background())) + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(utils.GetChangelogPath(dir))) + + readOnly, err := OpenDB(1, Options{Dir: dir, ReadOnly: true}) + require.NoError(t, err) + require.Equal(t, int64(1), readOnly.Version()) + require.NoError(t, readOnly.Close()) + require.NoDirExists(t, utils.GetChangelogPath(dir)) +} + func TestOpenDBFailureReleasesFileLock(t *testing.T) { dir := t.TempDir() db, err := OpenDB(0, Options{ diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 49a3693907..1dbf7bfec8 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -1124,15 +1124,6 @@ func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) } return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } - if height > 0 && db.Version() != height { - versionErr := fmt.Errorf("memiavl replay version mismatch: requested %d, reached %d; "+ - "the live changelog did not provide a complete path to the target; rerun the command", - height, db.Version()) - if closeErr := db.Close(); closeErr != nil { - return nil, errors.Join(versionErr, fmt.Errorf("close memiavl read-only replay: %w", closeErr)) - } - return nil, versionErr - } return db, nil } diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index ad06579c35..75806b3ee9 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -26,11 +26,7 @@ func TestOpenMemiAVLReplayReadOnlyReportsRetryWithoutRepair(t *testing.T) { dbDir := utils.GetCosmosSCStorePath(homeDir) segment := lastOperationsMemiAVLWALSegment(t, dbDir) - file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) - require.NoError(t, err) - _, err = file.Write([]byte{0x10}) - require.NoError(t, err) - require.NoError(t, file.Close()) + require.NoError(t, os.WriteFile(filepath.Clean(segment), []byte{0x01, 0xff}, 0o600)) before, err := os.ReadFile(filepath.Clean(segment)) require.NoError(t, err) diff --git a/sei-db/wal/changelog.go b/sei-db/wal/changelog.go index c3071b9679..2479c2ea6e 100644 --- a/sei-db/wal/changelog.go +++ b/sei-db/wal/changelog.go @@ -28,8 +28,8 @@ func NewChangelogWAL(dir string, config Config) (ChangelogWAL, error) { // OpenReadOnlyChangelogWAL opens an immutable point-in-time view of the // changelog segment files. It never creates, truncates, removes, or renames WAL -// files. A torn tail or an in-progress recovery marker returns ErrCorrupt so -// callers can fail and retry after the writer moves on. +// files. A missing directory is an empty view, and an incomplete final record +// is excluded. Malformed records and recovery markers return ErrCorrupt. func OpenReadOnlyChangelogWAL(dir string) (ChangelogWAL, error) { readOnly, err := openReadOnlyWAL( dir, diff --git a/sei-db/wal/readonly.go b/sei-db/wal/readonly.go index 8cc2d3ed01..acbd253f97 100644 --- a/sei-db/wal/readonly.go +++ b/sei-db/wal/readonly.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path/filepath" "sort" @@ -54,6 +55,9 @@ type readOnlyWAL[T any] struct { func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[T], error) { segments, err := listReadOnlySegments(dir) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return &readOnlyWAL[T]{unmarshal: unmarshal}, nil + } return nil, err } @@ -98,7 +102,7 @@ func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[ ErrCorrupt, path)) } - entries, err := indexReadOnlySegment(file, data) + entries, err := indexReadOnlySegment(file, data, i == len(segments)-1) if err != nil { return cleanup(fmt.Errorf("index WAL segment %s: %w", path, err)) } @@ -145,11 +149,14 @@ func listReadOnlySegments(dir string) ([]readOnlySegment, error) { return segments, nil } -func indexReadOnlySegment(file *os.File, data []byte) ([]readOnlyEntry, error) { +func indexReadOnlySegment(file *os.File, data []byte, tail bool) ([]readOnlyEntry, error) { entries := make([]readOnlyEntry, 0) for pos := 0; pos < len(data); { recordLen, err := loadNextBinaryEntry(data[pos:]) if err != nil { + if tail && isIncompleteBinaryEntry(data[pos:]) { + break + } return nil, err } size, prefixLen := binary.Uvarint(data[pos:]) @@ -164,6 +171,18 @@ func indexReadOnlySegment(file *os.File, data []byte) ([]readOnlyEntry, error) { return entries, nil } +func isIncompleteBinaryEntry(data []byte) bool { + size, prefixLen := binary.Uvarint(data) + if prefixLen == 0 { + return true + } + if prefixLen < 0 || size > math.MaxInt32 { + return false + } + entrySize := int(size) //nolint:gosec // size is at most math.MaxInt32. + return entrySize > len(data)-prefixLen +} + func (log *readOnlyWAL[T]) Write(T) error { return ErrReadOnly } @@ -212,11 +231,11 @@ func (log *readOnlyWAL[T]) ReadAt(index uint64) (T, error) { entry := log.entries[index-log.firstOffset] data := make([]byte, entry.size) if _, err := entry.file.ReadAt(data, entry.dataOffset); err != nil { - return zero, fmt.Errorf("read WAL offset %d: %w", index, err) + return zero, fmt.Errorf("%w: read WAL offset %d: %w", ErrCorrupt, index, err) } value, err := log.unmarshal(data) if err != nil { - return zero, fmt.Errorf("unmarshal WAL offset %d: %w", index, err) + return zero, fmt.Errorf("%w: unmarshal WAL offset %d: %w", ErrCorrupt, index, err) } return value, nil } diff --git a/sei-db/wal/readonly_test.go b/sei-db/wal/readonly_test.go index ebace3693a..67a2180285 100644 --- a/sei-db/wal/readonly_test.go +++ b/sei-db/wal/readonly_test.go @@ -44,7 +44,7 @@ func TestOpenReadOnlyChangelogWALReplaysWithoutMutation(t *testing.T) { require.NoError(t, readOnly.Close()) } -func TestOpenReadOnlyChangelogWALRejectsTornTailWithoutRepair(t *testing.T) { +func TestOpenReadOnlyChangelogWALUsesCompleteTailPrefixWithoutRepair(t *testing.T) { dir := t.TempDir() writable, err := NewChangelogWAL(dir, Config{}) require.NoError(t, err) @@ -59,11 +59,28 @@ func TestOpenReadOnlyChangelogWALRejectsTornTailWithoutRepair(t *testing.T) { require.NoError(t, file.Close()) before := snapshotWALFiles(t, dir) - _, err = OpenReadOnlyChangelogWAL(dir) - require.ErrorIs(t, err, ErrCorrupt) + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Equal(t, uint64(3), last) + require.NoError(t, readOnly.Close()) require.Equal(t, before, snapshotWALFiles(t, dir), "read-only open must not repair the source tail") } +func TestOpenReadOnlyChangelogWALRejectsMalformedTail(t *testing.T) { + dir := t.TempDir() + segment := filepath.Join(dir, "00000000000000000001") + require.NoError(t, os.WriteFile(segment, []byte{ + 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, + }, 0o600)) + + _, err := OpenReadOnlyChangelogWAL(dir) + require.ErrorIs(t, err, ErrCorrupt) +} + func TestOpenReadOnlyChangelogWALKeepsPointInTimeView(t *testing.T) { dir := t.TempDir() writable, err := NewChangelogWAL(dir, Config{}) @@ -121,8 +138,15 @@ func TestOpenReadOnlyChangelogWALEmptyDirectory(t *testing.T) { func TestOpenReadOnlyChangelogWALDoesNotCreateMissingDirectory(t *testing.T) { dir := filepath.Join(t.TempDir(), "missing") - _, err := OpenReadOnlyChangelogWAL(dir) - require.Error(t, err) + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + first, err := readOnly.FirstOffset() + require.NoError(t, err) + require.Zero(t, first) + last, err := readOnly.LastOffset() + require.NoError(t, err) + require.Zero(t, last) + require.NoError(t, readOnly.Close()) require.NoDirExists(t, dir) } @@ -136,6 +160,37 @@ func TestOpenReadOnlyChangelogWALClassifiesVanishedSegmentAsCorrupt(t *testing.T require.ErrorIs(t, err, os.ErrNotExist) } +func TestReadOnlyChangelogWALClassifiesEntryReadFailuresAsCorrupt(t *testing.T) { + t.Run("short read", func(t *testing.T) { + dir := t.TempDir() + writable, err := NewChangelogWAL(dir, Config{}) + require.NoError(t, err) + require.NoError(t, writable.Write(proto.ChangelogEntry{Version: 1})) + require.NoError(t, writable.Close()) + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, readOnly.Close()) }) + require.NoError(t, os.Truncate(lastPlainWALSegment(t, dir), 0)) + + _, err = readOnly.ReadAt(1) + require.ErrorIs(t, err, ErrCorrupt) + }) + + t.Run("unparseable entry", func(t *testing.T) { + dir := t.TempDir() + segment := filepath.Join(dir, "00000000000000000001") + require.NoError(t, os.WriteFile(segment, []byte{0x01, 0xff}, 0o600)) + + readOnly, err := OpenReadOnlyChangelogWAL(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, readOnly.Close()) }) + + _, err = readOnly.ReadAt(1) + require.ErrorIs(t, err, ErrCorrupt) + }) +} + func TestOpenReadOnlyChangelogWALConcurrentWriter(t *testing.T) { dir := t.TempDir() writable, err := NewChangelogWAL(dir, Config{}) From 9afce99de0ff0a38d854340bb4fa06ca2a571883 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 17:04:17 -0400 Subject: [PATCH 04/13] refactor(seidb): scope fail-loud WAL reads to digest Keep existing memiavl read-only callers on their prior WAL path and require digest replay to opt into immutable, fail-loud access. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 49 +++---- sei-db/state_db/sc/memiavl/db_test.go | 122 ++---------------- sei-db/state_db/sc/memiavl/opts.go | 7 + .../seidb/operations/evm_logical_digest.go | 7 +- sei-db/wal/changelog.go | 4 +- sei-db/wal/readonly.go | 23 +--- sei-db/wal/readonly_test.go | 21 +-- 7 files changed, 53 insertions(+), 180 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 302c5eb8b7..1bf828e3f2 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -30,16 +30,13 @@ const LockFileName = "LOCK" var ( errReadOnly = errors.New("db is read-only") - // ErrReadOnlyWALCorrupt means a read-only open observed an incomplete, + // ErrReadOnlyWALCorrupt means a FailOnWALRepair open observed an incomplete, // corrupt, or concurrently recovering changelog. The source WAL is left - // untouched; callers can retry after the writer finishes its current WAL - // operation. + // untouched. ErrReadOnlyWALCorrupt = errors.New("read-only changelog is incomplete or corrupt") - // ErrReadOnlyWALUnavailable means the immutable WAL view cannot replay - // every version from the selected snapshot through the requested target. - // The live writer may have pruned or advanced the changelog while the - // reader opened it; callers can retry against a new point-in-time view. + // ErrReadOnlyWALUnavailable means a FailOnWALRepair open cannot replay every + // version from the selected snapshot through the requested target. ErrReadOnlyWALUnavailable = errors.New("read-only changelog cannot reach the requested version") ) @@ -178,7 +175,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { streamHandler wal.ChangelogWAL ) defer func() { - if _err == nil { + if _err == nil || !opts.FailOnWALRepair { return } if streamHandler != nil { @@ -187,10 +184,6 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { if mtree != nil { _ = mtree.Close() } - if fileLock != nil { - _ = fileLock.Unlock() - _ = fileLock.Destroy() - } }() if err := opts.Validate(); err != nil { return nil, fmt.Errorf("invalid commit store options: %w", err) @@ -232,17 +225,15 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { // Snapshot mmap files are loaded with MADV_RANDOM in OpenSnapshot(). - // MemIAVL owns changelog lifecycle: always open the WAL here. Read-only - // callers still need replay to reconstruct non-snapshot versions, but they - // must not use the writable opener: it repairs a torn tail by truncating it - // and completes interrupted WAL truncations by renaming or removing files. - classifyReadOnlyWALError := func(err error) error { - if opts.ReadOnly && errors.Is(err, wal.ErrCorrupt) { + // MemIAVL owns changelog lifecycle: always open the WAL here. + // Digest tooling can explicitly disable repair when it reads a live WAL. + classifyFailOnWALRepairError := func(err error) error { + if opts.FailOnWALRepair && errors.Is(err, wal.ErrCorrupt) { return fmt.Errorf("%w; source WAL was not modified: %w", ErrReadOnlyWALCorrupt, err) } return err } - if opts.ReadOnly { + if opts.FailOnWALRepair { streamHandler, err = wal.OpenReadOnlyChangelogWAL(utils.GetChangelogPath(opts.Dir)) } else { streamHandler, err = wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ @@ -250,7 +241,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { }) } if err != nil { - err = classifyReadOnlyWALError(err) + err = classifyFailOnWALRepairError(err) if errors.Is(err, ErrReadOnlyWALCorrupt) { return nil, err } @@ -262,14 +253,16 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { var walHasEntries bool walIndexDelta, walHasEntries, err = computeWALIndexDelta(streamHandler) if err != nil { - err = classifyReadOnlyWALError(err) + if opts.FailOnWALRepair { + err = classifyFailOnWALRepairError(err) + } return nil, fmt.Errorf("failed to compute WAL index delta: %w", err) } // If WAL is empty, set delta so first WAL entry aligns with NextVersion(). if !walHasEntries { walIndexDelta = mtree.WorkingCommitInfo().Version - 1 } - if opts.ReadOnly && walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { + if opts.FailOnWALRepair && walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { firstIndex, firstErr := streamHandler.FirstOffset() if firstErr != nil { return nil, fmt.Errorf("read changelog first offset: %w", firstErr) @@ -291,11 +284,14 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { if walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { logger.Info("Start catching up and replaying the MemIAVL changelog file") if err := mtree.Catchup(context.Background(), streamHandler, walIndexDelta, targetVersion); err != nil { - return nil, classifyReadOnlyWALError(err) + if opts.FailOnWALRepair { + err = classifyFailOnWALRepairError(err) + } + return nil, err } logger.Info("finished replay and caught up to target version", "version", targetVersion) } - if opts.ReadOnly && targetVersion > 0 && mtree.Version() != targetVersion { + if opts.FailOnWALRepair && targetVersion > 0 && mtree.Version() != targetVersion { reached := mtree.Version() return nil, fmt.Errorf("%w: requested %d, reached %d", ErrReadOnlyWALUnavailable, targetVersion, reached) @@ -366,10 +362,6 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { snapshotWriterPool: workerPool, opts: opts, } - // The DB owns these resources from this point forward. - mtree = nil - streamHandler = nil - fileLock = nil // Apply initial stores on a fresh DB (version 0) so they get persisted to WAL. // This creates the trees and populates pendingLogEntry, which will be written @@ -381,7 +373,6 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { upgrades = append(upgrades, &proto.TreeNameUpgrade{Name: name}) } if err := db.ApplyUpgrades(upgrades); err != nil { - _ = db.Close() return nil, fmt.Errorf("failed to apply initial stores: %w", err) } } diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index 752505dafb..2019ff4b81 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -2,7 +2,6 @@ package memiavl import ( "context" - "encoding/binary" "encoding/hex" "os" "path/filepath" @@ -759,6 +758,9 @@ func TestInvalidOptions(t *testing.T) { _, err = OpenDB(0, Options{Dir: dir, ReadOnly: true, CreateIfMissing: true}) require.Error(t, err) + _, err = OpenDB(0, Options{Dir: dir, FailOnWALRepair: true}) + require.ErrorContains(t, err, "can't disable WAL repair in writable mode") + db, err := OpenDB(0, Options{Dir: dir, CreateIfMissing: true}) require.NoError(t, err) require.NoError(t, db.Close()) @@ -1119,7 +1121,7 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) { require.Equal(t, "snapshot-1", target) } -func TestReadOnlyOpenUsesCompleteWALPrefixWithoutRepair(t *testing.T) { +func TestFailOnWALRepairRejectsTornWALWithoutRepair(t *testing.T) { dir := t.TempDir() db, err := OpenDB(0, Options{ Dir: dir, @@ -1146,24 +1148,19 @@ func TestReadOnlyOpenUsesCompleteWALPrefixWithoutRepair(t *testing.T) { before, err := os.ReadFile(filepath.Clean(segment)) require.NoError(t, err) - readOnly, err := OpenDB(0, Options{Dir: dir, ReadOnly: true}) - require.NoError(t, err) - require.Equal(t, int64(3), readOnly.Version()) - require.NoError(t, readOnly.Close()) + _, err = OpenDB(0, Options{Dir: dir, ReadOnly: true, FailOnWALRepair: true}) + require.ErrorIs(t, err, ErrReadOnlyWALCorrupt) after, readErr := os.ReadFile(filepath.Clean(segment)) require.NoError(t, readErr) - require.Equal(t, before, after, "read-only open must leave a torn live tail untouched") - - _, err = OpenDB(4, Options{Dir: dir, ReadOnly: true}) - require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) + require.Equal(t, before, after, "fail-loud open must leave a torn live tail untouched") - repaired, err := OpenDB(0, Options{Dir: dir}) - require.NoError(t, err, "the writable owner must retain the existing tail-repair behavior") + repaired, err := OpenDB(0, Options{Dir: dir, ReadOnly: true}) + require.NoError(t, err, "ordinary read-only callers must retain the existing WAL behavior") require.Equal(t, int64(3), repaired.Version()) require.NoError(t, repaired.Close()) } -func TestReadOnlyOpenRejectsWALGap(t *testing.T) { +func TestFailOnWALRepairRejectsWALGap(t *testing.T) { dir := t.TempDir() db, err := OpenDB(0, Options{ Dir: dir, @@ -1182,13 +1179,13 @@ func TestReadOnlyOpenRejectsWALGap(t *testing.T) { require.NoError(t, db.GetWAL().TruncateBefore(2)) require.NoError(t, db.Close()) - _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true}) + _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true, FailOnWALRepair: true}) require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) require.Contains(t, err.Error(), "needs changelog version 1") require.Contains(t, err.Error(), "starts at version 2") } -func TestReadOnlyOpenRejectsShortWAL(t *testing.T) { +func TestFailOnWALRepairRejectsShortWAL(t *testing.T) { dir := t.TempDir() db, err := OpenDB(0, Options{ Dir: dir, @@ -1207,104 +1204,11 @@ func TestReadOnlyOpenRejectsShortWAL(t *testing.T) { require.NoError(t, db.GetWAL().TruncateAfter(2)) require.NoError(t, db.Close()) - _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true}) + _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true, FailOnWALRepair: true}) require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) require.Contains(t, err.Error(), "requested 3, reached 2") } -func TestReadOnlyOpenClassifiesReplayDecodeFailure(t *testing.T) { - dir := t.TempDir() - db, err := OpenDB(0, Options{ - Dir: dir, - CreateIfMissing: true, - InitialStores: []string{"test"}, - }) - require.NoError(t, err) - for i := 0; i < 2; i++ { - require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: "test", - Changeset: ChangeSets[i], - }})) - _, err = db.Commit() - require.NoError(t, err) - } - require.NoError(t, db.Close()) - - segment := lastMemiAVLWALSegment(t, dir) - data, err := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, err) - firstSize, firstPrefixLen := binary.Uvarint(data) - require.Positive(t, firstPrefixLen) - require.LessOrEqual(t, firstSize, uint64(len(data)-firstPrefixLen)) - secondPos := firstPrefixLen + int(firstSize) //nolint:gosec // bounded by the segment length above. - secondSize, secondPrefixLen := binary.Uvarint(data[secondPos:]) - require.Positive(t, secondPrefixLen) - require.LessOrEqual(t, secondSize, uint64(len(data)-secondPos-secondPrefixLen)) - secondDataStart := secondPos + secondPrefixLen - secondDataEnd := secondDataStart + int(secondSize) //nolint:gosec // bounded by the segment length above. - for i := secondDataStart; i < secondDataEnd; i++ { - data[i] = 0xff - } - require.NoError(t, os.WriteFile(filepath.Clean(segment), data, 0o600)) - - _, err = OpenDB(2, Options{Dir: dir, ReadOnly: true}) - require.ErrorIs(t, err, ErrReadOnlyWALCorrupt) - after, readErr := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, readErr) - require.Equal(t, data, after, "read-only replay must not repair an unparseable entry") -} - -func TestReadOnlyOpenTreatsMissingWALAsEmpty(t *testing.T) { - dir := t.TempDir() - db, err := OpenDB(0, Options{ - Dir: dir, - CreateIfMissing: true, - InitialStores: []string{"test"}, - }) - require.NoError(t, err) - require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: "test", - Changeset: ChangeSets[0], - }})) - _, err = db.Commit() - require.NoError(t, err) - require.NoError(t, db.RewriteSnapshot(context.Background())) - require.NoError(t, db.Close()) - require.NoError(t, os.RemoveAll(utils.GetChangelogPath(dir))) - - readOnly, err := OpenDB(1, Options{Dir: dir, ReadOnly: true}) - require.NoError(t, err) - require.Equal(t, int64(1), readOnly.Version()) - require.NoError(t, readOnly.Close()) - require.NoDirExists(t, utils.GetChangelogPath(dir)) -} - -func TestOpenDBFailureReleasesFileLock(t *testing.T) { - dir := t.TempDir() - db, err := OpenDB(0, Options{ - Dir: dir, - CreateIfMissing: true, - InitialStores: []string{"test"}, - }) - require.NoError(t, err) - require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: "test", - Changeset: ChangeSets[0], - }})) - _, err = db.Commit() - require.NoError(t, err) - require.NoError(t, db.Close()) - - require.NoError(t, os.Remove(currentPath(dir))) - _, err = OpenDB(1, Options{Dir: dir, LoadForOverwriting: true}) - require.ErrorContains(t, err, "fail to read current version") - - lock, err := LockFile(filepath.Join(dir, LockFileName)) - require.NoError(t, err, "failed OpenDB must release its exclusive lock") - require.NoError(t, lock.Unlock()) - require.NoError(t, lock.Destroy()) -} - func lastMemiAVLWALSegment(t *testing.T, dir string) string { t.Helper() entries, err := os.ReadDir(utils.GetChangelogPath(dir)) diff --git a/sei-db/state_db/sc/memiavl/opts.go b/sei-db/state_db/sc/memiavl/opts.go index 9c4a6f5d31..2d339283f3 100644 --- a/sei-db/state_db/sc/memiavl/opts.go +++ b/sei-db/state_db/sc/memiavl/opts.go @@ -18,6 +18,9 @@ type Options struct { InitialVersion uint32 // ReadOnly opens the database in read-only mode ReadOnly bool + // FailOnWALRepair opens an immutable changelog view and returns an error + // instead of repairing the WAL. It requires ReadOnly. + FailOnWALRepair bool // InitialStores are the initial store names when initializing an empty instance InitialStores []string // ZeroCopy if true, get and iterator methods return slices pointing to mmaped blob files @@ -47,6 +50,10 @@ func (opts Options) Validate() error { return errors.New("can't rollback db in read-only mode") } + if opts.FailOnWALRepair && !opts.ReadOnly { + return errors.New("can't disable WAL repair in writable mode") + } + return nil } diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 1dbf7bfec8..1093baef99 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -1108,9 +1108,10 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: dbDir, - ReadOnly: true, - ZeroCopy: true, + Dir: dbDir, + ReadOnly: true, + FailOnWALRepair: true, + ZeroCopy: true, }) if err != nil { if errors.Is(err, memiavl.ErrReadOnlyWALCorrupt) { diff --git a/sei-db/wal/changelog.go b/sei-db/wal/changelog.go index 2479c2ea6e..5877a29443 100644 --- a/sei-db/wal/changelog.go +++ b/sei-db/wal/changelog.go @@ -28,8 +28,8 @@ func NewChangelogWAL(dir string, config Config) (ChangelogWAL, error) { // OpenReadOnlyChangelogWAL opens an immutable point-in-time view of the // changelog segment files. It never creates, truncates, removes, or renames WAL -// files. A missing directory is an empty view, and an incomplete final record -// is excluded. Malformed records and recovery markers return ErrCorrupt. +// files. An incomplete tail, malformed record, or recovery marker returns +// ErrCorrupt. func OpenReadOnlyChangelogWAL(dir string) (ChangelogWAL, error) { readOnly, err := openReadOnlyWAL( dir, diff --git a/sei-db/wal/readonly.go b/sei-db/wal/readonly.go index acbd253f97..92e9374c0e 100644 --- a/sei-db/wal/readonly.go +++ b/sei-db/wal/readonly.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "math" "os" "path/filepath" "sort" @@ -55,9 +54,6 @@ type readOnlyWAL[T any] struct { func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[T], error) { segments, err := listReadOnlySegments(dir) if err != nil { - if errors.Is(err, os.ErrNotExist) { - return &readOnlyWAL[T]{unmarshal: unmarshal}, nil - } return nil, err } @@ -102,7 +98,7 @@ func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[ ErrCorrupt, path)) } - entries, err := indexReadOnlySegment(file, data, i == len(segments)-1) + entries, err := indexReadOnlySegment(file, data) if err != nil { return cleanup(fmt.Errorf("index WAL segment %s: %w", path, err)) } @@ -149,14 +145,11 @@ func listReadOnlySegments(dir string) ([]readOnlySegment, error) { return segments, nil } -func indexReadOnlySegment(file *os.File, data []byte, tail bool) ([]readOnlyEntry, error) { +func indexReadOnlySegment(file *os.File, data []byte) ([]readOnlyEntry, error) { entries := make([]readOnlyEntry, 0) for pos := 0; pos < len(data); { recordLen, err := loadNextBinaryEntry(data[pos:]) if err != nil { - if tail && isIncompleteBinaryEntry(data[pos:]) { - break - } return nil, err } size, prefixLen := binary.Uvarint(data[pos:]) @@ -171,18 +164,6 @@ func indexReadOnlySegment(file *os.File, data []byte, tail bool) ([]readOnlyEntr return entries, nil } -func isIncompleteBinaryEntry(data []byte) bool { - size, prefixLen := binary.Uvarint(data) - if prefixLen == 0 { - return true - } - if prefixLen < 0 || size > math.MaxInt32 { - return false - } - entrySize := int(size) //nolint:gosec // size is at most math.MaxInt32. - return entrySize > len(data)-prefixLen -} - func (log *readOnlyWAL[T]) Write(T) error { return ErrReadOnly } diff --git a/sei-db/wal/readonly_test.go b/sei-db/wal/readonly_test.go index 67a2180285..a3d1c56034 100644 --- a/sei-db/wal/readonly_test.go +++ b/sei-db/wal/readonly_test.go @@ -44,7 +44,7 @@ func TestOpenReadOnlyChangelogWALReplaysWithoutMutation(t *testing.T) { require.NoError(t, readOnly.Close()) } -func TestOpenReadOnlyChangelogWALUsesCompleteTailPrefixWithoutRepair(t *testing.T) { +func TestOpenReadOnlyChangelogWALRejectsTornTailWithoutRepair(t *testing.T) { dir := t.TempDir() writable, err := NewChangelogWAL(dir, Config{}) require.NoError(t, err) @@ -59,12 +59,8 @@ func TestOpenReadOnlyChangelogWALUsesCompleteTailPrefixWithoutRepair(t *testing. require.NoError(t, file.Close()) before := snapshotWALFiles(t, dir) - readOnly, err := OpenReadOnlyChangelogWAL(dir) - require.NoError(t, err) - last, err := readOnly.LastOffset() - require.NoError(t, err) - require.Equal(t, uint64(3), last) - require.NoError(t, readOnly.Close()) + _, err = OpenReadOnlyChangelogWAL(dir) + require.ErrorIs(t, err, ErrCorrupt) require.Equal(t, before, snapshotWALFiles(t, dir), "read-only open must not repair the source tail") } @@ -138,15 +134,8 @@ func TestOpenReadOnlyChangelogWALEmptyDirectory(t *testing.T) { func TestOpenReadOnlyChangelogWALDoesNotCreateMissingDirectory(t *testing.T) { dir := filepath.Join(t.TempDir(), "missing") - readOnly, err := OpenReadOnlyChangelogWAL(dir) - require.NoError(t, err) - first, err := readOnly.FirstOffset() - require.NoError(t, err) - require.Zero(t, first) - last, err := readOnly.LastOffset() - require.NoError(t, err) - require.Zero(t, last) - require.NoError(t, readOnly.Close()) + _, err := OpenReadOnlyChangelogWAL(dir) + require.Error(t, err) require.NoDirExists(t, dir) } From dcdfa4121a00f0959bdb876843b90ea5f632cc7d Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 17:14:23 -0400 Subject: [PATCH 05/13] refactor(seidb): check the changelog instead of reading it read-only The digest replay mode refused a torn changelog through a purpose-built read-only WAL reader and an opt-in memIAVL mode. A check before the open reaches the same outcome without either: the opener repairs only the tail segment, so reading that one segment answers whether the open would repair anything. This leaves memIAVL and the changelog opener untouched. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 82 +---- sei-db/state_db/sc/memiavl/db_test.go | 107 ------- sei-db/state_db/sc/memiavl/opts.go | 7 - .../seidb/operations/evm_logical_digest.go | 52 ++-- .../cmd/seidb/operations/memiavl_open_test.go | 8 +- sei-db/wal/changelog.go | 19 -- sei-db/wal/readonly.go | 252 --------------- sei-db/wal/readonly_test.go | 294 ------------------ sei-db/wal/verify.go | 66 ++++ sei-db/wal/verify_test.go | 122 ++++++++ 10 files changed, 232 insertions(+), 777 deletions(-) delete mode 100644 sei-db/wal/readonly.go delete mode 100644 sei-db/wal/readonly_test.go create mode 100644 sei-db/wal/verify.go create mode 100644 sei-db/wal/verify_test.go diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 1bf828e3f2..ddd18f675c 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -27,18 +27,7 @@ import ( const LockFileName = "LOCK" -var ( - errReadOnly = errors.New("db is read-only") - - // ErrReadOnlyWALCorrupt means a FailOnWALRepair open observed an incomplete, - // corrupt, or concurrently recovering changelog. The source WAL is left - // untouched. - ErrReadOnlyWALCorrupt = errors.New("read-only changelog is incomplete or corrupt") - - // ErrReadOnlyWALUnavailable means a FailOnWALRepair open cannot replay every - // version from the selected snapshot through the requested target. - ErrReadOnlyWALUnavailable = errors.New("read-only changelog cannot reach the requested version") -) +var errReadOnly = errors.New("db is read-only") // DB implements DB-like functionalities on top of MultiTree: // - async snapshot rewriting @@ -169,22 +158,9 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { ) }() var ( - err error - fileLock FileLock - mtree *MultiTree - streamHandler wal.ChangelogWAL + err error + fileLock FileLock ) - defer func() { - if _err == nil || !opts.FailOnWALRepair { - return - } - if streamHandler != nil { - _ = streamHandler.Close() - } - if mtree != nil { - _ = mtree.Close() - } - }() if err := opts.Validate(); err != nil { return nil, fmt.Errorf("invalid commit store options: %w", err) } @@ -218,7 +194,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { } path := filepath.Join(opts.Dir, snapshot) - mtree, err = LoadMultiTree(context.Background(), path, opts) + mtree, err := LoadMultiTree(context.Background(), path, opts) if err != nil { return nil, err } @@ -226,25 +202,11 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { // Snapshot mmap files are loaded with MADV_RANDOM in OpenSnapshot(). // MemIAVL owns changelog lifecycle: always open the WAL here. - // Digest tooling can explicitly disable repair when it reads a live WAL. - classifyFailOnWALRepairError := func(err error) error { - if opts.FailOnWALRepair && errors.Is(err, wal.ErrCorrupt) { - return fmt.Errorf("%w; source WAL was not modified: %w", ErrReadOnlyWALCorrupt, err) - } - return err - } - if opts.FailOnWALRepair { - streamHandler, err = wal.OpenReadOnlyChangelogWAL(utils.GetChangelogPath(opts.Dir)) - } else { - streamHandler, err = wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ - WriteBufferSize: opts.AsyncCommitBuffer, - }) - } + // Even in read-only mode we may need WAL replay to reconstruct non-snapshot versions. + streamHandler, err := wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ + WriteBufferSize: opts.AsyncCommitBuffer, + }) if err != nil { - err = classifyFailOnWALRepairError(err) - if errors.Is(err, ErrReadOnlyWALCorrupt) { - return nil, err - } return nil, fmt.Errorf("failed to open changelog WAL: %w", err) } @@ -253,49 +215,21 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { var walHasEntries bool walIndexDelta, walHasEntries, err = computeWALIndexDelta(streamHandler) if err != nil { - if opts.FailOnWALRepair { - err = classifyFailOnWALRepairError(err) - } return nil, fmt.Errorf("failed to compute WAL index delta: %w", err) } // If WAL is empty, set delta so first WAL entry aligns with NextVersion(). if !walHasEntries { walIndexDelta = mtree.WorkingCommitInfo().Version - 1 } - if opts.FailOnWALRepair && walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { - firstIndex, firstErr := streamHandler.FirstOffset() - if firstErr != nil { - return nil, fmt.Errorf("read changelog first offset: %w", firstErr) - } - if firstIndex > math.MaxInt64 { - return nil, fmt.Errorf("%w: first WAL offset %d overflows int64", ErrReadOnlyWALUnavailable, firstIndex) - } - firstVersion := int64(firstIndex) + walIndexDelta - firstNeeded := utils.NextVersion(mtree.Version(), mtree.initialVersion.Load()) - if firstVersion > firstNeeded { - snapshotVersion := mtree.Version() - return nil, fmt.Errorf("%w: selected snapshot version %d needs changelog version %d, "+ - "but the immutable WAL view starts at version %d", - ErrReadOnlyWALUnavailable, snapshotVersion, firstNeeded, firstVersion) - } - } // Replay WAL to catch up to target version (if WAL has entries) if walHasEntries && (targetVersion == 0 || targetVersion > mtree.Version()) { logger.Info("Start catching up and replaying the MemIAVL changelog file") if err := mtree.Catchup(context.Background(), streamHandler, walIndexDelta, targetVersion); err != nil { - if opts.FailOnWALRepair { - err = classifyFailOnWALRepairError(err) - } return nil, err } logger.Info("finished replay and caught up to target version", "version", targetVersion) } - if opts.FailOnWALRepair && targetVersion > 0 && mtree.Version() != targetVersion { - reached := mtree.Version() - return nil, fmt.Errorf("%w: requested %d, reached %d", - ErrReadOnlyWALUnavailable, targetVersion, reached) - } if opts.LoadForOverwriting && targetVersion > 0 { currentSnapshot, err := os.Readlink(currentPath(opts.Dir)) diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index 2019ff4b81..3cddd00f77 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "runtime/debug" - "sort" "strconv" "sync" "testing" @@ -758,9 +757,6 @@ func TestInvalidOptions(t *testing.T) { _, err = OpenDB(0, Options{Dir: dir, ReadOnly: true, CreateIfMissing: true}) require.Error(t, err) - _, err = OpenDB(0, Options{Dir: dir, FailOnWALRepair: true}) - require.ErrorContains(t, err, "can't disable WAL repair in writable mode") - db, err := OpenDB(0, Options{Dir: dir, CreateIfMissing: true}) require.NoError(t, err) require.NoError(t, db.Close()) @@ -1120,106 +1116,3 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) { require.NoError(t, err) require.Equal(t, "snapshot-1", target) } - -func TestFailOnWALRepairRejectsTornWALWithoutRepair(t *testing.T) { - dir := t.TempDir() - db, err := OpenDB(0, Options{ - Dir: dir, - CreateIfMissing: true, - InitialStores: []string{"test"}, - }) - require.NoError(t, err) - for i := 0; i < 3; i++ { - require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: "test", - Changeset: ChangeSets[i], - }})) - _, err := db.Commit() - require.NoError(t, err) - } - require.NoError(t, db.Close()) - - segment := lastMemiAVLWALSegment(t, dir) - file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) - require.NoError(t, err) - _, err = file.Write([]byte{0x10}) - require.NoError(t, err) - require.NoError(t, file.Close()) - before, err := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, err) - - _, err = OpenDB(0, Options{Dir: dir, ReadOnly: true, FailOnWALRepair: true}) - require.ErrorIs(t, err, ErrReadOnlyWALCorrupt) - after, readErr := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, readErr) - require.Equal(t, before, after, "fail-loud open must leave a torn live tail untouched") - - repaired, err := OpenDB(0, Options{Dir: dir, ReadOnly: true}) - require.NoError(t, err, "ordinary read-only callers must retain the existing WAL behavior") - require.Equal(t, int64(3), repaired.Version()) - require.NoError(t, repaired.Close()) -} - -func TestFailOnWALRepairRejectsWALGap(t *testing.T) { - dir := t.TempDir() - db, err := OpenDB(0, Options{ - Dir: dir, - CreateIfMissing: true, - InitialStores: []string{"test"}, - }) - require.NoError(t, err) - for i := 0; i < 3; i++ { - require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: "test", - Changeset: ChangeSets[i], - }})) - _, err := db.Commit() - require.NoError(t, err) - } - require.NoError(t, db.GetWAL().TruncateBefore(2)) - require.NoError(t, db.Close()) - - _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true, FailOnWALRepair: true}) - require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) - require.Contains(t, err.Error(), "needs changelog version 1") - require.Contains(t, err.Error(), "starts at version 2") -} - -func TestFailOnWALRepairRejectsShortWAL(t *testing.T) { - dir := t.TempDir() - db, err := OpenDB(0, Options{ - Dir: dir, - CreateIfMissing: true, - InitialStores: []string{"test"}, - }) - require.NoError(t, err) - for i := 0; i < 3; i++ { - require.NoError(t, db.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: "test", - Changeset: ChangeSets[i], - }})) - _, err := db.Commit() - require.NoError(t, err) - } - require.NoError(t, db.GetWAL().TruncateAfter(2)) - require.NoError(t, db.Close()) - - _, err = OpenDB(3, Options{Dir: dir, ReadOnly: true, FailOnWALRepair: true}) - require.ErrorIs(t, err, ErrReadOnlyWALUnavailable) - require.Contains(t, err.Error(), "requested 3, reached 2") -} - -func lastMemiAVLWALSegment(t *testing.T, dir string) string { - t.Helper() - entries, err := os.ReadDir(utils.GetChangelogPath(dir)) - require.NoError(t, err) - var names []string - for _, entry := range entries { - if !entry.IsDir() && len(entry.Name()) == 20 { - names = append(names, entry.Name()) - } - } - require.NotEmpty(t, names) - sort.Strings(names) - return filepath.Join(utils.GetChangelogPath(dir), names[len(names)-1]) -} diff --git a/sei-db/state_db/sc/memiavl/opts.go b/sei-db/state_db/sc/memiavl/opts.go index 2d339283f3..9c4a6f5d31 100644 --- a/sei-db/state_db/sc/memiavl/opts.go +++ b/sei-db/state_db/sc/memiavl/opts.go @@ -18,9 +18,6 @@ type Options struct { InitialVersion uint32 // ReadOnly opens the database in read-only mode ReadOnly bool - // FailOnWALRepair opens an immutable changelog view and returns an error - // instead of repairing the WAL. It requires ReadOnly. - FailOnWALRepair bool // InitialStores are the initial store names when initializing an empty instance InitialStores []string // ZeroCopy if true, get and iterator methods return slices pointing to mmaped blob files @@ -50,10 +47,6 @@ func (opts Options) Validate() error { return errors.New("can't rollback db in read-only mode") } - if opts.FailOnWALRepair && !opts.ReadOnly { - return errors.New("can't disable WAL repair in writable mode") - } - return nil } diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 1093baef99..ba070ceab2 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -14,12 +14,14 @@ import ( "sort" "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/migration" + "github.com/sei-protocol/sei-chain/sei-db/wal" "github.com/spf13/cobra" ) @@ -81,14 +83,15 @@ const ( // that exact height (or --height 0 for the current symlink). This is the // preferred mode whenever the target height lines up with an existing // snapshot boundary. -// - replay (SLOW): opens a non-mutating read-only WAL view, replays the -// changelog up to --height, then walks the in-memory/mmap tree. Roughly an -// order of magnitude slower than snapshot (changelog replay + per-leaf tree -// walk instead of a sequential file read). If a live writer leaves a torn -// tail in view, replay fails and asks the operator to rerun instead of -// repairing the source WAL. Use it only when no snapshot exists at the -// target height — e.g. nodes whose snapshot rewrite lags the tip, so an -// arbitrary comparison height has no snapshot- on disk. +// - replay (SLOW): opens a read-only DB, replays the changelog up to +// --height, then walks the in-memory/mmap tree. Roughly an order of +// magnitude slower than snapshot (changelog replay + per-leaf tree walk +// instead of a sequential file read). It refuses to run on a changelog whose +// tail a live writer is still filling, and asks the operator to rerun, +// because opening such a changelog would truncate that tail. Use it only +// when no snapshot exists at the target height — e.g. nodes whose snapshot +// rewrite lags the tip, so an arbitrary comparison height has no +// snapshot- on disk. // // The flatkv side is always a pebble WAL-replay-to-height and is fast // regardless. So when comparing across nodes, pick a height that is an existing @@ -1107,24 +1110,31 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization } func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { + // memiavl.OpenDB repairs a torn changelog tail by truncating it, even under + // ReadOnly. On a live node that tail is usually a write in progress, so + // refuse the run instead of letting the open damage the source. + if err := wal.VerifyIntact(utils.GetChangelogPath(dbDir)); err != nil { + if errors.Is(err, wal.ErrCorrupt) { + return nil, fmt.Errorf("memiavl changelog tail is incomplete or changing; live WAL was not "+ + "modified; rerun the command, and if the error persists after stopping seid, repair the "+ + "WAL offline: %w", err) + } + return nil, fmt.Errorf("verify memiavl changelog: %w", err) + } db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: dbDir, - ReadOnly: true, - FailOnWALRepair: true, - ZeroCopy: true, + Dir: dbDir, + ReadOnly: true, + ZeroCopy: true, }) if err != nil { - if errors.Is(err, memiavl.ErrReadOnlyWALCorrupt) { - return nil, fmt.Errorf("memiavl changelog tail is incomplete, corrupt, or changing; "+ - "live WAL was not modified; rerun the command, and if the error persists after stopping seid, "+ - "repair the WAL offline: %w", err) - } - if errors.Is(err, memiavl.ErrReadOnlyWALUnavailable) { - return nil, fmt.Errorf("the immutable memiavl changelog view could not reach height %d; "+ - "live WAL was not modified; rerun the command: %w", height, err) - } return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } + if height > 0 && db.Version() != height { + reached := db.Version() + _ = db.Close() + return nil, fmt.Errorf("memiavl replay reached version %d, not the requested height %d; "+ + "the changelog does not cover that height", reached, height) + } return db, nil } diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index 75806b3ee9..93f27cf5e8 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -13,7 +13,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/proto" ) -func TestOpenMemiAVLReplayReadOnlyReportsRetryWithoutRepair(t *testing.T) { +func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing.T) { homeDir := t.TempDir() store := newTestMemiavlStore(t, homeDir) require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ @@ -26,9 +26,11 @@ func TestOpenMemiAVLReplayReadOnlyReportsRetryWithoutRepair(t *testing.T) { dbDir := utils.GetCosmosSCStorePath(homeDir) segment := lastOperationsMemiAVLWALSegment(t, dbDir) - require.NoError(t, os.WriteFile(filepath.Clean(segment), []byte{0x01, 0xff}, 0o600)) - before, err := os.ReadFile(filepath.Clean(segment)) + committed, err := os.ReadFile(filepath.Clean(segment)) require.NoError(t, err) + require.NotEmpty(t, committed) + before := committed[:len(committed)-1] + require.NoError(t, os.WriteFile(filepath.Clean(segment), before, 0o600)) _, err = openMemiAVLReplayReadOnly(dbDir, 0) require.Error(t, err) diff --git a/sei-db/wal/changelog.go b/sei-db/wal/changelog.go index 5877a29443..fadb054f31 100644 --- a/sei-db/wal/changelog.go +++ b/sei-db/wal/changelog.go @@ -26,25 +26,6 @@ func NewChangelogWAL(dir string, config Config) (ChangelogWAL, error) { ) } -// OpenReadOnlyChangelogWAL opens an immutable point-in-time view of the -// changelog segment files. It never creates, truncates, removes, or renames WAL -// files. An incomplete tail, malformed record, or recovery marker returns -// ErrCorrupt. -func OpenReadOnlyChangelogWAL(dir string) (ChangelogWAL, error) { - readOnly, err := openReadOnlyWAL( - dir, - func(data []byte) (proto.ChangelogEntry, error) { - var entry proto.ChangelogEntry - err := entry.Unmarshal(data) - return entry, err - }, - ) - if err != nil { - return nil, err - } - return readOnly, nil -} - // FindFirstOffsetAfterVersion returns the first WAL offset whose entry version is // strictly greater than targetVersion. If no such entry exists, it returns // lastOffset+1. Changelog versions are monotonic, but empty blocks can advance diff --git a/sei-db/wal/readonly.go b/sei-db/wal/readonly.go deleted file mode 100644 index 92e9374c0e..0000000000 --- a/sei-db/wal/readonly.go +++ /dev/null @@ -1,252 +0,0 @@ -package wal - -import ( - "encoding/binary" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - - tidwallwal "github.com/tidwall/wal" -) - -var ( - // ErrReadOnly is returned when a caller tries to mutate a read-only WAL. - ErrReadOnly = errors.New("WAL is read-only") - - // ErrCorrupt identifies a malformed or unstable WAL view. It aliases the - // underlying tidwall sentinel so callers do not need to import the storage - // implementation only to decide whether a read-only open can be retried. - ErrCorrupt = tidwallwal.ErrCorrupt -) - -type readOnlySegment struct { - name string - index uint64 -} - -type readOnlyEntry struct { - file *os.File - dataOffset int64 - size int -} - -// readOnlyWAL is an immutable view of the plain segment files present when it -// opens. It does not use tidwall/wal.Open because that function creates files, -// opens the tail for writing, and completes interrupted truncations by removing -// and renaming segment files. -type readOnlyWAL[T any] struct { - unmarshal UnmarshalFn[T] - files []*os.File - entries []readOnlyEntry - firstOffset uint64 - closed atomic.Bool - closeOnce sync.Once - closeErr error -} - -func openReadOnlyWAL[T any](dir string, unmarshal UnmarshalFn[T]) (*readOnlyWAL[T], error) { - segments, err := listReadOnlySegments(dir) - if err != nil { - return nil, err - } - - log := &readOnlyWAL[T]{unmarshal: unmarshal} - cleanup := func(err error) (*readOnlyWAL[T], error) { - _ = log.Close() - return nil, err - } - - var nextIndex uint64 - for i, segment := range segments { - if i > 0 && segment.index != nextIndex { - return cleanup(fmt.Errorf("%w: segment %s starts at index %d, expected %d", - ErrCorrupt, segment.name, segment.index, nextIndex)) - } - - path := filepath.Join(dir, segment.name) - file, err := os.Open(filepath.Clean(path)) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return cleanup(fmt.Errorf("%w: WAL segment %s disappeared while opening: %w", - ErrCorrupt, path, err)) - } - return cleanup(fmt.Errorf("open WAL segment %s: %w", path, err)) - } - log.files = append(log.files, file) - - info, err := file.Stat() - if err != nil { - return cleanup(fmt.Errorf("stat WAL segment %s: %w", path, err)) - } - if !info.Mode().IsRegular() { - return cleanup(fmt.Errorf("%w: WAL segment %s is not a regular file", ErrCorrupt, path)) - } - - data, err := io.ReadAll(io.NewSectionReader(file, 0, info.Size())) - if err != nil { - return cleanup(fmt.Errorf("read WAL segment %s: %w", path, err)) - } - if int64(len(data)) != info.Size() { - return cleanup(fmt.Errorf("%w: WAL segment %s changed while it was read", - ErrCorrupt, path)) - } - - entries, err := indexReadOnlySegment(file, data) - if err != nil { - return cleanup(fmt.Errorf("index WAL segment %s: %w", path, err)) - } - if len(entries) == 0 && i != len(segments)-1 { - return cleanup(fmt.Errorf("%w: non-tail WAL segment %s is empty", ErrCorrupt, path)) - } - if len(log.entries) == 0 && len(entries) > 0 { - log.firstOffset = segment.index - } - log.entries = append(log.entries, entries...) - nextIndex = segment.index + uint64(len(entries)) - } - return log, nil -} - -func listReadOnlySegments(dir string) ([]readOnlySegment, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return nil, fmt.Errorf("read WAL directory %s: %w", dir, err) - } - - segments := make([]readOnlySegment, 0, len(entries)) - for _, entry := range entries { - if entry.IsDir() { - continue - } - name := entry.Name() - if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { - return nil, fmt.Errorf("%w: WAL recovery marker %s is present; retry after the writer finishes", - ErrCorrupt, name) - } - if len(name) != 20 { - continue - } - index, err := strconv.ParseUint(name, 10, 64) - if err != nil || index == 0 { - continue - } - segments = append(segments, readOnlySegment{name: name, index: index}) - } - sort.Slice(segments, func(i, j int) bool { - return segments[i].index < segments[j].index - }) - return segments, nil -} - -func indexReadOnlySegment(file *os.File, data []byte) ([]readOnlyEntry, error) { - entries := make([]readOnlyEntry, 0) - for pos := 0; pos < len(data); { - recordLen, err := loadNextBinaryEntry(data[pos:]) - if err != nil { - return nil, err - } - size, prefixLen := binary.Uvarint(data[pos:]) - entrySize := int(size) //nolint:gosec // loadNextBinaryEntry rejects sizes above math.MaxInt32. - entries = append(entries, readOnlyEntry{ - file: file, - dataOffset: int64(pos + prefixLen), - size: entrySize, - }) - pos += recordLen - } - return entries, nil -} - -func (log *readOnlyWAL[T]) Write(T) error { - return ErrReadOnly -} - -func (log *readOnlyWAL[T]) TruncateBefore(uint64) error { - return ErrReadOnly -} - -func (log *readOnlyWAL[T]) TruncateAfter(uint64) error { - return ErrReadOnly -} - -func (log *readOnlyWAL[T]) TruncateAll() error { - return ErrReadOnly -} - -func (log *readOnlyWAL[T]) FirstOffset() (uint64, error) { - if log.closed.Load() { - return 0, os.ErrClosed - } - if len(log.entries) == 0 { - return 0, nil - } - return log.firstOffset, nil -} - -func (log *readOnlyWAL[T]) LastOffset() (uint64, error) { - if log.closed.Load() { - return 0, os.ErrClosed - } - if len(log.entries) == 0 { - return 0, nil - } - return log.firstOffset + uint64(len(log.entries)) - 1, nil -} - -func (log *readOnlyWAL[T]) ReadAt(index uint64) (T, error) { - var zero T - if log.closed.Load() { - return zero, os.ErrClosed - } - if index < log.firstOffset || index-log.firstOffset >= uint64(len(log.entries)) { - return zero, fmt.Errorf("read WAL offset %d: out of range", index) - } - - entry := log.entries[index-log.firstOffset] - data := make([]byte, entry.size) - if _, err := entry.file.ReadAt(data, entry.dataOffset); err != nil { - return zero, fmt.Errorf("%w: read WAL offset %d: %w", ErrCorrupt, index, err) - } - value, err := log.unmarshal(data) - if err != nil { - return zero, fmt.Errorf("%w: unmarshal WAL offset %d: %w", ErrCorrupt, index, err) - } - return value, nil -} - -func (log *readOnlyWAL[T]) Replay(start, end uint64, processFn func(index uint64, entry T) error) error { - if end < start { - return nil - } - for index := start; index <= end; index++ { - entry, err := log.ReadAt(index) - if err != nil { - return err - } - if err := processFn(index, entry); err != nil { - return fmt.Errorf("process WAL offset %d: %w", index, err) - } - } - return nil -} - -func (log *readOnlyWAL[T]) Close() error { - log.closeOnce.Do(func() { - log.closed.Store(true) - var errs []error - for _, file := range log.files { - if err := file.Close(); err != nil { - errs = append(errs, err) - } - } - log.closeErr = errors.Join(errs...) - }) - return log.closeErr -} diff --git a/sei-db/wal/readonly_test.go b/sei-db/wal/readonly_test.go deleted file mode 100644 index a3d1c56034..0000000000 --- a/sei-db/wal/readonly_test.go +++ /dev/null @@ -1,294 +0,0 @@ -package wal - -import ( - "os" - "path/filepath" - "sort" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/sei-protocol/sei-chain/sei-db/proto" -) - -func TestOpenReadOnlyChangelogWALReplaysWithoutMutation(t *testing.T) { - dir := t.TempDir() - writable, err := NewChangelogWAL(dir, Config{}) - require.NoError(t, err) - writeReadOnlyTestData(t, writable) - require.NoError(t, writable.Close()) - - before := snapshotWALFiles(t, dir) - readOnly, err := OpenReadOnlyChangelogWAL(dir) - require.NoError(t, err) - - first, err := readOnly.FirstOffset() - require.NoError(t, err) - require.Equal(t, uint64(1), first) - last, err := readOnly.LastOffset() - require.NoError(t, err) - require.Equal(t, uint64(3), last) - - var names []string - require.NoError(t, readOnly.Replay(first, last, func(_ uint64, entry proto.ChangelogEntry) error { - names = append(names, entry.Changesets[0].Name) - return nil - })) - require.Equal(t, []string{"test", "test", "test"}, names) - require.Equal(t, before, snapshotWALFiles(t, dir)) - - require.ErrorIs(t, readOnly.Write(proto.ChangelogEntry{}), ErrReadOnly) - require.ErrorIs(t, readOnly.TruncateBefore(2), ErrReadOnly) - require.ErrorIs(t, readOnly.TruncateAfter(2), ErrReadOnly) - require.NoError(t, readOnly.Close()) - require.NoError(t, readOnly.Close()) -} - -func TestOpenReadOnlyChangelogWALRejectsTornTailWithoutRepair(t *testing.T) { - dir := t.TempDir() - writable, err := NewChangelogWAL(dir, Config{}) - require.NoError(t, err) - writeReadOnlyTestData(t, writable) - require.NoError(t, writable.Close()) - - segment := lastPlainWALSegment(t, dir) - file, err := os.OpenFile(filepath.Clean(segment), os.O_WRONLY|os.O_APPEND, 0) - require.NoError(t, err) - _, err = file.Write([]byte{0x10}) // declares a 16-byte record whose payload has not arrived - require.NoError(t, err) - require.NoError(t, file.Close()) - before := snapshotWALFiles(t, dir) - - _, err = OpenReadOnlyChangelogWAL(dir) - require.ErrorIs(t, err, ErrCorrupt) - require.Equal(t, before, snapshotWALFiles(t, dir), "read-only open must not repair the source tail") -} - -func TestOpenReadOnlyChangelogWALRejectsMalformedTail(t *testing.T) { - dir := t.TempDir() - segment := filepath.Join(dir, "00000000000000000001") - require.NoError(t, os.WriteFile(segment, []byte{ - 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, - }, 0o600)) - - _, err := OpenReadOnlyChangelogWAL(dir) - require.ErrorIs(t, err, ErrCorrupt) -} - -func TestOpenReadOnlyChangelogWALKeepsPointInTimeView(t *testing.T) { - dir := t.TempDir() - writable, err := NewChangelogWAL(dir, Config{}) - require.NoError(t, err) - require.NoError(t, writable.Write(proto.ChangelogEntry{Version: 1})) - - readOnly, err := OpenReadOnlyChangelogWAL(dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, readOnly.Close()) }) - - require.NoError(t, writable.Write(proto.ChangelogEntry{Version: 2})) - require.NoError(t, writable.Close()) - - last, err := readOnly.LastOffset() - require.NoError(t, err) - require.Equal(t, uint64(1), last) - _, err = readOnly.ReadAt(2) - require.Error(t, err) -} - -func TestOpenReadOnlyChangelogWALRejectsRecoveryMarkers(t *testing.T) { - for _, suffix := range []string{".START", ".END"} { - t.Run(suffix, func(t *testing.T) { - dir := t.TempDir() - marker := filepath.Join(dir, "00000000000000000001"+suffix) - require.NoError(t, os.WriteFile(marker, nil, 0o600)) - before := snapshotWALFiles(t, dir) - - _, err := OpenReadOnlyChangelogWAL(dir) - require.ErrorIs(t, err, ErrCorrupt) - require.Equal(t, before, snapshotWALFiles(t, dir), - "read-only open must not complete writable WAL recovery") - }) - } -} - -func TestOpenReadOnlyChangelogWALEmptyDirectory(t *testing.T) { - dir := t.TempDir() - - readOnly, err := OpenReadOnlyChangelogWAL(dir) - require.NoError(t, err) - first, err := readOnly.FirstOffset() - require.NoError(t, err) - require.Zero(t, first) - last, err := readOnly.LastOffset() - require.NoError(t, err) - require.Zero(t, last) - require.NoError(t, readOnly.Close()) - - entries, err := os.ReadDir(dir) - require.NoError(t, err) - require.Empty(t, entries, "read-only open must not create an initial segment") -} - -func TestOpenReadOnlyChangelogWALDoesNotCreateMissingDirectory(t *testing.T) { - dir := filepath.Join(t.TempDir(), "missing") - - _, err := OpenReadOnlyChangelogWAL(dir) - require.Error(t, err) - require.NoDirExists(t, dir) -} - -func TestOpenReadOnlyChangelogWALClassifiesVanishedSegmentAsCorrupt(t *testing.T) { - dir := t.TempDir() - segment := filepath.Join(dir, "00000000000000000001") - require.NoError(t, os.Symlink(filepath.Join(dir, "vanished"), segment)) - - _, err := OpenReadOnlyChangelogWAL(dir) - require.ErrorIs(t, err, ErrCorrupt) - require.ErrorIs(t, err, os.ErrNotExist) -} - -func TestReadOnlyChangelogWALClassifiesEntryReadFailuresAsCorrupt(t *testing.T) { - t.Run("short read", func(t *testing.T) { - dir := t.TempDir() - writable, err := NewChangelogWAL(dir, Config{}) - require.NoError(t, err) - require.NoError(t, writable.Write(proto.ChangelogEntry{Version: 1})) - require.NoError(t, writable.Close()) - - readOnly, err := OpenReadOnlyChangelogWAL(dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, readOnly.Close()) }) - require.NoError(t, os.Truncate(lastPlainWALSegment(t, dir), 0)) - - _, err = readOnly.ReadAt(1) - require.ErrorIs(t, err, ErrCorrupt) - }) - - t.Run("unparseable entry", func(t *testing.T) { - dir := t.TempDir() - segment := filepath.Join(dir, "00000000000000000001") - require.NoError(t, os.WriteFile(segment, []byte{0x01, 0xff}, 0o600)) - - readOnly, err := OpenReadOnlyChangelogWAL(dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, readOnly.Close()) }) - - _, err = readOnly.ReadAt(1) - require.ErrorIs(t, err, ErrCorrupt) - }) -} - -func TestOpenReadOnlyChangelogWALConcurrentWriter(t *testing.T) { - dir := t.TempDir() - writable, err := NewChangelogWAL(dir, Config{}) - require.NoError(t, err) - - const versions = 500 - done := make(chan struct{}) - writerErr := make(chan error, 1) - go func() { - var runErr error - for version := 1; version <= versions && runErr == nil; version++ { - runErr = writable.Write(proto.ChangelogEntry{Version: int64(version)}) - if runErr != nil || version <= 20 || version%10 != 0 { - continue - } - first, err := writable.FirstOffset() - if err != nil { - runErr = err - break - } - keepFrom := uint64(version - 20) - if keepFrom > first { - runErr = writable.TruncateBefore(keepFrom) - } - } - if closeErr := writable.Close(); runErr == nil { - runErr = closeErr - } - writerErr <- runErr - close(done) - }() - -reading: - for { - select { - case <-done: - break reading - default: - } - - readOnly, err := OpenReadOnlyChangelogWAL(dir) - if err != nil { - continue // source changed while opening; fail-closed and retry is valid - } - first, err := readOnly.FirstOffset() - require.NoError(t, err) - last, err := readOnly.LastOffset() - require.NoError(t, err) - if first > 0 { - require.GreaterOrEqual(t, last, first) - entry, err := readOnly.ReadAt(last) - require.NoError(t, err) - require.Equal(t, int64(last), entry.Version) - } - require.NoError(t, readOnly.Close()) - } - require.NoError(t, <-writerErr) - - readOnly, err := OpenReadOnlyChangelogWAL(dir) - require.NoError(t, err) - defer func() { require.NoError(t, readOnly.Close()) }() - last, err := readOnly.LastOffset() - require.NoError(t, err) - require.Equal(t, uint64(versions), last) - entry, err := readOnly.ReadAt(last) - require.NoError(t, err) - require.Equal(t, int64(versions), entry.Version) -} - -func lastPlainWALSegment(t *testing.T, dir string) string { - t.Helper() - entries, err := os.ReadDir(dir) - require.NoError(t, err) - var names []string - for _, entry := range entries { - if !entry.IsDir() && len(entry.Name()) == 20 { - names = append(names, entry.Name()) - } - } - require.NotEmpty(t, names) - sort.Strings(names) - return filepath.Join(dir, names[len(names)-1]) -} - -func snapshotWALFiles(t *testing.T, dir string) map[string][]byte { - t.Helper() - files := make(map[string][]byte) - entries, err := os.ReadDir(dir) - require.NoError(t, err) - for _, entry := range entries { - if entry.IsDir() { - continue - } - data, err := os.ReadFile(filepath.Clean(filepath.Join(dir, entry.Name()))) - require.NoError(t, err) - files[entry.Name()] = data - } - return files -} - -func writeReadOnlyTestData(t *testing.T, changelog ChangelogWAL) { - t.Helper() - for i, changeset := range ChangeSets { - require.NoError(t, changelog.Write(proto.ChangelogEntry{ - Version: int64(i + 1), - Changesets: []*proto.NamedChangeSet{{ - Name: "test", - Changeset: changeset, - }}, - })) - } -} diff --git a/sei-db/wal/verify.go b/sei-db/wal/verify.go new file mode 100644 index 0000000000..71302b443b --- /dev/null +++ b/sei-db/wal/verify.go @@ -0,0 +1,66 @@ +package wal + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/tidwall/wal" +) + +// ErrCorrupt reports a changelog that cannot be read without repair. +var ErrCorrupt = wal.ErrCorrupt + +// segmentNameLen is the length of a changelog segment file name. +const segmentNameLen = 20 + +// VerifyIntact reports whether the changelog in dir can be opened without +// repair. It returns ErrCorrupt when the tail segment ends mid-record or an +// interrupted truncation is still in progress, and never modifies dir. +// +// A reader on a live node calls this before it opens the changelog, because the +// opener repairs what it finds: a torn tail is truncated, and an interrupted +// truncation is completed by renaming or removing segments. On a live node a +// torn tail is usually a write in progress rather than lasting damage, so the +// caller reruns instead of repairing. +func VerifyIntact(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("read changelog dir %s: %w", dir, err) + } + + // os.ReadDir sorts by name, and segment names are zero-padded, so the last + // match is the tail segment the opener would truncate. + var tail string + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || len(name) < segmentNameLen { + continue + } + if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { + return fmt.Errorf("%w: changelog truncation marker %s is present in %s", ErrCorrupt, name, dir) + } + tail = name + } + if tail == "" { + return nil + } + + path := filepath.Join(dir, tail) + data, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return fmt.Errorf("read changelog segment %s: %w", path, err) + } + for pos := 0; pos < len(data); { + n, err := loadNextBinaryEntry(data[pos:]) + if err != nil { + return fmt.Errorf("%w: changelog segment %s ends mid-record at offset %d", ErrCorrupt, path, pos) + } + pos += n + } + return nil +} diff --git a/sei-db/wal/verify_test.go b/sei-db/wal/verify_test.go new file mode 100644 index 0000000000..1b6f945dd4 --- /dev/null +++ b/sei-db/wal/verify_test.go @@ -0,0 +1,122 @@ +package wal + +import ( + "encoding/binary" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/wal" +) + +func TestVerifyIntactAcceptsCompleteChangelog(t *testing.T) { + dir := writeTestSegment(t, appendBinaryEntry(appendBinaryEntry(nil, []byte("one")), []byte("two"))) + before := snapshotDir(t, dir) + + require.NoError(t, VerifyIntact(dir)) + require.Equal(t, before, snapshotDir(t, dir)) +} + +func TestVerifyIntactRejectsTornTail(t *testing.T) { + complete := appendBinaryEntry(nil, []byte("complete")) + torn := appendBinaryEntry(complete, []byte("torn")) + dir := writeTestSegment(t, torn[:len(torn)-1]) + before := snapshotDir(t, dir) + + err := VerifyIntact(dir) + require.ErrorIs(t, err, ErrCorrupt) + require.ErrorIs(t, err, wal.ErrCorrupt) + require.Contains(t, err.Error(), "ends mid-record") + require.Equal(t, before, snapshotDir(t, dir)) +} + +func TestVerifyIntactRejectsInterruptedTruncation(t *testing.T) { + dir := writeTestSegment(t, appendBinaryEntry(nil, []byte("complete"))) + marker := filepath.Join(dir, "00000000000000000002.START") + require.NoError(t, os.WriteFile(marker, appendBinaryEntry(nil, []byte("moved")), 0o600)) + before := snapshotDir(t, dir) + + err := VerifyIntact(dir) + require.ErrorIs(t, err, ErrCorrupt) + require.Contains(t, err.Error(), "truncation marker") + require.Equal(t, before, snapshotDir(t, dir)) +} + +func TestVerifyIntactAcceptsMissingOrEmptyChangelog(t *testing.T) { + missing := filepath.Join(t.TempDir(), "changelog") + require.NoError(t, VerifyIntact(missing)) + require.NoFileExists(t, missing) + + require.NoError(t, VerifyIntact(t.TempDir())) +} + +func TestVerifyIntactAcceptsAChangelogTheWritableOpenerWrote(t *testing.T) { + dir := t.TempDir() + log, err := open(dir, nil) + require.NoError(t, err) + require.NoError(t, log.Write(1, []byte("entry"))) + require.NoError(t, log.Close()) + + require.NoError(t, VerifyIntact(dir)) +} + +// TestVerifyIntactRejectsWhatTheWritableOpenerTruncates pins the reason the +// check exists: the same tail it rejects is one the writable opener repairs in +// place. +func TestVerifyIntactRejectsWhatTheWritableOpenerTruncates(t *testing.T) { + dir := t.TempDir() + log, err := open(dir, nil) + require.NoError(t, err) + require.NoError(t, log.Write(1, []byte("entry"))) + require.NoError(t, log.Close()) + + segment := filepath.Join(dir, "00000000000000000001") + data, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + require.NoError(t, os.WriteFile(segment, append(data, 0x08), 0o600)) + before := snapshotDir(t, dir) + + require.ErrorIs(t, VerifyIntact(dir), ErrCorrupt) + require.Equal(t, before, snapshotDir(t, dir)) + + log, err = open(dir, nil) + require.NoError(t, err) + require.NoError(t, log.Close()) + require.NotEqual(t, before, snapshotDir(t, dir)) +} + +// writeTestSegment creates a changelog directory holding data as its only +// segment, and returns the directory. +func writeTestSegment(t *testing.T, data []byte) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "00000000000000000001"), data, 0o600)) + return dir +} + +// snapshotDir returns the contents of every file in dir, keyed by name. +func snapshotDir(t *testing.T, dir string) map[string][]byte { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + files := make(map[string][]byte, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + data, readErr := os.ReadFile(filepath.Clean(filepath.Join(dir, entry.Name()))) + require.NoError(t, readErr) + files[entry.Name()] = data + } + return files +} + +// appendBinaryEntry appends payload to data in the binary changelog framing of +// a size varint followed by the payload. +func appendBinaryEntry(data []byte, payload []byte) []byte { + var size [binary.MaxVarintLen64]byte + n := binary.PutUvarint(size[:], uint64(len(payload))) + data = append(data, size[:n]...) + return append(data, payload...) +} From 37c745f76f1ca0d9f9d2fc30b3ff0fcb6d5ea571 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 17:48:00 -0400 Subject: [PATCH 06/13] refactor(wal): move VerifyIntact beside the repair it avoids VerifyIntact sat in its own file, away from truncateCorruptedTail and loadNextBinaryEntry, which are the repair it guards against and the framing it reuses. Co-authored-by: Cursor --- sei-db/wal/utils.go | 58 ++++++++++++++++++ sei-db/wal/verify.go | 66 --------------------- sei-db/wal/verify_test.go | 122 -------------------------------------- sei-db/wal/wal_test.go | 100 +++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 188 deletions(-) delete mode 100644 sei-db/wal/verify.go delete mode 100644 sei-db/wal/verify_test.go diff --git a/sei-db/wal/utils.go b/sei-db/wal/utils.go index a33cf5f33d..725024a198 100644 --- a/sei-db/wal/utils.go +++ b/sei-db/wal/utils.go @@ -4,9 +4,11 @@ import ( "bytes" "encoding/binary" "errors" + "fmt" "math" "os" "path/filepath" + "strings" "unsafe" "github.com/tidwall/gjson" @@ -32,6 +34,62 @@ func GetLastIndex(dir string) (index uint64, err error) { return rlog.LastIndex() } +// ErrCorrupt reports a log that cannot be read without repair. +var ErrCorrupt = wal.ErrCorrupt + +// segmentNameLen is the length of a log segment file name. +const segmentNameLen = 20 + +// VerifyIntact reports whether the binary log in dir can be opened without +// repair. It returns ErrCorrupt when the tail segment ends mid-record or an +// interrupted truncation is still in progress, and never modifies dir. +// +// A reader on a live node calls this before it opens the log, because open +// repairs what it finds: truncateCorruptedTail cuts a torn tail, and tidwall +// completes an interrupted truncation by renaming or removing segments. On a +// live node a torn tail is usually a write in progress rather than lasting +// damage, so the caller reruns instead of repairing. +func VerifyIntact(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("read wal dir %s: %w", dir, err) + } + + // os.ReadDir sorts by name, and segment names are zero-padded, so the last + // match is the tail segment open would truncate. + var tail string + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || len(name) < segmentNameLen { + continue + } + if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { + return fmt.Errorf("%w: truncation marker %s is present in %s", ErrCorrupt, name, dir) + } + tail = name + } + if tail == "" { + return nil + } + + path := filepath.Join(dir, tail) + data, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return fmt.Errorf("read wal segment %s: %w", path, err) + } + for pos := 0; pos < len(data); { + n, err := loadNextBinaryEntry(data[pos:]) + if err != nil { + return fmt.Errorf("%w: segment %s ends mid-record at offset %d", ErrCorrupt, path, pos) + } + pos += n + } + return nil +} + // truncateCorruptedTail truncates the corrupted tail func truncateCorruptedTail(path string, format wal.LogFormat) error { data, err := os.ReadFile(filepath.Clean(path)) diff --git a/sei-db/wal/verify.go b/sei-db/wal/verify.go deleted file mode 100644 index 71302b443b..0000000000 --- a/sei-db/wal/verify.go +++ /dev/null @@ -1,66 +0,0 @@ -package wal - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/tidwall/wal" -) - -// ErrCorrupt reports a changelog that cannot be read without repair. -var ErrCorrupt = wal.ErrCorrupt - -// segmentNameLen is the length of a changelog segment file name. -const segmentNameLen = 20 - -// VerifyIntact reports whether the changelog in dir can be opened without -// repair. It returns ErrCorrupt when the tail segment ends mid-record or an -// interrupted truncation is still in progress, and never modifies dir. -// -// A reader on a live node calls this before it opens the changelog, because the -// opener repairs what it finds: a torn tail is truncated, and an interrupted -// truncation is completed by renaming or removing segments. On a live node a -// torn tail is usually a write in progress rather than lasting damage, so the -// caller reruns instead of repairing. -func VerifyIntact(dir string) error { - entries, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return fmt.Errorf("read changelog dir %s: %w", dir, err) - } - - // os.ReadDir sorts by name, and segment names are zero-padded, so the last - // match is the tail segment the opener would truncate. - var tail string - for _, entry := range entries { - name := entry.Name() - if entry.IsDir() || len(name) < segmentNameLen { - continue - } - if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { - return fmt.Errorf("%w: changelog truncation marker %s is present in %s", ErrCorrupt, name, dir) - } - tail = name - } - if tail == "" { - return nil - } - - path := filepath.Join(dir, tail) - data, err := os.ReadFile(filepath.Clean(path)) - if err != nil { - return fmt.Errorf("read changelog segment %s: %w", path, err) - } - for pos := 0; pos < len(data); { - n, err := loadNextBinaryEntry(data[pos:]) - if err != nil { - return fmt.Errorf("%w: changelog segment %s ends mid-record at offset %d", ErrCorrupt, path, pos) - } - pos += n - } - return nil -} diff --git a/sei-db/wal/verify_test.go b/sei-db/wal/verify_test.go deleted file mode 100644 index 1b6f945dd4..0000000000 --- a/sei-db/wal/verify_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package wal - -import ( - "encoding/binary" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - "github.com/tidwall/wal" -) - -func TestVerifyIntactAcceptsCompleteChangelog(t *testing.T) { - dir := writeTestSegment(t, appendBinaryEntry(appendBinaryEntry(nil, []byte("one")), []byte("two"))) - before := snapshotDir(t, dir) - - require.NoError(t, VerifyIntact(dir)) - require.Equal(t, before, snapshotDir(t, dir)) -} - -func TestVerifyIntactRejectsTornTail(t *testing.T) { - complete := appendBinaryEntry(nil, []byte("complete")) - torn := appendBinaryEntry(complete, []byte("torn")) - dir := writeTestSegment(t, torn[:len(torn)-1]) - before := snapshotDir(t, dir) - - err := VerifyIntact(dir) - require.ErrorIs(t, err, ErrCorrupt) - require.ErrorIs(t, err, wal.ErrCorrupt) - require.Contains(t, err.Error(), "ends mid-record") - require.Equal(t, before, snapshotDir(t, dir)) -} - -func TestVerifyIntactRejectsInterruptedTruncation(t *testing.T) { - dir := writeTestSegment(t, appendBinaryEntry(nil, []byte("complete"))) - marker := filepath.Join(dir, "00000000000000000002.START") - require.NoError(t, os.WriteFile(marker, appendBinaryEntry(nil, []byte("moved")), 0o600)) - before := snapshotDir(t, dir) - - err := VerifyIntact(dir) - require.ErrorIs(t, err, ErrCorrupt) - require.Contains(t, err.Error(), "truncation marker") - require.Equal(t, before, snapshotDir(t, dir)) -} - -func TestVerifyIntactAcceptsMissingOrEmptyChangelog(t *testing.T) { - missing := filepath.Join(t.TempDir(), "changelog") - require.NoError(t, VerifyIntact(missing)) - require.NoFileExists(t, missing) - - require.NoError(t, VerifyIntact(t.TempDir())) -} - -func TestVerifyIntactAcceptsAChangelogTheWritableOpenerWrote(t *testing.T) { - dir := t.TempDir() - log, err := open(dir, nil) - require.NoError(t, err) - require.NoError(t, log.Write(1, []byte("entry"))) - require.NoError(t, log.Close()) - - require.NoError(t, VerifyIntact(dir)) -} - -// TestVerifyIntactRejectsWhatTheWritableOpenerTruncates pins the reason the -// check exists: the same tail it rejects is one the writable opener repairs in -// place. -func TestVerifyIntactRejectsWhatTheWritableOpenerTruncates(t *testing.T) { - dir := t.TempDir() - log, err := open(dir, nil) - require.NoError(t, err) - require.NoError(t, log.Write(1, []byte("entry"))) - require.NoError(t, log.Close()) - - segment := filepath.Join(dir, "00000000000000000001") - data, err := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, err) - require.NoError(t, os.WriteFile(segment, append(data, 0x08), 0o600)) - before := snapshotDir(t, dir) - - require.ErrorIs(t, VerifyIntact(dir), ErrCorrupt) - require.Equal(t, before, snapshotDir(t, dir)) - - log, err = open(dir, nil) - require.NoError(t, err) - require.NoError(t, log.Close()) - require.NotEqual(t, before, snapshotDir(t, dir)) -} - -// writeTestSegment creates a changelog directory holding data as its only -// segment, and returns the directory. -func writeTestSegment(t *testing.T, data []byte) string { - t.Helper() - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "00000000000000000001"), data, 0o600)) - return dir -} - -// snapshotDir returns the contents of every file in dir, keyed by name. -func snapshotDir(t *testing.T, dir string) map[string][]byte { - t.Helper() - entries, err := os.ReadDir(dir) - require.NoError(t, err) - files := make(map[string][]byte, len(entries)) - for _, entry := range entries { - if entry.IsDir() { - continue - } - data, readErr := os.ReadFile(filepath.Clean(filepath.Join(dir, entry.Name()))) - require.NoError(t, readErr) - files[entry.Name()] = data - } - return files -} - -// appendBinaryEntry appends payload to data in the binary changelog framing of -// a size varint followed by the payload. -func appendBinaryEntry(data []byte, payload []byte) []byte { - var size [binary.MaxVarintLen64]byte - n := binary.PutUvarint(size[:], uint64(len(payload))) - data = append(data, size[:n]...) - return append(data, payload...) -} diff --git a/sei-db/wal/wal_test.go b/sei-db/wal/wal_test.go index a83d5fded3..a475c69f9b 100644 --- a/sei-db/wal/wal_test.go +++ b/sei-db/wal/wal_test.go @@ -1,6 +1,7 @@ package wal import ( + "encoding/binary" "fmt" "os" "path/filepath" @@ -67,6 +68,105 @@ func TestOpenAndCorruptedTail(t *testing.T) { } } +func TestVerifyIntactAcceptsCompleteLog(t *testing.T) { + dir := writeTestSegment(t, appendBinaryEntry(appendBinaryEntry(nil, []byte("one")), []byte("two"))) + before := snapshotDir(t, dir) + + require.NoError(t, VerifyIntact(dir)) + require.Equal(t, before, snapshotDir(t, dir)) +} + +func TestVerifyIntactRejectsTornTail(t *testing.T) { + torn := appendBinaryEntry(appendBinaryEntry(nil, []byte("complete")), []byte("torn")) + dir := writeTestSegment(t, torn[:len(torn)-1]) + before := snapshotDir(t, dir) + + err := VerifyIntact(dir) + require.ErrorIs(t, err, ErrCorrupt) + require.Contains(t, err.Error(), "ends mid-record") + require.Equal(t, before, snapshotDir(t, dir)) +} + +func TestVerifyIntactRejectsInterruptedTruncation(t *testing.T) { + dir := writeTestSegment(t, appendBinaryEntry(nil, []byte("complete"))) + marker := filepath.Join(dir, "00000000000000000002.START") + require.NoError(t, os.WriteFile(marker, appendBinaryEntry(nil, []byte("moved")), 0o600)) + before := snapshotDir(t, dir) + + err := VerifyIntact(dir) + require.ErrorIs(t, err, ErrCorrupt) + require.Contains(t, err.Error(), "truncation marker") + require.Equal(t, before, snapshotDir(t, dir)) +} + +func TestVerifyIntactAcceptsMissingOrEmptyLog(t *testing.T) { + missing := filepath.Join(t.TempDir(), "changelog") + require.NoError(t, VerifyIntact(missing)) + require.NoFileExists(t, missing) + + require.NoError(t, VerifyIntact(t.TempDir())) +} + +// TestVerifyIntactRejectsWhatOpenTruncates pins the reason VerifyIntact exists: +// the tail it rejects is one open repairs in place. +func TestVerifyIntactRejectsWhatOpenTruncates(t *testing.T) { + dir := t.TempDir() + log, err := open(dir, nil) + require.NoError(t, err) + require.NoError(t, log.Write(1, []byte("entry"))) + require.NoError(t, log.Close()) + require.NoError(t, VerifyIntact(dir)) + + segment := filepath.Join(dir, "00000000000000000001") + data, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + require.NoError(t, os.WriteFile(segment, append(data, 0x08), 0o600)) + before := snapshotDir(t, dir) + + require.ErrorIs(t, VerifyIntact(dir), ErrCorrupt) + require.Equal(t, before, snapshotDir(t, dir)) + + log, err = open(dir, nil) + require.NoError(t, err) + require.NoError(t, log.Close()) + require.NotEqual(t, before, snapshotDir(t, dir)) +} + +// writeTestSegment creates a log directory holding data as its only segment, +// and returns the directory. +func writeTestSegment(t *testing.T, data []byte) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "00000000000000000001"), data, 0o600)) + return dir +} + +// snapshotDir returns the contents of every file in dir, keyed by name. +func snapshotDir(t *testing.T, dir string) map[string][]byte { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + files := make(map[string][]byte, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + data, readErr := os.ReadFile(filepath.Clean(filepath.Join(dir, entry.Name()))) + require.NoError(t, readErr) + files[entry.Name()] = data + } + return files +} + +// appendBinaryEntry appends payload to data in the binary log framing of a size +// varint followed by the payload. +func appendBinaryEntry(data []byte, payload []byte) []byte { + var size [binary.MaxVarintLen64]byte + n := binary.PutUvarint(size[:], uint64(len(payload))) + data = append(data, size[:n]...) + return append(data, payload...) +} + func TestReplay(t *testing.T) { changelog := prepareTestData(t) var total = 0 From 9075ecbee09f3d77b8e891bc3e3b62cff1b5794c Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 21 Aug 2026 23:02:39 -0400 Subject: [PATCH 07/13] fix(seidb): reject a digest replay that skipped pruned versions Catchup starts at the changelog's first offset whenever the snapshot ends before it, so a changelog pruned past the snapshot replays a contiguous suffix, reaches the requested height, and omits the versions in between. Comparing the final version to --height does not catch that, and a digest missing intermediate versions reads as a state mismatch between nodes. Co-authored-by: Cursor --- .../seidb/operations/evm_logical_digest.go | 40 ++++++++++-- .../cmd/seidb/operations/memiavl_open_test.go | 61 +++++++++++++++++++ 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index ba070ceab2..942cf713cf 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -1129,15 +1129,47 @@ func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) if err != nil { return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } - if height > 0 && db.Version() != height { - reached := db.Version() + if err := verifyReplayCoverage(db, height); err != nil { _ = db.Close() - return nil, fmt.Errorf("memiavl replay reached version %d, not the requested height %d; "+ - "the changelog does not cover that height", reached, height) + return nil, err } return db, nil } +// verifyReplayCoverage reports whether the opened DB replayed every version +// between its snapshot and height. +// +// The final version alone does not prove coverage. Catchup starts at the +// changelog's first offset whenever the snapshot ends before it, so a changelog +// pruned past the snapshot replays a contiguous suffix, reaches the requested +// height, and silently omits the versions in between. +func verifyReplayCoverage(db *memiavl.DB, height int64) error { + if height > 0 && db.Version() != height { + return fmt.Errorf("memiavl replay reached version %d, not the requested height %d; "+ + "the changelog does not cover that height", db.Version(), height) + } + snapshotVersion := db.SnapshotVersion() + if db.Version() <= snapshotVersion { + return nil + } + firstOffset, err := db.GetWAL().FirstOffset() + if err != nil { + return fmt.Errorf("read memiavl changelog first offset: %w", err) + } + if firstOffset == 0 { + return nil + } + // #nosec G115 -- WAL offsets are far below MaxInt64 in practice. + firstVersion := int64(firstOffset) + db.GetWALIndexDelta() + if firstVersion > snapshotVersion+1 { + return fmt.Errorf("the memiavl changelog starts at version %d but the snapshot ends at version "+ + "%d, so versions %d-%d would be missing from the replay; digest a height at or below %d, "+ + "or use --memiavl-open-mode snapshot", + firstVersion, snapshotVersion, snapshotVersion+1, firstVersion-1, snapshotVersion) + } + return nil +} + func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error { db, err := openMemiAVLReplayReadOnly(dbDir, height) if err != nil { diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index 93f27cf5e8..2a6693b4fe 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -1,6 +1,7 @@ package operations import ( + "context" "os" "path/filepath" "sort" @@ -11,6 +12,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/keys" "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/wal" ) func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing.T) { @@ -42,6 +44,65 @@ func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing. require.Equal(t, before, after) } +func TestOpenMemiAVLReplayReadOnlyAcceptsAFullyCoveredHeight(t *testing.T) { + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + for nonce := uint64(1); nonce <= 3; nonce++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}}, + }})) + _, err := store.Commit() + require.NoError(t, err) + } + require.NoError(t, store.Close()) + + db, err := openMemiAVLReplayReadOnly(utils.GetCosmosSCStorePath(homeDir), 3) + require.NoError(t, err) + defer func() { _ = db.Close() }() + require.Equal(t, int64(3), db.Version()) +} + +// TestOpenMemiAVLReplayReadOnlyRejectsAPrunedChangelogGap covers a changelog +// pruned past the snapshot. Replay reaches the requested height from a +// contiguous suffix, so the final version looks correct while the versions +// between the snapshot and the changelog's first entry were never applied. +func TestOpenMemiAVLReplayReadOnlyRejectsAPrunedChangelogGap(t *testing.T) { + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + commit := func(nonce uint64) { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}}, + }})) + _, err := store.Commit() + require.NoError(t, err) + } + commit(1) + commit(2) + // Snapshot version 2 so the evm tree survives without the changelog entry + // that creates it, leaving the pruned gap as the only defect. + require.NoError(t, store.GetDB().RewriteSnapshot(context.Background())) + commit(3) + commit(4) + commit(5) + require.NoError(t, store.Close()) + + dbDir := utils.GetCosmosSCStorePath(homeDir) + changelog, err := wal.NewChangelogWAL(utils.GetChangelogPath(dbDir), wal.Config{}) + require.NoError(t, err) + require.NoError(t, changelog.TruncateBefore(5)) + require.NoError(t, changelog.Close()) + + db, err := openMemiAVLReplayReadOnly(dbDir, 5) + if db != nil { + _ = db.Close() + } + require.Error(t, err) + require.Contains(t, err.Error(), "would be missing from the replay") + require.Contains(t, err.Error(), "--memiavl-open-mode snapshot") +} + func lastOperationsMemiAVLWALSegment(t *testing.T, dbDir string) string { t.Helper() changelogDir := utils.GetChangelogPath(dbDir) From 38feadd597da4f94c25693153ea5de14f6cbd344 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 27 Aug 2026 09:39:49 -0400 Subject: [PATCH 08/13] test(seidb): adopt the CommitStore.Commit version cross-check main gave CommitStore.Commit a version argument. The merge was textually clean because it touched no file this branch changed, so the break only showed up at compile time. Co-authored-by: Cursor --- sei-db/tools/cmd/seidb/operations/memiavl_open_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index 2a6693b4fe..438f9f1c5f 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -22,7 +22,7 @@ func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing. Name: keys.EVMStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}}, }})) - _, err := store.Commit() + _, err := store.Commit(store.Version() + 1) require.NoError(t, err) require.NoError(t, store.Close()) @@ -52,7 +52,7 @@ func TestOpenMemiAVLReplayReadOnlyAcceptsAFullyCoveredHeight(t *testing.T) { Name: keys.EVMStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}}, }})) - _, err := store.Commit() + _, err := store.Commit(store.Version() + 1) require.NoError(t, err) } require.NoError(t, store.Close()) @@ -75,7 +75,7 @@ func TestOpenMemiAVLReplayReadOnlyRejectsAPrunedChangelogGap(t *testing.T) { Name: keys.EVMStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}}, }})) - _, err := store.Commit() + _, err := store.Commit(store.Version() + 1) require.NoError(t, err) } commit(1) From d47e81bb9a76d55de7f55dcc622de402497eb40c Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 27 Aug 2026 12:00:33 -0400 Subject: [PATCH 09/13] fix(seidb): stop the gap check from rejecting seeded chains The check flagged any distance between the snapshot and the changelog's first entry. A chain seeded above height 1 has that distance legitimately: its changelog starts at the initial version while initEmptyDB leaves the snapshot at version 0, and the versions in between never existed. Ask instead whether the offset the replay needs existed and was pruned, which is the branch of Catchup's clamp that loses data. Co-authored-by: Cursor --- .../seidb/operations/evm_logical_digest.go | 24 +++++++++++------ .../cmd/seidb/operations/memiavl_open_test.go | 27 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 942cf713cf..c7495c93b0 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -1159,15 +1159,23 @@ func verifyReplayCoverage(db *memiavl.DB, height int64) error { if firstOffset == 0 { return nil } - // #nosec G115 -- WAL offsets are far below MaxInt64 in practice. - firstVersion := int64(firstOffset) + db.GetWALIndexDelta() - if firstVersion > snapshotVersion+1 { - return fmt.Errorf("the memiavl changelog starts at version %d but the snapshot ends at version "+ - "%d, so versions %d-%d would be missing from the replay; digest a height at or below %d, "+ - "or use --memiavl-open-mode snapshot", - firstVersion, snapshotVersion, snapshotVersion+1, firstVersion-1, snapshotVersion) + // #nosec G115 -- changelog offsets stay far below MaxInt64. + firstIndex := int64(firstOffset) + delta := db.GetWALIndexDelta() + // Catchup raises its start offset to the changelog's first offset. That is + // correct when the version after the snapshot never had an offset, which is how + // a chain whose initial version is above 1 looks until its first snapshot + // rewrite. It drops committed versions only when the offset existed and was + // pruned away. + wantIndex := snapshotVersion + 1 - delta + if wantIndex <= 0 || wantIndex >= firstIndex { + return nil } - return nil + firstVersion := firstIndex + delta + return fmt.Errorf("the memiavl changelog starts at version %d but the snapshot ends at version "+ + "%d, so versions %d-%d would be missing from the replay; digest a height at or below %d, "+ + "or use --memiavl-open-mode snapshot", + firstVersion, snapshotVersion, snapshotVersion+1, firstVersion-1, snapshotVersion) } func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error { diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index 438f9f1c5f..5df81aba0f 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -63,6 +63,33 @@ func TestOpenMemiAVLReplayReadOnlyAcceptsAFullyCoveredHeight(t *testing.T) { require.Equal(t, int64(3), db.Version()) } +// TestOpenMemiAVLReplayReadOnlyAcceptsAnInitialVersionAboveOne covers a chain +// seeded above height 1. Its changelog starts at the initial version while +// initEmptyDB leaves the snapshot at version 0, so the versions below the +// changelog never existed and the replay is complete despite the distance. +func TestOpenMemiAVLReplayReadOnlyAcceptsAnInitialVersionAboveOne(t *testing.T) { + const initialVersion = 1000 + + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + require.NoError(t, store.SetInitialVersion(initialVersion)) + for nonce := uint64(1); nonce <= 2; nonce++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}}, + }})) + _, err := store.Commit(store.Version() + 1) + require.NoError(t, err) + } + require.NoError(t, store.Close()) + + db, err := openMemiAVLReplayReadOnly(utils.GetCosmosSCStorePath(homeDir), initialVersion+1) + require.NoError(t, err) + defer func() { _ = db.Close() }() + require.Equal(t, int64(initialVersion+1), db.Version()) + require.Equal(t, int64(0), db.SnapshotVersion()) +} + // TestOpenMemiAVLReplayReadOnlyRejectsAPrunedChangelogGap covers a changelog // pruned past the snapshot. Replay reaches the requested height from a // contiguous suffix, so the final version looks correct while the versions From efde1047a7ae7bb25421dbca30994eb4b5d11d10 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 28 Aug 2026 11:03:23 -0400 Subject: [PATCH 10/13] refactor(seidb): drop the digest replay-coverage guard tryTruncateWAL anchors its changelog cut at the earliest retained snapshot and seekSnapshot refuses a height below every snapshot, so the pruned-gap the guard reported cannot arise unless the data directory was assembled or pruned by hand. It also had to special-case a chain seeded above height 1, where the distance it measured is legitimate. Keep the pre-flight check the change exists for, and keep one positive control for it. Guarding a silent memIAVL replay gap belongs in memIAVL rather than in one caller. Co-authored-by: Cursor --- .../seidb/operations/evm_logical_digest.go | 46 ------------ .../cmd/seidb/operations/memiavl_open_test.go | 74 +------------------ 2 files changed, 4 insertions(+), 116 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index c7495c93b0..576293286f 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -1129,55 +1129,9 @@ func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) if err != nil { return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } - if err := verifyReplayCoverage(db, height); err != nil { - _ = db.Close() - return nil, err - } return db, nil } -// verifyReplayCoverage reports whether the opened DB replayed every version -// between its snapshot and height. -// -// The final version alone does not prove coverage. Catchup starts at the -// changelog's first offset whenever the snapshot ends before it, so a changelog -// pruned past the snapshot replays a contiguous suffix, reaches the requested -// height, and silently omits the versions in between. -func verifyReplayCoverage(db *memiavl.DB, height int64) error { - if height > 0 && db.Version() != height { - return fmt.Errorf("memiavl replay reached version %d, not the requested height %d; "+ - "the changelog does not cover that height", db.Version(), height) - } - snapshotVersion := db.SnapshotVersion() - if db.Version() <= snapshotVersion { - return nil - } - firstOffset, err := db.GetWAL().FirstOffset() - if err != nil { - return fmt.Errorf("read memiavl changelog first offset: %w", err) - } - if firstOffset == 0 { - return nil - } - // #nosec G115 -- changelog offsets stay far below MaxInt64. - firstIndex := int64(firstOffset) - delta := db.GetWALIndexDelta() - // Catchup raises its start offset to the changelog's first offset. That is - // correct when the version after the snapshot never had an offset, which is how - // a chain whose initial version is above 1 looks until its first snapshot - // rewrite. It drops committed versions only when the offset existed and was - // pruned away. - wantIndex := snapshotVersion + 1 - delta - if wantIndex <= 0 || wantIndex >= firstIndex { - return nil - } - firstVersion := firstIndex + delta - return fmt.Errorf("the memiavl changelog starts at version %d but the snapshot ends at version "+ - "%d, so versions %d-%d would be missing from the replay; digest a height at or below %d, "+ - "or use --memiavl-open-mode snapshot", - firstVersion, snapshotVersion, snapshotVersion+1, firstVersion-1, snapshotVersion) -} - func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error { db, err := openMemiAVLReplayReadOnly(dbDir, height) if err != nil { diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index 5df81aba0f..e3d752186e 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -1,7 +1,6 @@ package operations import ( - "context" "os" "path/filepath" "sort" @@ -12,7 +11,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/keys" "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/proto" - "github.com/sei-protocol/sei-chain/sei-db/wal" ) func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing.T) { @@ -44,7 +42,10 @@ func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing. require.Equal(t, before, after) } -func TestOpenMemiAVLReplayReadOnlyAcceptsAFullyCoveredHeight(t *testing.T) { +// TestOpenMemiAVLReplayReadOnlyAcceptsACompleteChangelog is the positive +// control for the pre-flight check: a changelog a real store wrote and closed +// opens and replays as it did before the check existed. +func TestOpenMemiAVLReplayReadOnlyAcceptsACompleteChangelog(t *testing.T) { homeDir := t.TempDir() store := newTestMemiavlStore(t, homeDir) for nonce := uint64(1); nonce <= 3; nonce++ { @@ -63,73 +64,6 @@ func TestOpenMemiAVLReplayReadOnlyAcceptsAFullyCoveredHeight(t *testing.T) { require.Equal(t, int64(3), db.Version()) } -// TestOpenMemiAVLReplayReadOnlyAcceptsAnInitialVersionAboveOne covers a chain -// seeded above height 1. Its changelog starts at the initial version while -// initEmptyDB leaves the snapshot at version 0, so the versions below the -// changelog never existed and the replay is complete despite the distance. -func TestOpenMemiAVLReplayReadOnlyAcceptsAnInitialVersionAboveOne(t *testing.T) { - const initialVersion = 1000 - - homeDir := t.TempDir() - store := newTestMemiavlStore(t, homeDir) - require.NoError(t, store.SetInitialVersion(initialVersion)) - for nonce := uint64(1); nonce <= 2; nonce++ { - require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: keys.EVMStoreKey, - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}}, - }})) - _, err := store.Commit(store.Version() + 1) - require.NoError(t, err) - } - require.NoError(t, store.Close()) - - db, err := openMemiAVLReplayReadOnly(utils.GetCosmosSCStorePath(homeDir), initialVersion+1) - require.NoError(t, err) - defer func() { _ = db.Close() }() - require.Equal(t, int64(initialVersion+1), db.Version()) - require.Equal(t, int64(0), db.SnapshotVersion()) -} - -// TestOpenMemiAVLReplayReadOnlyRejectsAPrunedChangelogGap covers a changelog -// pruned past the snapshot. Replay reaches the requested height from a -// contiguous suffix, so the final version looks correct while the versions -// between the snapshot and the changelog's first entry were never applied. -func TestOpenMemiAVLReplayReadOnlyRejectsAPrunedChangelogGap(t *testing.T) { - homeDir := t.TempDir() - store := newTestMemiavlStore(t, homeDir) - commit := func(nonce uint64) { - require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: keys.EVMStoreKey, - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}}, - }})) - _, err := store.Commit(store.Version() + 1) - require.NoError(t, err) - } - commit(1) - commit(2) - // Snapshot version 2 so the evm tree survives without the changelog entry - // that creates it, leaving the pruned gap as the only defect. - require.NoError(t, store.GetDB().RewriteSnapshot(context.Background())) - commit(3) - commit(4) - commit(5) - require.NoError(t, store.Close()) - - dbDir := utils.GetCosmosSCStorePath(homeDir) - changelog, err := wal.NewChangelogWAL(utils.GetChangelogPath(dbDir), wal.Config{}) - require.NoError(t, err) - require.NoError(t, changelog.TruncateBefore(5)) - require.NoError(t, changelog.Close()) - - db, err := openMemiAVLReplayReadOnly(dbDir, 5) - if db != nil { - _ = db.Close() - } - require.Error(t, err) - require.Contains(t, err.Error(), "would be missing from the replay") - require.Contains(t, err.Error(), "--memiavl-open-mode snapshot") -} - func lastOperationsMemiAVLWALSegment(t *testing.T, dbDir string) string { t.Helper() changelogDir := utils.GetChangelogPath(dbDir) From 5eb3d8ed2c0c93330a068bb615a0974c5b859694 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 28 Aug 2026 14:10:53 -0400 Subject: [PATCH 11/13] fix(seidb): stop the digest replay open from repairing the changelog Checking the changelog and then opening it anyway left the repair reachable. The gap between the two spans LoadMultiTree, which is orders of magnitude longer than a torn tail survives, so a tail torn after the check was truncated by the open exactly as before. The check only ever covered the conditions that persist. Refuse inside the open instead. Config.NoRepairOnOpen makes open return ErrCorrupt for a torn tail rather than truncating it, and refuse a directory holding a .START/.END marker before wal.Open completes that truncation. memiavl passes it through as Options.NoChangelogRepair, and the digest sets it for replay mode. Default is off, so every existing caller keeps the repair. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 1 + sei-db/state_db/sc/memiavl/opts.go | 5 + .../seidb/operations/evm_logical_digest.go | 22 ++-- sei-db/wal/utils.go | 60 +--------- sei-db/wal/wal.go | 46 +++++++- sei-db/wal/wal_test.go | 103 ++++++++++-------- 6 files changed, 121 insertions(+), 116 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index b342cbfd76..0d1c8b88d1 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -213,6 +213,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { // Even in read-only mode we may need WAL replay to reconstruct non-snapshot versions. streamHandler, err := wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ WriteBufferSize: opts.AsyncCommitBuffer, + NoRepairOnOpen: opts.NoChangelogRepair, }) if err != nil { return nil, fmt.Errorf("failed to open changelog WAL: %w", err) diff --git a/sei-db/state_db/sc/memiavl/opts.go b/sei-db/state_db/sc/memiavl/opts.go index 9c4a6f5d31..9443444ddd 100644 --- a/sei-db/state_db/sc/memiavl/opts.go +++ b/sei-db/state_db/sc/memiavl/opts.go @@ -18,6 +18,11 @@ type Options struct { InitialVersion uint32 // ReadOnly opens the database in read-only mode ReadOnly bool + // NoChangelogRepair makes the open fail with wal.ErrCorrupt instead of + // repairing a torn changelog tail. A reader of a directory another process + // is writing sets it, because ReadOnly alone does not stop that repair from + // truncating a record the writer has committed. + NoChangelogRepair bool // InitialStores are the initial store names when initializing an empty instance InitialStores []string // ZeroCopy if true, get and iterator methods return slices pointing to mmaped blob files diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 576293286f..59cf53171e 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -14,7 +14,6 @@ import ( "sort" "github.com/sei-protocol/sei-chain/sei-db/common/keys" - "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" @@ -1110,23 +1109,20 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization } func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { - // memiavl.OpenDB repairs a torn changelog tail by truncating it, even under - // ReadOnly. On a live node that tail is usually a write in progress, so - // refuse the run instead of letting the open damage the source. - if err := wal.VerifyIntact(utils.GetChangelogPath(dbDir)); err != nil { + db, err := memiavl.OpenDB(height, memiavl.Options{ + Dir: dbDir, + ReadOnly: true, + ZeroCopy: true, + NoChangelogRepair: true, + }) + if err != nil { + // NoChangelogRepair turns the repair memiavl would otherwise perform into + // this error, so the source is untouched whichever case it was. if errors.Is(err, wal.ErrCorrupt) { return nil, fmt.Errorf("memiavl changelog tail is incomplete or changing; live WAL was not "+ "modified; rerun the command, and if the error persists after stopping seid, repair the "+ "WAL offline: %w", err) } - return nil, fmt.Errorf("verify memiavl changelog: %w", err) - } - db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: dbDir, - ReadOnly: true, - ZeroCopy: true, - }) - if err != nil { return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } return db, nil diff --git a/sei-db/wal/utils.go b/sei-db/wal/utils.go index 725024a198..e526c8bf4b 100644 --- a/sei-db/wal/utils.go +++ b/sei-db/wal/utils.go @@ -4,11 +4,9 @@ import ( "bytes" "encoding/binary" "errors" - "fmt" "math" "os" "path/filepath" - "strings" "unsafe" "github.com/tidwall/gjson" @@ -26,7 +24,7 @@ func GetLastIndex(dir string) (index uint64, err error) { rlog, err := open(dir, &wal.Options{ NoSync: true, NoCopy: true, - }) + }, false) if err != nil { return 0, err } @@ -34,62 +32,10 @@ func GetLastIndex(dir string) (index uint64, err error) { return rlog.LastIndex() } -// ErrCorrupt reports a log that cannot be read without repair. +// ErrCorrupt reports a log that cannot be read without repair. A caller that +// opened with Config.NoRepairOnOpen classifies the failure with it. var ErrCorrupt = wal.ErrCorrupt -// segmentNameLen is the length of a log segment file name. -const segmentNameLen = 20 - -// VerifyIntact reports whether the binary log in dir can be opened without -// repair. It returns ErrCorrupt when the tail segment ends mid-record or an -// interrupted truncation is still in progress, and never modifies dir. -// -// A reader on a live node calls this before it opens the log, because open -// repairs what it finds: truncateCorruptedTail cuts a torn tail, and tidwall -// completes an interrupted truncation by renaming or removing segments. On a -// live node a torn tail is usually a write in progress rather than lasting -// damage, so the caller reruns instead of repairing. -func VerifyIntact(dir string) error { - entries, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return fmt.Errorf("read wal dir %s: %w", dir, err) - } - - // os.ReadDir sorts by name, and segment names are zero-padded, so the last - // match is the tail segment open would truncate. - var tail string - for _, entry := range entries { - name := entry.Name() - if entry.IsDir() || len(name) < segmentNameLen { - continue - } - if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { - return fmt.Errorf("%w: truncation marker %s is present in %s", ErrCorrupt, name, dir) - } - tail = name - } - if tail == "" { - return nil - } - - path := filepath.Join(dir, tail) - data, err := os.ReadFile(filepath.Clean(path)) - if err != nil { - return fmt.Errorf("read wal segment %s: %w", path, err) - } - for pos := 0; pos < len(data); { - n, err := loadNextBinaryEntry(data[pos:]) - if err != nil { - return fmt.Errorf("%w: segment %s ends mid-record at offset %d", ErrCorrupt, path, pos) - } - pos += n - } - return nil -} - // truncateCorruptedTail truncates the corrupted tail func truncateCorruptedTail(path string, format wal.LogFormat) error { data, err := os.ReadFile(filepath.Clean(path)) diff --git a/sei-db/wal/wal.go b/sei-db/wal/wal.go index b13fd12af1..6fc206180d 100644 --- a/sei-db/wal/wal.go +++ b/sei-db/wal/wal.go @@ -94,6 +94,12 @@ type Config struct { // AllowEmpty permits removing all entries via TruncateAll. // When false (default), at least one entry must remain after truncation. AllowEmpty bool + + // NoRepairOnOpen makes the open fail with ErrCorrupt instead of repairing a + // torn tail. A reader of a log another process is writing sets it, because + // there a torn tail is usually that writer mid-append rather than lasting + // damage, and the repair would truncate a committed record. + NoRepairOnOpen bool } // NewWAL creates a new generic write-ahead log that persists entries. @@ -120,7 +126,7 @@ func NewWAL[T any]( NoSync: !config.FsyncEnabled, NoCopy: !config.DeepCopyEnabled, AllowEmpty: config.AllowEmpty, - }) + }, config.NoRepairOnOpen) if err != nil { return nil, err } @@ -542,12 +548,22 @@ func (walLog *WAL[T]) Close() error { return nil } -// open opens the replay log, try to truncate the corrupted tail if there's any -func open(dir string, opts *wal.Options) (*wal.Log, error) { +// open opens the replay log, try to truncate the corrupted tail if there's any. +// When noRepair is set it returns ErrCorrupt instead for both repairs the open +// would otherwise perform, and leaves dir byte for byte as it found it. +func open(dir string, opts *wal.Options, noRepair bool) (*wal.Log, error) { if opts == nil { opts = wal.DefaultOptions } + if noRepair { + if err := checkNoTruncationMarker(dir); err != nil { + return nil, err + } + } rlog, err := wal.Open(dir, opts) + if errors.Is(err, wal.ErrCorrupt) && noRepair { + return nil, err + } if errors.Is(err, wal.ErrCorrupt) { // try to truncate corrupted tail var fis []os.DirEntry @@ -576,6 +592,30 @@ func open(dir string, opts *wal.Options) (*wal.Log, error) { return rlog, err } +// checkNoTruncationMarker returns ErrCorrupt when dir holds a segment an +// interrupted truncation left behind, which wal.Open completes by renaming and +// removing segments without reporting anything. +// +// Checking before the open is sound here where it would not be for a torn tail: +// a marker persists until something completes that truncation, so finding none +// means the open below will not find one either. +func checkNoTruncationMarker(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("read wal dir %s: %w", dir, err) + } + for _, entry := range entries { + name := entry.Name() + if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { + return fmt.Errorf("%w: truncation marker %s is present in %s", ErrCorrupt, name, dir) + } + } + return nil +} + // The main loop doing work in the background. func (walLog *WAL[T]) mainLoop() { diff --git a/sei-db/wal/wal_test.go b/sei-db/wal/wal_test.go index a475c69f9b..0ad542bf73 100644 --- a/sei-db/wal/wal_test.go +++ b/sei-db/wal/wal_test.go @@ -58,7 +58,7 @@ func TestOpenAndCorruptedTail(t *testing.T) { _, err = wal.Open(dir, opts) require.Equal(t, wal.ErrCorrupt, err) - log, err := open(dir, opts) + log, err := open(dir, opts, false) require.NoError(t, err) lastIndex, err := log.LastIndex() @@ -68,76 +68,93 @@ func TestOpenAndCorruptedTail(t *testing.T) { } } -func TestVerifyIntactAcceptsCompleteLog(t *testing.T) { - dir := writeTestSegment(t, appendBinaryEntry(appendBinaryEntry(nil, []byte("one")), []byte("two"))) +// TestOpenNoRepairRefusesTheTornTailOpenWouldTruncate pins both halves of the +// option: the tail it refuses is one the repairing open truncates in place. +func TestOpenNoRepairRefusesTheTornTailOpenWouldTruncate(t *testing.T) { + dir := writeTornTestLog(t) before := snapshotDir(t, dir) - require.NoError(t, VerifyIntact(dir)) - require.Equal(t, before, snapshotDir(t, dir)) -} - -func TestVerifyIntactRejectsTornTail(t *testing.T) { - torn := appendBinaryEntry(appendBinaryEntry(nil, []byte("complete")), []byte("torn")) - dir := writeTestSegment(t, torn[:len(torn)-1]) - before := snapshotDir(t, dir) - - err := VerifyIntact(dir) + log, err := open(dir, nil, true) + require.Nil(t, log) require.ErrorIs(t, err, ErrCorrupt) - require.Contains(t, err.Error(), "ends mid-record") require.Equal(t, before, snapshotDir(t, dir)) + + log, err = open(dir, nil, false) + require.NoError(t, err) + require.NoError(t, log.Close()) + require.NotEqual(t, before, snapshotDir(t, dir)) } -func TestVerifyIntactRejectsInterruptedTruncation(t *testing.T) { - dir := writeTestSegment(t, appendBinaryEntry(nil, []byte("complete"))) +// TestOpenNoRepairRefusesTheTruncationMarkerOpenCompletes covers the second +// repair, which wal.Open performs without reporting an error. +func TestOpenNoRepairRefusesTheTruncationMarkerOpenCompletes(t *testing.T) { + dir := t.TempDir() + log, err := open(dir, nil, false) + require.NoError(t, err) + require.NoError(t, log.Write(1, []byte("entry"))) + require.NoError(t, log.Close()) marker := filepath.Join(dir, "00000000000000000002.START") require.NoError(t, os.WriteFile(marker, appendBinaryEntry(nil, []byte("moved")), 0o600)) before := snapshotDir(t, dir) - err := VerifyIntact(dir) + log, err = open(dir, nil, true) + require.Nil(t, log) require.ErrorIs(t, err, ErrCorrupt) require.Contains(t, err.Error(), "truncation marker") require.Equal(t, before, snapshotDir(t, dir)) -} - -func TestVerifyIntactAcceptsMissingOrEmptyLog(t *testing.T) { - missing := filepath.Join(t.TempDir(), "changelog") - require.NoError(t, VerifyIntact(missing)) - require.NoFileExists(t, missing) - require.NoError(t, VerifyIntact(t.TempDir())) + log, err = open(dir, nil, false) + require.NoError(t, err) + require.NoError(t, log.Close()) + require.NotEqual(t, before, snapshotDir(t, dir)) } -// TestVerifyIntactRejectsWhatOpenTruncates pins the reason VerifyIntact exists: -// the tail it rejects is one open repairs in place. -func TestVerifyIntactRejectsWhatOpenTruncates(t *testing.T) { +// TestOpenNoRepairAcceptsACompleteLog is the positive control: the option only +// refuses a log the open would have repaired. +func TestOpenNoRepairAcceptsACompleteLog(t *testing.T) { dir := t.TempDir() - log, err := open(dir, nil) + log, err := open(dir, nil, false) require.NoError(t, err) require.NoError(t, log.Write(1, []byte("entry"))) require.NoError(t, log.Close()) - require.NoError(t, VerifyIntact(dir)) + before := snapshotDir(t, dir) - segment := filepath.Join(dir, "00000000000000000001") - data, err := os.ReadFile(filepath.Clean(segment)) + log, err = open(dir, nil, true) + require.NoError(t, err) + last, err := log.LastIndex() require.NoError(t, err) - require.NoError(t, os.WriteFile(segment, append(data, 0x08), 0o600)) + require.Equal(t, uint64(1), last) + require.NoError(t, log.Close()) + require.Equal(t, before, snapshotDir(t, dir)) +} + +// TestNewWALNoRepairOnOpenReachesTheOpen pins that the config field is wired +// through, since only the digest tool sets it today. +func TestNewWALNoRepairOnOpenReachesTheOpen(t *testing.T) { + dir := writeTornTestLog(t) before := snapshotDir(t, dir) - require.ErrorIs(t, VerifyIntact(dir), ErrCorrupt) + changelog, err := NewChangelogWAL(dir, Config{NoRepairOnOpen: true}) + require.Nil(t, changelog) + require.ErrorIs(t, err, ErrCorrupt) require.Equal(t, before, snapshotDir(t, dir)) - - log, err = open(dir, nil) - require.NoError(t, err) - require.NoError(t, log.Close()) - require.NotEqual(t, before, snapshotDir(t, dir)) } -// writeTestSegment creates a log directory holding data as its only segment, -// and returns the directory. -func writeTestSegment(t *testing.T, data []byte) string { +// writeTornTestLog creates a log directory whose only segment ends mid-record, +// the way a reader sees a writer mid-append, and returns the directory. +func writeTornTestLog(t *testing.T) string { t.Helper() dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "00000000000000000001"), data, 0o600)) + log, err := open(dir, nil, false) + require.NoError(t, err) + require.NoError(t, log.Write(1, []byte("entry"))) + require.NoError(t, log.Close()) + + segment := filepath.Join(dir, "00000000000000000001") + data, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + torn := appendBinaryEntry(data, []byte("torn")) + require.NoError(t, os.WriteFile(segment, torn[:len(torn)-1], 0o600)) return dir } @@ -271,7 +288,7 @@ func TestOpenWithNilOptions(t *testing.T) { dir := t.TempDir() // Test that open function handles nil options correctly - log, err := open(dir, nil) + log, err := open(dir, nil, false) require.NoError(t, err) require.NotNil(t, log) From 431de98b91dcfe9c91d720289cc22b6e54db2e7d Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 28 Aug 2026 15:18:04 -0400 Subject: [PATCH 12/13] fix(seidb): refuse a digest replay of a directory a writer holds The marker check could not hold. tidwall writes a .START segment partway through every successful TruncateFront, not only an interrupted one, so it appears after the check exactly as a torn tail does. Completing that truncation underneath the writer makes its own remove fail, and tidwall sets l.corrupt on any error past that point, so the node's appends fail until it restarts. No pre-open check fixes this, so exclude the writer instead. Options.RequireExclusive takes the LOCK under ReadOnly too, and the digest refuses a directory seid has open, offering snapshot mode. Under that exclusion a torn tail is damage rather than an append in flight, so the message asks for an offline repair rather than a rerun, and reading the directory before the open is sound because nothing can change it. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 13 ++++++- sei-db/state_db/sc/memiavl/filelock.go | 4 +++ sei-db/state_db/sc/memiavl/opts.go | 11 ++++-- .../seidb/operations/evm_logical_digest.go | 16 ++++++--- .../cmd/seidb/operations/memiavl_open_test.go | 35 +++++++++++++++++-- sei-db/wal/wal.go | 23 +++++++----- 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 0d1c8b88d1..54c38f2cf3 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -169,6 +169,15 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { err error fileLock FileLock ) + // A failed open hands back no DB, so nothing else can release the lock. The + // process usually exits either way, but a caller that recovers and retries + // would otherwise lock itself out. + defer func() { + if _err != nil && fileLock != nil { + _ = fileLock.Unlock() + _ = fileLock.Destroy() + } + }() if err := opts.Validate(); err != nil { return nil, fmt.Errorf("invalid commit store options: %w", err) } @@ -179,12 +188,14 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { } } - if !opts.ReadOnly { + if !opts.ReadOnly || opts.RequireExclusive { fileLock, err = LockFile(filepath.Join(opts.Dir, LockFileName)) if err != nil { return nil, fmt.Errorf("fail to lock db: %w", err) } + } + if !opts.ReadOnly { // cleanup any temporary directories left by interrupted snapshot rewrite if err := removeTmpDirs(opts.Dir); err != nil { return nil, fmt.Errorf("fail to cleanup tmp directories: %w", err) diff --git a/sei-db/state_db/sc/memiavl/filelock.go b/sei-db/state_db/sc/memiavl/filelock.go index 6becb39b82..7ff49cabfd 100644 --- a/sei-db/state_db/sc/memiavl/filelock.go +++ b/sei-db/state_db/sc/memiavl/filelock.go @@ -11,6 +11,10 @@ type FileLock interface { Destroy() error } +// ErrLocked reports that another process holds the lock, which for a database +// directory means a writer has it open. +var ErrLocked = filelock.ErrLocked + func LockFile(fname string) (FileLock, error) { path, err := filepath.Abs(fname) if err != nil { diff --git a/sei-db/state_db/sc/memiavl/opts.go b/sei-db/state_db/sc/memiavl/opts.go index 9443444ddd..fe75af75d1 100644 --- a/sei-db/state_db/sc/memiavl/opts.go +++ b/sei-db/state_db/sc/memiavl/opts.go @@ -19,10 +19,15 @@ type Options struct { // ReadOnly opens the database in read-only mode ReadOnly bool // NoChangelogRepair makes the open fail with wal.ErrCorrupt instead of - // repairing a torn changelog tail. A reader of a directory another process - // is writing sets it, because ReadOnly alone does not stop that repair from - // truncating a record the writer has committed. + // repairing the changelog. Set it with RequireExclusive, which is what makes + // the refusal meaningful: without a writer excluded, the conditions it + // refuses also occur transiently and the repair still races the writer. NoChangelogRepair bool + // RequireExclusive takes the directory's lock even under ReadOnly, so the + // open fails with ErrLocked while a writer has it. A read-only tool that + // must not disturb the directory sets it, because the changelog opener + // mutates the directory whether or not the DB API can write. + RequireExclusive bool // InitialStores are the initial store names when initializing an empty instance InitialStores []string // ZeroCopy if true, get and iterator methods return slices pointing to mmaped blob files diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 59cf53171e..d3092dfa8c 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -1114,14 +1114,20 @@ func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) ReadOnly: true, ZeroCopy: true, NoChangelogRepair: true, + RequireExclusive: true, }) if err != nil { - // NoChangelogRepair turns the repair memiavl would otherwise perform into - // this error, so the source is untouched whichever case it was. + if errors.Is(err, memiavl.ErrLocked) { + return nil, fmt.Errorf("another process has %s open, and replaying the changelog of a "+ + "directory being written would let the changelog opener truncate a record that writer "+ + "has committed; stop seid and rerun, or use --memiavl-open-mode snapshot: %w", dbDir, err) + } + // RequireExclusive rules out a writer, so a torn tail here is damage left + // behind rather than an append in flight. NoChangelogRepair reported it + // instead of truncating it. if errors.Is(err, wal.ErrCorrupt) { - return nil, fmt.Errorf("memiavl changelog tail is incomplete or changing; live WAL was not "+ - "modified; rerun the command, and if the error persists after stopping seid, repair the "+ - "WAL offline: %w", err) + return nil, fmt.Errorf("memiavl changelog is damaged and was left unmodified; repair it "+ + "offline, or use --memiavl-open-mode snapshot: %w", err) } return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index e3d752186e..8391d8bda7 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -11,6 +11,8 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/keys" "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" + "github.com/sei-protocol/sei-chain/sei-db/wal" ) func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing.T) { @@ -34,14 +36,43 @@ func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing. _, err = openMemiAVLReplayReadOnly(dbDir, 0) require.Error(t, err) - require.Contains(t, err.Error(), "live WAL was not modified") - require.Contains(t, err.Error(), "rerun the command") + require.ErrorIs(t, err, wal.ErrCorrupt) + require.Contains(t, err.Error(), "left unmodified") after, readErr := os.ReadFile(filepath.Clean(segment)) require.NoError(t, readErr) require.Equal(t, before, after) } +// TestOpenMemiAVLReplayReadOnlyRefusesADirectoryAWriterHasOpen covers what a +// pre-open check cannot: while a writer holds the directory, the changelog +// opener can complete a truncation that writer is partway through, which leaves +// its log marked corrupt. Replaying a live directory is refused outright. +func TestOpenMemiAVLReplayReadOnlyRefusesADirectoryAWriterHasOpen(t *testing.T) { + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}}, + }})) + _, err := store.Commit(store.Version() + 1) + require.NoError(t, err) + + // The store is still open, holding the lock the way a running seid does. + dbDir := utils.GetCosmosSCStorePath(homeDir) + db, err := openMemiAVLReplayReadOnly(dbDir, 0) + require.Nil(t, db) + require.ErrorIs(t, err, memiavl.ErrLocked) + require.Contains(t, err.Error(), "stop seid and rerun") + require.Contains(t, err.Error(), "--memiavl-open-mode snapshot") + + // The same directory opens once that writer releases it. + require.NoError(t, store.Close()) + db, err = openMemiAVLReplayReadOnly(dbDir, 0) + require.NoError(t, err) + require.NoError(t, db.Close()) +} + // TestOpenMemiAVLReplayReadOnlyAcceptsACompleteChangelog is the positive // control for the pre-flight check: a changelog a real store wrote and closed // opens and replays as it did before the check existed. diff --git a/sei-db/wal/wal.go b/sei-db/wal/wal.go index 6fc206180d..c6d0b4ac46 100644 --- a/sei-db/wal/wal.go +++ b/sei-db/wal/wal.go @@ -95,10 +95,14 @@ type Config struct { // When false (default), at least one entry must remain after truncation. AllowEmpty bool - // NoRepairOnOpen makes the open fail with ErrCorrupt instead of repairing a - // torn tail. A reader of a log another process is writing sets it, because - // there a torn tail is usually that writer mid-append rather than lasting - // damage, and the repair would truncate a committed record. + // NoRepairOnOpen makes the open fail with ErrCorrupt instead of repairing the + // log. It refuses a tail ending mid-record, and a directory holding the + // .START or .END segment an interrupted truncation left behind. + // + // The caller must already exclude a concurrent writer. Both conditions also + // occur transiently while a writer appends or truncates, so against a live + // log this reports damage that is not there, and cannot prevent the open from + // completing a truncation that writer is in the middle of. NoRepairOnOpen bool } @@ -550,7 +554,8 @@ func (walLog *WAL[T]) Close() error { // open opens the replay log, try to truncate the corrupted tail if there's any. // When noRepair is set it returns ErrCorrupt instead for both repairs the open -// would otherwise perform, and leaves dir byte for byte as it found it. +// would otherwise perform, leaving dir as it found it, which holds for a caller +// that has excluded writers per Config.NoRepairOnOpen. func open(dir string, opts *wal.Options, noRepair bool) (*wal.Log, error) { if opts == nil { opts = wal.DefaultOptions @@ -596,9 +601,11 @@ func open(dir string, opts *wal.Options, noRepair bool) (*wal.Log, error) { // interrupted truncation left behind, which wal.Open completes by renaming and // removing segments without reporting anything. // -// Checking before the open is sound here where it would not be for a torn tail: -// a marker persists until something completes that truncation, so finding none -// means the open below will not find one either. +// It reads dir before the open, which is sound only for a caller that has +// excluded writers, as Config.NoRepairOnOpen requires. A writer creates the same +// segment partway through every successful truncation, so against a live log +// this both reports damage that is not there and misses the marker that appears +// after it looks. func checkNoTruncationMarker(dir string) error { entries, err := os.ReadDir(dir) if err != nil { From 9c8aac8eea166974c47d6eabecd1ce7a3ea706da Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 28 Aug 2026 15:29:43 -0400 Subject: [PATCH 13/13] refactor(seidb): drop the changelog no-repair option RequireExclusive made it redundant. With no writer on the directory, a torn tail or a leftover truncation marker is damage rather than a race, and the repair for it is the one seid performs at its next start, so refusing it here changes nothing about what the node replays. It only withheld a diagnostic. Restores sei-db/wal to its state on main, so the change no longer touches that package: the refusal is the lock, and the lock alone. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 1 - sei-db/state_db/sc/memiavl/opts.go | 5 - .../seidb/operations/evm_logical_digest.go | 32 ++--- .../cmd/seidb/operations/memiavl_open_test.go | 69 ++-------- sei-db/wal/utils.go | 6 +- sei-db/wal/wal.go | 53 +------- sei-db/wal/wal_test.go | 121 +----------------- 7 files changed, 28 insertions(+), 259 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 54c38f2cf3..df0fecfe57 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -224,7 +224,6 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { // Even in read-only mode we may need WAL replay to reconstruct non-snapshot versions. streamHandler, err := wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ WriteBufferSize: opts.AsyncCommitBuffer, - NoRepairOnOpen: opts.NoChangelogRepair, }) if err != nil { return nil, fmt.Errorf("failed to open changelog WAL: %w", err) diff --git a/sei-db/state_db/sc/memiavl/opts.go b/sei-db/state_db/sc/memiavl/opts.go index fe75af75d1..27f7b92fb2 100644 --- a/sei-db/state_db/sc/memiavl/opts.go +++ b/sei-db/state_db/sc/memiavl/opts.go @@ -18,11 +18,6 @@ type Options struct { InitialVersion uint32 // ReadOnly opens the database in read-only mode ReadOnly bool - // NoChangelogRepair makes the open fail with wal.ErrCorrupt instead of - // repairing the changelog. Set it with RequireExclusive, which is what makes - // the refusal meaningful: without a writer excluded, the conditions it - // refuses also occur transiently and the repair still races the writer. - NoChangelogRepair bool // RequireExclusive takes the directory's lock even under ReadOnly, so the // open fails with ErrLocked while a writer has it. A read-only tool that // must not disturb the directory sets it, because the changelog opener diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index d3092dfa8c..ad0c930977 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -20,7 +20,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/migration" - "github.com/sei-protocol/sei-chain/sei-db/wal" "github.com/spf13/cobra" ) @@ -85,12 +84,11 @@ const ( // - replay (SLOW): opens a read-only DB, replays the changelog up to // --height, then walks the in-memory/mmap tree. Roughly an order of // magnitude slower than snapshot (changelog replay + per-leaf tree walk -// instead of a sequential file read). It refuses to run on a changelog whose -// tail a live writer is still filling, and asks the operator to rerun, -// because opening such a changelog would truncate that tail. Use it only -// when no snapshot exists at the target height — e.g. nodes whose snapshot -// rewrite lags the tip, so an arbitrary comparison height has no -// snapshot- on disk. +// instead of a sequential file read). It requires the node stopped, because +// opening a changelog its writer still holds lets the opener repair, and +// therefore damage, that changelog. Use it only when no snapshot exists at +// the target height — e.g. nodes whose snapshot rewrite lags the tip, so an +// arbitrary comparison height has no snapshot- on disk. // // The flatkv side is always a pebble WAL-replay-to-height and is fast // regardless. So when comparing across nodes, pick a height that is an existing @@ -1110,24 +1108,16 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: dbDir, - ReadOnly: true, - ZeroCopy: true, - NoChangelogRepair: true, - RequireExclusive: true, + Dir: dbDir, + ReadOnly: true, + ZeroCopy: true, + RequireExclusive: true, }) if err != nil { if errors.Is(err, memiavl.ErrLocked) { return nil, fmt.Errorf("another process has %s open, and replaying the changelog of a "+ - "directory being written would let the changelog opener truncate a record that writer "+ - "has committed; stop seid and rerun, or use --memiavl-open-mode snapshot: %w", dbDir, err) - } - // RequireExclusive rules out a writer, so a torn tail here is damage left - // behind rather than an append in flight. NoChangelogRepair reported it - // instead of truncating it. - if errors.Is(err, wal.ErrCorrupt) { - return nil, fmt.Errorf("memiavl changelog is damaged and was left unmodified; repair it "+ - "offline, or use --memiavl-open-mode snapshot: %w", err) + "directory being written lets the changelog opener truncate a record that writer has "+ + "committed; stop seid and rerun, or use --memiavl-open-mode snapshot: %w", dbDir, err) } return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index 8391d8bda7..c87398e115 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -1,9 +1,6 @@ package operations import ( - "os" - "path/filepath" - "sort" "testing" "github.com/stretchr/testify/require" @@ -12,42 +9,13 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" - "github.com/sei-protocol/sei-chain/sei-db/wal" ) -func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutRepair(t *testing.T) { - homeDir := t.TempDir() - store := newTestMemiavlStore(t, homeDir) - require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ - Name: keys.EVMStoreKey, - Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}}, - }})) - _, err := store.Commit(store.Version() + 1) - require.NoError(t, err) - require.NoError(t, store.Close()) - - dbDir := utils.GetCosmosSCStorePath(homeDir) - segment := lastOperationsMemiAVLWALSegment(t, dbDir) - committed, err := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, err) - require.NotEmpty(t, committed) - before := committed[:len(committed)-1] - require.NoError(t, os.WriteFile(filepath.Clean(segment), before, 0o600)) - - _, err = openMemiAVLReplayReadOnly(dbDir, 0) - require.Error(t, err) - require.ErrorIs(t, err, wal.ErrCorrupt) - require.Contains(t, err.Error(), "left unmodified") - - after, readErr := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, readErr) - require.Equal(t, before, after) -} - -// TestOpenMemiAVLReplayReadOnlyRefusesADirectoryAWriterHasOpen covers what a -// pre-open check cannot: while a writer holds the directory, the changelog -// opener can complete a truncation that writer is partway through, which leaves -// its log marked corrupt. Replaying a live directory is refused outright. +// TestOpenMemiAVLReplayReadOnlyRefusesADirectoryAWriterHasOpen covers what no +// check of the directory can: the changelog opener repairs a torn tail and +// completes an interrupted truncation, and both conditions occur transiently +// while a writer appends or truncates. Replaying a directory a writer holds is +// refused instead. func TestOpenMemiAVLReplayReadOnlyRefusesADirectoryAWriterHasOpen(t *testing.T) { homeDir := t.TempDir() store := newTestMemiavlStore(t, homeDir) @@ -66,17 +34,18 @@ func TestOpenMemiAVLReplayReadOnlyRefusesADirectoryAWriterHasOpen(t *testing.T) require.Contains(t, err.Error(), "stop seid and rerun") require.Contains(t, err.Error(), "--memiavl-open-mode snapshot") - // The same directory opens once that writer releases it. + // The same directory opens once that writer releases it, so the refusal + // tracks the writer rather than something the first open left behind. require.NoError(t, store.Close()) db, err = openMemiAVLReplayReadOnly(dbDir, 0) require.NoError(t, err) require.NoError(t, db.Close()) } -// TestOpenMemiAVLReplayReadOnlyAcceptsACompleteChangelog is the positive -// control for the pre-flight check: a changelog a real store wrote and closed -// opens and replays as it did before the check existed. -func TestOpenMemiAVLReplayReadOnlyAcceptsACompleteChangelog(t *testing.T) { +// TestOpenMemiAVLReplayReadOnlyReplaysAStoppedNode is the positive control: the +// lock is the only thing the refusal adds, so a released directory replays as +// it did before. +func TestOpenMemiAVLReplayReadOnlyReplaysAStoppedNode(t *testing.T) { homeDir := t.TempDir() store := newTestMemiavlStore(t, homeDir) for nonce := uint64(1); nonce <= 3; nonce++ { @@ -94,19 +63,3 @@ func TestOpenMemiAVLReplayReadOnlyAcceptsACompleteChangelog(t *testing.T) { defer func() { _ = db.Close() }() require.Equal(t, int64(3), db.Version()) } - -func lastOperationsMemiAVLWALSegment(t *testing.T, dbDir string) string { - t.Helper() - changelogDir := utils.GetChangelogPath(dbDir) - entries, err := os.ReadDir(changelogDir) - require.NoError(t, err) - var names []string - for _, entry := range entries { - if !entry.IsDir() && len(entry.Name()) == 20 { - names = append(names, entry.Name()) - } - } - require.NotEmpty(t, names) - sort.Strings(names) - return filepath.Join(changelogDir, names[len(names)-1]) -} diff --git a/sei-db/wal/utils.go b/sei-db/wal/utils.go index e526c8bf4b..a33cf5f33d 100644 --- a/sei-db/wal/utils.go +++ b/sei-db/wal/utils.go @@ -24,7 +24,7 @@ func GetLastIndex(dir string) (index uint64, err error) { rlog, err := open(dir, &wal.Options{ NoSync: true, NoCopy: true, - }, false) + }) if err != nil { return 0, err } @@ -32,10 +32,6 @@ func GetLastIndex(dir string) (index uint64, err error) { return rlog.LastIndex() } -// ErrCorrupt reports a log that cannot be read without repair. A caller that -// opened with Config.NoRepairOnOpen classifies the failure with it. -var ErrCorrupt = wal.ErrCorrupt - // truncateCorruptedTail truncates the corrupted tail func truncateCorruptedTail(path string, format wal.LogFormat) error { data, err := os.ReadFile(filepath.Clean(path)) diff --git a/sei-db/wal/wal.go b/sei-db/wal/wal.go index c6d0b4ac46..b13fd12af1 100644 --- a/sei-db/wal/wal.go +++ b/sei-db/wal/wal.go @@ -94,16 +94,6 @@ type Config struct { // AllowEmpty permits removing all entries via TruncateAll. // When false (default), at least one entry must remain after truncation. AllowEmpty bool - - // NoRepairOnOpen makes the open fail with ErrCorrupt instead of repairing the - // log. It refuses a tail ending mid-record, and a directory holding the - // .START or .END segment an interrupted truncation left behind. - // - // The caller must already exclude a concurrent writer. Both conditions also - // occur transiently while a writer appends or truncates, so against a live - // log this reports damage that is not there, and cannot prevent the open from - // completing a truncation that writer is in the middle of. - NoRepairOnOpen bool } // NewWAL creates a new generic write-ahead log that persists entries. @@ -130,7 +120,7 @@ func NewWAL[T any]( NoSync: !config.FsyncEnabled, NoCopy: !config.DeepCopyEnabled, AllowEmpty: config.AllowEmpty, - }, config.NoRepairOnOpen) + }) if err != nil { return nil, err } @@ -552,23 +542,12 @@ func (walLog *WAL[T]) Close() error { return nil } -// open opens the replay log, try to truncate the corrupted tail if there's any. -// When noRepair is set it returns ErrCorrupt instead for both repairs the open -// would otherwise perform, leaving dir as it found it, which holds for a caller -// that has excluded writers per Config.NoRepairOnOpen. -func open(dir string, opts *wal.Options, noRepair bool) (*wal.Log, error) { +// open opens the replay log, try to truncate the corrupted tail if there's any +func open(dir string, opts *wal.Options) (*wal.Log, error) { if opts == nil { opts = wal.DefaultOptions } - if noRepair { - if err := checkNoTruncationMarker(dir); err != nil { - return nil, err - } - } rlog, err := wal.Open(dir, opts) - if errors.Is(err, wal.ErrCorrupt) && noRepair { - return nil, err - } if errors.Is(err, wal.ErrCorrupt) { // try to truncate corrupted tail var fis []os.DirEntry @@ -597,32 +576,6 @@ func open(dir string, opts *wal.Options, noRepair bool) (*wal.Log, error) { return rlog, err } -// checkNoTruncationMarker returns ErrCorrupt when dir holds a segment an -// interrupted truncation left behind, which wal.Open completes by renaming and -// removing segments without reporting anything. -// -// It reads dir before the open, which is sound only for a caller that has -// excluded writers, as Config.NoRepairOnOpen requires. A writer creates the same -// segment partway through every successful truncation, so against a live log -// this both reports damage that is not there and misses the marker that appears -// after it looks. -func checkNoTruncationMarker(dir string) error { - entries, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return fmt.Errorf("read wal dir %s: %w", dir, err) - } - for _, entry := range entries { - name := entry.Name() - if strings.HasSuffix(name, ".START") || strings.HasSuffix(name, ".END") { - return fmt.Errorf("%w: truncation marker %s is present in %s", ErrCorrupt, name, dir) - } - } - return nil -} - // The main loop doing work in the background. func (walLog *WAL[T]) mainLoop() { diff --git a/sei-db/wal/wal_test.go b/sei-db/wal/wal_test.go index 0ad542bf73..a83d5fded3 100644 --- a/sei-db/wal/wal_test.go +++ b/sei-db/wal/wal_test.go @@ -1,7 +1,6 @@ package wal import ( - "encoding/binary" "fmt" "os" "path/filepath" @@ -58,7 +57,7 @@ func TestOpenAndCorruptedTail(t *testing.T) { _, err = wal.Open(dir, opts) require.Equal(t, wal.ErrCorrupt, err) - log, err := open(dir, opts, false) + log, err := open(dir, opts) require.NoError(t, err) lastIndex, err := log.LastIndex() @@ -68,122 +67,6 @@ func TestOpenAndCorruptedTail(t *testing.T) { } } -// TestOpenNoRepairRefusesTheTornTailOpenWouldTruncate pins both halves of the -// option: the tail it refuses is one the repairing open truncates in place. -func TestOpenNoRepairRefusesTheTornTailOpenWouldTruncate(t *testing.T) { - dir := writeTornTestLog(t) - before := snapshotDir(t, dir) - - log, err := open(dir, nil, true) - require.Nil(t, log) - require.ErrorIs(t, err, ErrCorrupt) - require.Equal(t, before, snapshotDir(t, dir)) - - log, err = open(dir, nil, false) - require.NoError(t, err) - require.NoError(t, log.Close()) - require.NotEqual(t, before, snapshotDir(t, dir)) -} - -// TestOpenNoRepairRefusesTheTruncationMarkerOpenCompletes covers the second -// repair, which wal.Open performs without reporting an error. -func TestOpenNoRepairRefusesTheTruncationMarkerOpenCompletes(t *testing.T) { - dir := t.TempDir() - log, err := open(dir, nil, false) - require.NoError(t, err) - require.NoError(t, log.Write(1, []byte("entry"))) - require.NoError(t, log.Close()) - marker := filepath.Join(dir, "00000000000000000002.START") - require.NoError(t, os.WriteFile(marker, appendBinaryEntry(nil, []byte("moved")), 0o600)) - before := snapshotDir(t, dir) - - log, err = open(dir, nil, true) - require.Nil(t, log) - require.ErrorIs(t, err, ErrCorrupt) - require.Contains(t, err.Error(), "truncation marker") - require.Equal(t, before, snapshotDir(t, dir)) - - log, err = open(dir, nil, false) - require.NoError(t, err) - require.NoError(t, log.Close()) - require.NotEqual(t, before, snapshotDir(t, dir)) -} - -// TestOpenNoRepairAcceptsACompleteLog is the positive control: the option only -// refuses a log the open would have repaired. -func TestOpenNoRepairAcceptsACompleteLog(t *testing.T) { - dir := t.TempDir() - log, err := open(dir, nil, false) - require.NoError(t, err) - require.NoError(t, log.Write(1, []byte("entry"))) - require.NoError(t, log.Close()) - before := snapshotDir(t, dir) - - log, err = open(dir, nil, true) - require.NoError(t, err) - last, err := log.LastIndex() - require.NoError(t, err) - require.Equal(t, uint64(1), last) - require.NoError(t, log.Close()) - require.Equal(t, before, snapshotDir(t, dir)) -} - -// TestNewWALNoRepairOnOpenReachesTheOpen pins that the config field is wired -// through, since only the digest tool sets it today. -func TestNewWALNoRepairOnOpenReachesTheOpen(t *testing.T) { - dir := writeTornTestLog(t) - before := snapshotDir(t, dir) - - changelog, err := NewChangelogWAL(dir, Config{NoRepairOnOpen: true}) - require.Nil(t, changelog) - require.ErrorIs(t, err, ErrCorrupt) - require.Equal(t, before, snapshotDir(t, dir)) -} - -// writeTornTestLog creates a log directory whose only segment ends mid-record, -// the way a reader sees a writer mid-append, and returns the directory. -func writeTornTestLog(t *testing.T) string { - t.Helper() - dir := t.TempDir() - log, err := open(dir, nil, false) - require.NoError(t, err) - require.NoError(t, log.Write(1, []byte("entry"))) - require.NoError(t, log.Close()) - - segment := filepath.Join(dir, "00000000000000000001") - data, err := os.ReadFile(filepath.Clean(segment)) - require.NoError(t, err) - torn := appendBinaryEntry(data, []byte("torn")) - require.NoError(t, os.WriteFile(segment, torn[:len(torn)-1], 0o600)) - return dir -} - -// snapshotDir returns the contents of every file in dir, keyed by name. -func snapshotDir(t *testing.T, dir string) map[string][]byte { - t.Helper() - entries, err := os.ReadDir(dir) - require.NoError(t, err) - files := make(map[string][]byte, len(entries)) - for _, entry := range entries { - if entry.IsDir() { - continue - } - data, readErr := os.ReadFile(filepath.Clean(filepath.Join(dir, entry.Name()))) - require.NoError(t, readErr) - files[entry.Name()] = data - } - return files -} - -// appendBinaryEntry appends payload to data in the binary log framing of a size -// varint followed by the payload. -func appendBinaryEntry(data []byte, payload []byte) []byte { - var size [binary.MaxVarintLen64]byte - n := binary.PutUvarint(size[:], uint64(len(payload))) - data = append(data, size[:n]...) - return append(data, payload...) -} - func TestReplay(t *testing.T) { changelog := prepareTestData(t) var total = 0 @@ -288,7 +171,7 @@ func TestOpenWithNilOptions(t *testing.T) { dir := t.TempDir() // Test that open function handles nil options correctly - log, err := open(dir, nil, false) + log, err := open(dir, nil) require.NoError(t, err) require.NotNil(t, log)