diff --git a/cmd/billyfuzz/main.go b/cmd/billyfuzz/main.go index 4a524a5..6dbb7f9 100644 --- a/cmd/billyfuzz/main.go +++ b/cmd/billyfuzz/main.go @@ -3,7 +3,6 @@ package main import ( crand "crypto/rand" "crypto/sha256" - "encoding/hex" "fmt" "math/rand" "os" @@ -68,18 +67,17 @@ func doOpenDb(ctx *cli.Context, onData billy.OnDataFn) (billy.Database, error) { func doFuzz(ctx *cli.Context) error { var ( - hasher = sha256.New() hashes = make(map[uint64]string) onData = func(key uint64, size uint32, data []byte) { if verbose { fmt.Printf("init key %x val %x\n", key, data[:20]) } - hasher.Reset() - hashes[key] = hex.EncodeToString(hasher.Sum(data)) + hashes[key] = fmt.Sprintf("%x", sha256.Sum256(data)) } db, err = doOpenDb(ctx, onData) ) if err != nil { + fmt.Printf("Error opening db: %v\n", err) return err } var ( @@ -109,8 +107,7 @@ func doFuzz(ctx *cli.Context) error { l := int(min) + rand.Intn(int(max-min)) data := make([]byte, l) _, _ = crand.Read(data) - hasher.Reset() - sum := hex.EncodeToString(hasher.Sum(data)) + sum := fmt.Sprintf("%x", sha256.Sum256(data)) key, err := db.Put(data) if err != nil { panic(err) @@ -126,19 +123,23 @@ func doFuzz(ctx *cli.Context) error { for key, want = range hashes { break } - //fmt.Printf("Checking %d bytes data at key %d\n", len(want), key) data, err := db.Get(key) if err != nil { + fmt.Printf("Checking data at key %d\n", key) panic(err) } // check the data - hasher.Reset() - have := hex.EncodeToString(hasher.Sum(data)) + have := fmt.Sprintf("%x", sha256.Sum256(data)) if have != want { - panic(fmt.Sprintf("key %v\nhave %v\n, want %v\n", key, have, want)) + fmt.Printf("key %v\nhave %d bytes, hash %v\n, want %v\n", key, + len(data), have, want) + panic("GET failure") } case 2: // DELETE var key uint64 + if len(hashes) == 0 { + continue + } for key = range hashes { break } diff --git a/db.go b/db.go index 8c8f867..15d453c 100644 --- a/db.go +++ b/db.go @@ -86,9 +86,20 @@ type database struct { } type Options struct { - Path string + // Path is the path to where the billy-files are stored. An empty value + // means 'memory-mode', in which nothing is stored to disk. + Path string + // ShelfFileSize is the maximum size of files used. A value of zero means that + // there is no maximum. + ShelfFileSize uint64 + // ShelfFileCount is the amount of files to open for each shelf. The total + // storage capacity of any shelf is thus `ShelfFileSize * ShelfFileCount`. + // If `ShelfFileSize==0`, then this value is not used. + ShelfFileCount int + // Readonly means that the db cannot be used for writing. Readonly bool - Snappy bool // unused for now + // Snappy is not used + Snappy bool } // Open opens a (new or existing) database, with configurable limits. The given @@ -113,7 +124,8 @@ func Open(opts Options, slotSizeFn SlotSizeFn, onData OnDataFn) (Database, error return nil, fmt.Errorf("slot sizes must be in increasing order") } prevSlotSize = slotSize - shelf, err := openShelf(opts.Path, slotSize, wrapShelfDataFn(len(db.shelves), slotSize, onData), opts.Readonly) + shelf, err := openShelf(opts.Path, slotSize, wrapShelfDataFn(len(db.shelves), slotSize, onData), + opts.ShelfFileSize, opts.ShelfFileCount, opts.Readonly) if err != nil { db.Close() // Close shelves return nil, err diff --git a/shelf.go b/shelf.go index 3a88c66..0e21e01 100644 --- a/shelf.go +++ b/shelf.go @@ -68,10 +68,13 @@ type shelfHeader struct { // If the shelf already exists, it's opened and read, which populates the // internal gap-list. // The onData callback is optional, and can be nil. -func openShelf(path string, slotSize uint32, onData onShelfDataFn, readonly bool) (*shelf, error) { +func openShelf(path string, slotSize uint32, onData onShelfDataFn, maxFileSize uint64, nFiles int, readonly bool) (*shelf, error) { if slotSize < minSlotSize { return nil, fmt.Errorf("slot size %d smaller than minimum (%d)", slotSize, minSlotSize) } + if maxFileSize != 0 && nFiles == 0 { + return nil, fmt.Errorf("number of files (%d) must be non-zero if max file size (%d) set", nFiles, maxFileSize) + } if path != "" { // empty path == in-memory database if finfo, err := os.Stat(path); err != nil { return nil, err @@ -83,17 +86,11 @@ func openShelf(path string, slotSize uint32, onData onShelfDataFn, readonly bool fileSize int h = shelfHeader{Magic, curVersion, slotSize} fname = fmt.Sprintf("bkt_%08d.bag", slotSize) - flags = os.O_RDWR | os.O_CREATE - ) - if readonly { - flags = os.O_RDONLY - } - var ( - f store - err error + f store + err error ) if path != "" { - f, err = os.OpenFile(filepath.Join(path, fname), flags, 0666) + f, err = newCappedFile(filepath.Join(path, fname), nFiles, maxFileSize, readonly) if err != nil { return nil, err } @@ -286,7 +283,7 @@ func (s *shelf) Get(slot uint64) ([]byte, error) { } data, err := s.readSlot(make([]byte, s.slotSize), slot) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrBadIndex, err) + return nil, fmt.Errorf("%w: slot %d, slotsize %d, %v", ErrBadIndex, slot, s.slotSize, err) } return data, nil } diff --git a/shelf_test.go b/shelf_test.go index 08fa13b..427e339 100644 --- a/shelf_test.go +++ b/shelf_test.go @@ -42,11 +42,15 @@ func TestBasicsOnDisk(t *testing.T) { testBasics(t, t.TempDir()) } func testBasics(t *testing.T, path string) { { // Pre-instance failures // can't open non-existing directory - if _, err := openShelf("/baz/bonk/foobar/gazonk", 10, nil, false); err == nil { + if _, err := openShelf("/baz/bonk/foobar/gazonk", 10, nil, 0, 0, false); err == nil { t.Fatal("expected error") } // Can't point path to a file - if _, err := openShelf("./README.md", 10, nil, false); err == nil { + if _, err := openShelf("./README.md", 10, nil, 0, 0, false); err == nil { + t.Fatal("expected error") + } + // Can't provide nonzero maxfilesize but zero files + if _, err := openShelf("foo", 10, nil, 1, 0, false); err == nil { t.Fatal("expected error") } } @@ -226,7 +230,7 @@ func checkIdentical(fileA, fileB string) error { func setup(t *testing.T, path string) (*shelf, func()) { t.Helper() - a, err := openShelf(path, 200, nil, false) + a, err := openShelf(path, 200, nil, 0, 0, false) if err != nil { t.Fatal(err) } @@ -379,12 +383,12 @@ func TestCompaction(t *testing.T) { haveOnData = append(haveOnData, data[0]) } /// Now open them as shelves - a, err = openShelf(pA, 10, onData, false) + a, err = openShelf(pA, 10, onData, 0, 0, false) if err != nil { t.Fatal(err) } a.Close() - b, err = openShelf(pB, 10, nil, false) + b, err = openShelf(pB, 10, nil, 0, 0, false) if err != nil { t.Fatal(err) } @@ -438,7 +442,7 @@ func TestCompaction2(t *testing.T) { p := t.TempDir() /// Now open them as shelves openAndStore := func(data string) { - a, err := openShelf(p, 10, nil, false) + a, err := openShelf(p, 10, nil, 0, 0, false) if err != nil { t.Fatal(err) } @@ -453,14 +457,14 @@ func TestCompaction2(t *testing.T) { var data []byte _, err := openShelf(p, 10, func(slot uint64, x []byte) { data = append(data, x...) - }, false) + }, 0, 0, false) if err != nil { t.Fatal(err) } return string(data) } openAndDel := func(deletes ...int) { - a, err := openShelf(p, 10, nil, false) + a, err := openShelf(p, 10, nil, 0, 0, false) if err != nil { t.Fatal(err) } @@ -496,7 +500,7 @@ func TestCompaction2(t *testing.T) { func TestShelfRO(t *testing.T) { p := t.TempDir() - a, err := openShelf(p, 20, nil, false) + a, err := openShelf(p, 20, nil, 0, 0, false) if err != nil { t.Fatal(err) } @@ -526,7 +530,7 @@ func TestShelfRO(t *testing.T) { out := new(strings.Builder) a, err = openShelf(p, 20, func(slot uint64, data []byte) { fmt.Fprintf(out, "%d:%d, ", slot, len(data)) - }, true) + }, 0, 0, true) if err != nil { t.Fatal(err) } @@ -554,7 +558,7 @@ func TestShelfRO(t *testing.T) { out = new(strings.Builder) a, err = openShelf(p, 20, func(slot uint64, data []byte) { fmt.Fprintf(out, "%d:%d, ", slot, len(data)) - }, false) + }, 0, 0, false) if err != nil { t.Fatal(err) } @@ -570,7 +574,7 @@ func TestShelfRO(t *testing.T) { func TestDelete(t *testing.T) { p := t.TempDir() - a, err := openShelf(p, 20, nil, false) + a, err := openShelf(p, 20, nil, 0, 0, false) if err != nil { t.Fatal(err) } @@ -642,7 +646,7 @@ func TestVersion(t *testing.T) { if err := os.WriteFile(filepath.Join(p, fname), tc.hdr, 0o777); err != nil { t.Fatal(err) } - _, err := openShelf(p, size, nil, false) + _, err := openShelf(p, size, nil, 0, 0, false) if err == nil { t.Fatal("expected error") } @@ -651,3 +655,47 @@ func TestVersion(t *testing.T) { } } } + +func TestCappedTruncate(t *testing.T) { + p := t.TempDir() + + // Max file size 100 + // Max files 5, total capacity 100 + // SHelf size 27 + 4 = 31 + s, err := openShelf(p, 27+itemHeaderSize, nil, 100, 5, false) + if err != nil { + t.Fatal(err) + } + // Fill up the capped files and delete an item to trigger a truncation + var keys []uint64 + for i := byte(1); i <= 10; i++ { + if key, err := s.Put(bytes.Repeat([]byte{i}, 27)); err != nil { + t.Fatalf("failed to put item %d: %v", i, err) + } else { + keys = append(keys, key) + } + } + err = s.Delete(keys[len(keys)-1]) + if err != nil { + t.Fatal(err) + } + keys = keys[:len(keys)-1] + // Reopen the shelf to compact it + s.Close() + s, err = openShelf(p, 27+itemHeaderSize, nil, 100, 5, false) + if err != nil { + t.Fatal(err) + } + // Verify that all values survived compactions + for i, key := range keys { + val, err := s.Get(key) + if err != nil { + t.Errorf("failed to retrieve slot %d: %v", i, err) + continue + } + want := bytes.Repeat([]byte{byte(i + 1)}, 27) + if !bytes.Equal(val, want) { + t.Errorf("item %d mismatch: have %x, want %x", i, val, want) + } + } +} diff --git a/store_capped.go b/store_capped.go new file mode 100644 index 0000000..a3d2d2b --- /dev/null +++ b/store_capped.go @@ -0,0 +1,171 @@ +// bagdb: Simple datastorage +// Copyright 2021 billy authors +// SPDX-License-Identifier: BSD-3-Clause + +package billy + +import ( + "fmt" + "os" +) + +// cappedFile has an API-surface as if it were one file, but maps +// to a set of files on disk. These files are all capped in size +// to maxFileSize. +type cappedFile struct { + cap uint64 + files []*os.File +} + +// newCappedFile creates a cappedFile +func newCappedFile(basename string, nFiles int, cap uint64, readonly bool) (*cappedFile, error) { + flags := os.O_RDWR | os.O_CREATE + if readonly { + flags = os.O_RDONLY + } + if cap == 0 { + nFiles = 1 + } + var files []*os.File + for i := 0; i < nFiles; i++ { + var ( + f *os.File + err error + ) + if i == 0 { + f, err = os.OpenFile(basename, flags, 0666) + } else { + f, err = os.OpenFile(fmt.Sprintf("%v.cap.%d", basename, i), flags, 0666) + } + if err != nil { + // Clean-up: close opened files + for _, f := range files { + f.Close() + } + return nil, err + } + files = append(files, f) + } + return &cappedFile{ + cap: cap, + files: files, + }, nil +} + +// WriteAt writes len(b) bytes to the file starting at byte offset off. +// It returns the number of bytes written and an error, if any. +// WriteAt returns a non-nil error when n != len(b). +// +// Internally, a write will only touch one file, and may cause the cap to be exceeeded. +func (cf *cappedFile) WriteAt(data []byte, off int64) (n int, err error) { + var ( + fNum = uint64(0) + fOffset = off + ) + if cf.cap > 0 { + fNum = uint64(off) / cf.cap + fOffset = off % int64(cf.cap) + } + // Check if the write is out of bounds + if fNum >= uint64(len(cf.files)) { + return 0, ErrBadIndex + } + //fmt.Printf("file-%d, write %d bytes @ %d (total offset %d)\n", fNum, len(data), fOffset, off) + return cf.files[fNum].WriteAt(data, fOffset) +} + +// ReadAt reads len(b) bytes from the file(s) starting at byte offset off. +// It returns the number of bytes read and the error, if any. +func (cf *cappedFile) ReadAt(b []byte, off int64) (n int, err error) { + var ( + fNum = uint64(0) + fOffset = off + ) + if cf.cap > 0 { + fNum = uint64(off) / cf.cap + fOffset = off % int64(cf.cap) + } + // Check if the read is out of bounds + if fNum >= uint64(len(cf.files)) { + return 0, ErrBadIndex + } + return cf.files[fNum].ReadAt(b, fOffset) +} + +// Sync calls *os.File Sync on the backing-files. +func (cf *cappedFile) Sync() error { + var err error + for _, f := range cf.files { + if e := f.Sync(); e != nil && err == nil { + err = e + } + } + return err +} + +// Close closes all files. +func (cf *cappedFile) Close() error { + var err error + for _, f := range cf.files { + if e := f.Close(); e != nil && err == nil { + err = e + } + } + return err +} + +// Truncate changes the size of the file. +func (cf *cappedFile) Truncate(size int64) error { + if cf.cap == 0 { + return cf.files[0].Truncate(size) + } + // Files fully below the truncation limit are left in place. This is subtly + // wrong: the os.File Stat() operation _expands_ a file if it is too small, + // so ideally we should maybe "truncate up" the files in passing. + // However, it's possible that the files have exceeded the capcacity, + // and we must not truncate them. + + i := int(uint64(size) / cf.cap) + //fmt.Printf("Truncate file-%d to %d (total size %d)\n", i, size%int64(cf.cap), size) + if err := cf.files[i].Truncate(size % int64(cf.cap)); err != nil { + return err + } + // Files fully above the truncation limit are truncated to zero. + for i++; i < len(cf.files); i++ { + //fmt.Printf("Truncate file-%d to 0\n", i) + if err := cf.files[i].Truncate(0); err != nil { + return err + } + } + return nil +} + +func (cf *cappedFile) Stat() (os.FileInfo, error) { + var size int64 + var err error + var uncounted int64 + for _, f := range cf.files { + finfo, e := f.Stat() + if e != nil { + if err != nil { + err = e + } + continue + } + s := finfo.Size() + if s == 0 { + size += uncounted + break + } + if cf.cap == 0 || s < int64(cf.cap) { + // File is not at capacity. No need to continue. + size += s + break + } else { + // File is at or over capacity. Add cf.cap bytes, and remember the overflow + size += int64(cf.cap) + uncounted = s - int64(cf.cap) + } + } + return &fileinfoMock{size: size}, err +} diff --git a/store_capped_test.go b/store_capped_test.go new file mode 100644 index 0000000..be391ec --- /dev/null +++ b/store_capped_test.go @@ -0,0 +1,189 @@ +// bagdb: Simple datastorage +// Copyright 2021 billy authors +// SPDX-License-Identifier: BSD-3-Clause + +package billy + +import ( + "bytes" + "errors" + "io" + "os" + "testing" +) + +// wipe deletes the files. +func wipe(t *testing.T, cf *cappedFile) { + t.Helper() + var err error + for _, f := range cf.files { + if e := os.RemoveAll(f.Name()); e != nil && err == nil { + err = e + } + } + if err != nil { + t.Fatal(err) + } +} + +func diskSize(t *testing.T, cf *cappedFile) int { + t.Helper() + var size int + for _, f := range cf.files { + finfo, err := f.Stat() + if err != nil { + t.Fatal(err) + } + size += int(finfo.Size()) + //t.Logf("file %d size %d total %d", j, finfo.Size(), have) + } + return size +} + +func TestWriteAt(t *testing.T) { + // Max filesize: 50 bytes. + // 10 individual files. + // Mox storage capacity: 500 bytes. + f, err := newCappedFile("multifile-test", 10, 50, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { wipe(t, f) }) + + { + // Writing data at offset 55 means that it should write into + // the second file, five bytes in. + // [0,0,0,0,0,m,o,o,h] + want := []byte("mooh") + if _, err := f.WriteAt(want, 55); err != nil { + t.Fatal(err) + } + have := make([]byte, 4) + if _, err := f.ReadAt(have, 55); err != nil { + t.Fatal(err) + } + if !bytes.Equal(have, want) { + t.Fatalf("have %x want %x", have, want) + } + } + + { + // Test data that starts at one file, and severely exceeds the cap. + want := []byte("miao01234567890123456789012345678901234567890123456789woof") + if _, err := f.WriteAt(want, 96); err != nil { + t.Fatal(err) + } + have := make([]byte, 58) + if _, err := f.ReadAt(have, 96); err != nil { + t.Fatal(err) + } + if !bytes.Equal(have, want) { + t.Fatalf("have %x want %x", have, want) + } + // We can now do a 'small write' into the second file, which should not + // 'disturb' the data in the first file. This is an implementation + // quirk, not a "desired feature", but still should be checked by tests. + if _, err := f.WriteAt(make([]byte, 50), 100); err != nil { + t.Fatal(err) + } + // Check original data in first file + have = make([]byte, 58) + if _, err := f.ReadAt(have, 96); err != nil { + t.Fatal(err) + } + if !bytes.Equal(have, want) { + t.Fatalf("have %x want %x", have, want) + } + } +} + +func TestTruncate(t *testing.T) { + // Max filesize: 50 bytes. + // 10 individual files. + // Mox storage capacity: 500 bytes. + f, err := newCappedFile("multifile-test", 10, 50, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { wipe(t, f) }) + // Fill with data + for i := 0; i < 47; i++ { + if _, err := f.WriteAt(make([]byte, 10), int64(20+i*10)); err != nil { + t.Fatal(err) + } + } + // The total size of all files should == 490 + if have, want := diskSize(t, f), 490; have != want { + t.Fatalf("have %d want %d", have, want) + } + for i := 480; i > 0; i -= 10 { + if err := f.Truncate(int64(i)); err != nil { + t.Fatal(err) + } + // The total size of all files should == i + if have, want := diskSize(t, f), i; have != want { + t.Fatalf("have %d want %d", have, want) + } + } + // And "truncate" back up again + // NO longer supported + //for i := 0; i < 480; i += 10 { + // if err := f.Truncate(int64(i)); err != nil { + // t.Fatal(err) + // } + // // The total size of all files should == i + // if have, want := diskSize(t, f), i; have != want { + // t.Fatalf("have %d want %d", have, want) + // } + //} +} + +func TestReadonly(t *testing.T) { + // Create in readonly -- should fail + f, err := newCappedFile("multifile-ro-test", 10, 50, true) + if err == nil { + t.Fatal("want error trying to create files in readonly, got none") + } + // Create in RW, should be ok + f, err = newCappedFile("multifile-ro-test", 10, 50, false) + if err != nil { + t.Fatal(err) + } + f.Close() + // The files now exist, so should be ok to open + f, err = newCappedFile("multifile-ro-test", 10, 50, true) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { wipe(t, f) }) + if _, err := f.WriteAt([]byte("mooh"), 55); err == nil { + t.Fatalf("want error trying to write files in readonly, got none") + } +} + +func TestOutOfBounds(t *testing.T) { + f, err := newCappedFile("multifile-ro-test", 10, 50, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + f.Close() + wipe(t, f) + }) + // Read starting OOB + if _, err := f.ReadAt(make([]byte, 10), 501); !errors.Is(err, ErrBadIndex) { + t.Fatalf("want %v have %v", ErrBadIndex, err) + } + // Read reaching into OOB + if _, err := f.ReadAt(make([]byte, 10), 495); !errors.Is(err, io.EOF) { + t.Fatalf("want %v have %v", ErrBadIndex, err) + } + // Write starting OOB + if _, err := f.WriteAt(make([]byte, 10), 501); !errors.Is(err, ErrBadIndex) { + t.Fatalf("want %v have %v", ErrBadIndex, err) + } + // Write exceeding the global cap (not enforced) + if _, err := f.WriteAt(make([]byte, 10), 495); err != nil { + t.Fatal(err) + } +}