shelf: use files capped to certain size

cmd/billyfuzz/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	crand "crypto/rand"
 	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
 	"math/rand"
 	"os"
@@ -68,18 +67,17 @@ func doOpenDb(ctx *cli.Context, onData billy.OnDataFn) (billy.Database, error) {
 
 func doFuzz(ctx *cli.Context) error {
 	var (
-		hasher = sha256.New()
 		hashes = make(map[uint64]string)
 		onData = func(key uint64, size uint32, data []byte) {
 			if verbose {
 				fmt.Printf("init key %x val %x\n", key, data[:20])
 			}
-			hasher.Reset()
-			hashes[key] = hex.EncodeToString(hasher.Sum(data))
+			hashes[key] = fmt.Sprintf("%x", sha256.Sum256(data))
 		}
 		db, err = doOpenDb(ctx, onData)
 	)
 	if err != nil {
+		fmt.Printf("Error opening db: %v\n", err)
 		return err
 	}
 	var (
@@ -109,8 +107,7 @@ func doFuzz(ctx *cli.Context) error {
 			l := int(min) + rand.Intn(int(max-min))
 			data := make([]byte, l)
 			_, _ = crand.Read(data)
-			hasher.Reset()
-			sum := hex.EncodeToString(hasher.Sum(data))
+			sum := fmt.Sprintf("%x", sha256.Sum256(data))
 			key, err := db.Put(data)
 			if err != nil {
 				panic(err)
@@ -126,19 +123,23 @@ func doFuzz(ctx *cli.Context) error {
 			for key, want = range hashes {
 				break
 			}
-			//fmt.Printf("Checking %d bytes data at key %d\n", len(want), key)
 			data, err := db.Get(key)
 			if err != nil {
+				fmt.Printf("Checking data at key %d\n", key)
 				panic(err)
 			}
 			// check the data
-			hasher.Reset()
-			have := hex.EncodeToString(hasher.Sum(data))
+			have := fmt.Sprintf("%x", sha256.Sum256(data))
 			if have != want {
-				panic(fmt.Sprintf("key %v\nhave %v\n, want %v\n", key, have, want))
+				fmt.Printf("key %v\nhave %d bytes, hash %v\n, want %v\n", key,
+					len(data), have, want)
+				panic("GET failure")
 			}
 		case 2: // DELETE
 			var key uint64
+			if len(hashes) == 0 {
+				continue
+			}
 			for key = range hashes {
 				break
 			}

db.go

@@ -86,9 +86,20 @@ type database struct {
 }
 
 type Options struct {
-	Path     string
+	// Path is the path to where the billy-files are stored. An empty value
+	// means 'memory-mode', in which nothing is stored to disk.
+	Path string
+	// ShelfFileSize is the maximum size of files used. A value of zero means that
+	// there is no maximum.
+	ShelfFileSize uint64
+	// ShelfFileCount is the amount of files to open for each shelf. The total
+	// storage capacity of any shelf is thus `ShelfFileSize * ShelfFileCount`.
+	// If `ShelfFileSize==0`, then this value is not used.
+	ShelfFileCount int
+	// Readonly means that the db cannot be used for writing.
 	Readonly bool
-	Snappy   bool // unused for now
+	// Snappy is not used
+	Snappy bool
 }
 
 // Open opens a (new or existing) database, with configurable limits. The given
@@ -113,7 +124,8 @@ func Open(opts Options, slotSizeFn SlotSizeFn, onData OnDataFn) (Database, error
 			return nil, fmt.Errorf("slot sizes must be in increasing order")
 		}
 		prevSlotSize = slotSize
-		shelf, err := openShelf(opts.Path, slotSize, wrapShelfDataFn(len(db.shelves), slotSize, onData), opts.Readonly)
+		shelf, err := openShelf(opts.Path, slotSize, wrapShelfDataFn(len(db.shelves), slotSize, onData),
+			opts.ShelfFileSize, opts.ShelfFileCount, opts.Readonly)
 		if err != nil {
 			db.Close() // Close shelves
 			return nil, err

shelf.go

@@ -68,10 +68,13 @@ type shelfHeader struct {
 // If the shelf already exists, it's opened and read, which populates the
 // internal gap-list.
 // The onData callback is optional, and can be nil.
-func openShelf(path string, slotSize uint32, onData onShelfDataFn, readonly bool) (*shelf, error) {
+func openShelf(path string, slotSize uint32, onData onShelfDataFn, maxFileSize uint64, nFiles int, readonly bool) (*shelf, error) {
 	if slotSize < minSlotSize {
 		return nil, fmt.Errorf("slot size %d smaller than minimum (%d)", slotSize, minSlotSize)
 	}
+	if maxFileSize != 0 && nFiles == 0 {
+		return nil, fmt.Errorf("number of files (%d) must be non-zero if max file size (%d) set", nFiles, maxFileSize)
+	}
 	if path != "" { // empty path == in-memory database
 		if finfo, err := os.Stat(path); err != nil {
 			return nil, err
@@ -83,17 +86,11 @@ func openShelf(path string, slotSize uint32, onData onShelfDataFn, readonly bool
 		fileSize int
 		h        = shelfHeader{Magic, curVersion, slotSize}
 		fname    = fmt.Sprintf("bkt_%08d.bag", slotSize)
-		flags    = os.O_RDWR | os.O_CREATE
-	)
-	if readonly {
-		flags = os.O_RDONLY
-	}
-	var (
-		f   store
-		err error
+		f        store
+		err      error
 	)
 	if path != "" {
-		f, err = os.OpenFile(filepath.Join(path, fname), flags, 0666)
+		f, err = newCappedFile(filepath.Join(path, fname), nFiles, maxFileSize, readonly)
 		if err != nil {
 			return nil, err
 		}
@@ -286,7 +283,7 @@ func (s *shelf) Get(slot uint64) ([]byte, error) {
 	}
 	data, err := s.readSlot(make([]byte, s.slotSize), slot)
 	if err != nil {
-		return nil, fmt.Errorf("%w: %v", ErrBadIndex, err)
+		return nil, fmt.Errorf("%w: slot %d, slotsize %d, %v", ErrBadIndex, slot, s.slotSize, err)
 	}
 	return data, nil
 }

shelf_test.go

@@ -42,11 +42,15 @@ func TestBasicsOnDisk(t *testing.T)   { testBasics(t, t.TempDir()) }
 func testBasics(t *testing.T, path string) {
 	{ // Pre-instance failures
 		// can't open non-existing directory
-		if _, err := openShelf("/baz/bonk/foobar/gazonk", 10, nil, false); err == nil {
+		if _, err := openShelf("/baz/bonk/foobar/gazonk", 10, nil, 0, 0, false); err == nil {
 			t.Fatal("expected error")
 		}
 		// Can't point path to a file
-		if _, err := openShelf("./README.md", 10, nil, false); err == nil {
+		if _, err := openShelf("./README.md", 10, nil, 0, 0, false); err == nil {
+			t.Fatal("expected error")
+		}
+		// Can't provide nonzero maxfilesize but zero files
+		if _, err := openShelf("foo", 10, nil, 1, 0, false); err == nil {
 			t.Fatal("expected error")
 		}
 	}
@@ -226,7 +230,7 @@ func checkIdentical(fileA, fileB string) error {
 
 func setup(t *testing.T, path string) (*shelf, func()) {
 	t.Helper()
-	a, err := openShelf(path, 200, nil, false)
+	a, err := openShelf(path, 200, nil, 0, 0, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -379,12 +383,12 @@ func TestCompaction(t *testing.T) {
 		haveOnData = append(haveOnData, data[0])
 	}
 	/// Now open them as shelves
-	a, err = openShelf(pA, 10, onData, false)
+	a, err = openShelf(pA, 10, onData, 0, 0, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 	a.Close()
-	b, err = openShelf(pB, 10, nil, false)
+	b, err = openShelf(pB, 10, nil, 0, 0, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -438,7 +442,7 @@ func TestCompaction2(t *testing.T) {
 	p := t.TempDir()
 	/// Now open them as shelves
 	openAndStore := func(data string) {
-		a, err := openShelf(p, 10, nil, false)
+		a, err := openShelf(p, 10, nil, 0, 0, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -453,14 +457,14 @@ func TestCompaction2(t *testing.T) {
 		var data []byte
 		_, err := openShelf(p, 10, func(slot uint64, x []byte) {
 			data = append(data, x...)
-		}, false)
+		}, 0, 0, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		return string(data)
 	}
 	openAndDel := func(deletes ...int) {
-		a, err := openShelf(p, 10, nil, false)
+		a, err := openShelf(p, 10, nil, 0, 0, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -496,7 +500,7 @@ func TestCompaction2(t *testing.T) {
 func TestShelfRO(t *testing.T) {
 	p := t.TempDir()
 
-	a, err := openShelf(p, 20, nil, false)
+	a, err := openShelf(p, 20, nil, 0, 0, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -526,7 +530,7 @@ func TestShelfRO(t *testing.T) {
 	out := new(strings.Builder)
 	a, err = openShelf(p, 20, func(slot uint64, data []byte) {
 		fmt.Fprintf(out, "%d:%d, ", slot, len(data))
-	}, true)
+	}, 0, 0, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -554,7 +558,7 @@ func TestShelfRO(t *testing.T) {
 	out = new(strings.Builder)
 	a, err = openShelf(p, 20, func(slot uint64, data []byte) {
 		fmt.Fprintf(out, "%d:%d, ", slot, len(data))
-	}, false)
+	}, 0, 0, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -570,7 +574,7 @@ func TestShelfRO(t *testing.T) {
 
 func TestDelete(t *testing.T) {
 	p := t.TempDir()
-	a, err := openShelf(p, 20, nil, false)
+	a, err := openShelf(p, 20, nil, 0, 0, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -642,7 +646,7 @@ func TestVersion(t *testing.T) {
 		if err := os.WriteFile(filepath.Join(p, fname), tc.hdr, 0o777); err != nil {
 			t.Fatal(err)
 		}
-		_, err := openShelf(p, size, nil, false)
+		_, err := openShelf(p, size, nil, 0, 0, false)
 		if err == nil {
 			t.Fatal("expected error")
 		}
@@ -651,3 +655,47 @@ func TestVersion(t *testing.T) {
 		}
 	}
 }
+
+func TestCappedTruncate(t *testing.T) {
+	p := t.TempDir()
+
+	// Max file size 100
+	// Max files 5, total capacity 100
+	// SHelf size 27 + 4 = 31
+	s, err := openShelf(p, 27+itemHeaderSize, nil, 100, 5, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Fill up the capped files and delete an item to trigger a truncation
+	var keys []uint64
+	for i := byte(1); i <= 10; i++ {
+		if key, err := s.Put(bytes.Repeat([]byte{i}, 27)); err != nil {
+			t.Fatalf("failed to put item %d: %v", i, err)
+		} else {
+			keys = append(keys, key)
+		}
+	}
+	err = s.Delete(keys[len(keys)-1])
+	if err != nil {
+		t.Fatal(err)
+	}
+	keys = keys[:len(keys)-1]
+	// Reopen the shelf to compact it
+	s.Close()
+	s, err = openShelf(p, 27+itemHeaderSize, nil, 100, 5, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Verify that all values survived compactions
+	for i, key := range keys {
+		val, err := s.Get(key)
+		if err != nil {
+			t.Errorf("failed to retrieve slot %d: %v", i, err)
+			continue
+		}
+		want := bytes.Repeat([]byte{byte(i + 1)}, 27)
+		if !bytes.Equal(val, want) {
+			t.Errorf("item %d mismatch: have %x, want %x", i, val, want)
+		}
+	}
+}