From 10cfd47f4a82635266f18f0ef1c7ec751e03ae6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Stojko?= Date: Fri, 14 Aug 2026 13:59:54 +0000 Subject: [PATCH] Add get_live_bytes() to report the current heap footprint of theta, tuple and kll sketches The sketch classes expose get_serialized_size_bytes() (an upper bound on the serialized form) but no accessor for the heap they are actually holding right now. get_live_bytes() reports that exact live footprint in O(1), which is useful for memory accounting when many mutable sketches are kept in memory at once. The accessor lives on theta_update_sketch_base (the hash table shared by theta and tuple update sketches, unions and intersections): it returns (1 << lg_cur_size_) * sizeof(Entry), or 0 when the table is unallocated. The public update_theta_sketch/tuple update sketch, theta/tuple union and theta/tuple intersection forward to it; tuple's union and intersection reuse the theta base, so no separate implementation is needed there. kll is not a hash table, so kll_sketch::get_live_bytes() sums the items_ buffer (items_size_ elements) and the levels_ vector, mirroring the sketch's allocate() sites. Adds test cases (theta update sketch, theta union, theta intersection, tuple update sketch, kll) asserting the footprint is a power-of-two entry count where applicable, is 0 for an unallocated intersection, and only grows as entries are inserted. Co-authored-by: Isaac --- kll/include/kll_sketch.hpp | 12 +++++++++++ kll/test/kll_sketch_test.cpp | 19 +++++++++++++++++ theta/include/theta_intersection.hpp | 7 +++++++ theta/include/theta_intersection_base.hpp | 3 +++ theta/include/theta_sketch.hpp | 8 ++++++++ theta/include/theta_union.hpp | 7 +++++++ theta/include/theta_union_base.hpp | 3 +++ theta/include/theta_update_sketch_base.hpp | 7 +++++++ theta/test/theta_intersection_test.cpp | 15 ++++++++++++++ theta/test/theta_sketch_test.cpp | 24 ++++++++++++++++++++++ theta/test/theta_union_test.cpp | 15 ++++++++++++++ tuple/include/tuple_intersection.hpp | 7 +++++++ tuple/include/tuple_sketch.hpp | 8 ++++++++ tuple/include/tuple_union.hpp | 7 +++++++ tuple/test/tuple_sketch_test.cpp | 19 +++++++++++++++++ 15 files changed, 161 insertions(+) diff --git a/kll/include/kll_sketch.hpp b/kll/include/kll_sketch.hpp index d672c419..55c62ac8 100644 --- a/kll/include/kll_sketch.hpp +++ b/kll/include/kll_sketch.hpp @@ -410,6 +410,18 @@ class kll_sketch { template, typename std::enable_if::value, int>::type = 0> size_t get_serialized_size_bytes(const SerDe& sd = SerDe()) const; + /** + * Returns the number of bytes currently allocated on the heap by this sketch: the items_ buffer + * (items_size_ elements) plus the levels_ vector. Unlike a hash-table sketch this is not a table + * footprint; it mirrors the sketch's allocate() sites. This is the exact live heap footprint + * right now, not an upper bound like get_serialized_size_bytes(). + * @return the current allocated heap size of the sketch in bytes + */ + size_t get_live_bytes() const { + return (items_ == nullptr ? 0 : static_cast(items_size_) * sizeof(T)) + + levels_.capacity() * sizeof(uint32_t); + } + /** * Returns upper bound on the serialized size of a sketch given a parameter k and stream * length. The resulting size is an overestimate to make sure actual sketches don't exceed it. diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp index b8d0b031..6ad6eb17 100644 --- a/kll/test/kll_sketch_test.cpp +++ b/kll/test/kll_sketch_test.cpp @@ -835,4 +835,23 @@ TEST_CASE("kll sketch", "[kll_sketch]") { REQUIRE(test_allocator_total_bytes == 0); } +TEST_CASE("kll sketch: get_live_bytes", "[kll_sketch]") { + kll_sketch sketch; + // a fresh sketch has already allocated its items buffer and level boundaries + const size_t empty_bytes = sketch.get_live_bytes(); + REQUIRE(empty_bytes > 0); + + // the items buffer and levels only grow as the sketch fills, and grow past the initial size + size_t prev_bytes = empty_bytes; + bool non_decreasing = true; + for (int i = 0; i < 1000000; ++i) { + sketch.update(static_cast(i)); + const size_t bytes = sketch.get_live_bytes(); + if (bytes < prev_bytes) non_decreasing = false; + prev_bytes = bytes; + } + REQUIRE(non_decreasing); + REQUIRE(prev_bytes > empty_bytes); +} + } /* namespace datasketches */ diff --git a/theta/include/theta_intersection.hpp b/theta/include/theta_intersection.hpp index 9b82d274..fba44a9a 100644 --- a/theta/include/theta_intersection.hpp +++ b/theta/include/theta_intersection.hpp @@ -83,6 +83,13 @@ class theta_intersection_alloc { */ bool has_result() const; + /** + * Returns the number of bytes currently allocated for this intersection's internal hash table. + * This is the exact live heap footprint right now, not an upper bound. + * @return the current allocated size of the internal hash table in bytes + */ + size_t get_live_bytes() const { return state_.get_live_bytes(); } + private: State state_; }; diff --git a/theta/include/theta_intersection_base.hpp b/theta/include/theta_intersection_base.hpp index c0345902..d7c9f890 100644 --- a/theta/include/theta_intersection_base.hpp +++ b/theta/include/theta_intersection_base.hpp @@ -46,6 +46,9 @@ class theta_intersection_base { const Policy& get_policy() const; + // Live heap footprint of the internal hash table in bytes. + size_t get_live_bytes() const { return table_.get_live_bytes(); } + private: Policy policy_; bool is_valid_; diff --git a/theta/include/theta_sketch.hpp b/theta/include/theta_sketch.hpp index 4aab4b92..0c26299f 100644 --- a/theta/include/theta_sketch.hpp +++ b/theta/include/theta_sketch.hpp @@ -341,6 +341,14 @@ class update_theta_sketch_alloc: public theta_sketch_alloc { virtual const_iterator begin() const; virtual const_iterator end() const; + /** + * Returns the number of bytes currently allocated for this sketch's internal hash table. + * This is the exact live heap footprint of the retained entries right now, not an upper bound + * like get_serialized_size_bytes(). + * @return the current allocated size of the internal hash table in bytes + */ + size_t get_live_bytes() const { return table_.get_live_bytes(); } + private: theta_table table_; diff --git a/theta/include/theta_union.hpp b/theta/include/theta_union.hpp index 4c62f4f0..d8acbb5e 100644 --- a/theta/include/theta_union.hpp +++ b/theta/include/theta_union.hpp @@ -74,6 +74,13 @@ class theta_union_alloc { /// Reset the union to the initial empty state void reset(); + /** + * Returns the number of bytes currently allocated for this union's internal hash table. + * This is the exact live heap footprint right now, not an upper bound. + * @return the current allocated size of the internal hash table in bytes + */ + size_t get_live_bytes() const { return state_.get_live_bytes(); } + private: State state_; diff --git a/theta/include/theta_union_base.hpp b/theta/include/theta_union_base.hpp index 6da10b26..71d7e0ac 100644 --- a/theta/include/theta_union_base.hpp +++ b/theta/include/theta_union_base.hpp @@ -49,6 +49,9 @@ class theta_union_base { void reset(); + // Live heap footprint of the internal hash table in bytes. + size_t get_live_bytes() const { return table_.get_live_bytes(); } + private: Policy policy_; hash_table table_; diff --git a/theta/include/theta_update_sketch_base.hpp b/theta/include/theta_update_sketch_base.hpp index 91013638..21d6e8b7 100644 --- a/theta/include/theta_update_sketch_base.hpp +++ b/theta/include/theta_update_sketch_base.hpp @@ -62,6 +62,13 @@ struct theta_update_sketch_base { iterator begin() const; iterator end() const; + // Bytes currently allocated for the entries_ hash table: (1 << lg_cur_size_) entries, or 0 when + // the table is unallocated (lg_cur_size_ == 0 leaves entries_ == nullptr). This is the exact live + // heap footprint of the table right now, not an upper bound like the serialized-size estimates. + size_t get_live_bytes() const { + return entries_ == nullptr ? 0 : (static_cast(1) << lg_cur_size_) * sizeof(Entry); + } + // resize threshold = 0.5 tuned for speed static constexpr double RESIZE_THRESHOLD = 0.5; // hash table rebuild threshold = 15/16 diff --git a/theta/test/theta_intersection_test.cpp b/theta/test/theta_intersection_test.cpp index 3c2f139b..e6469869 100644 --- a/theta/test/theta_intersection_test.cpp +++ b/theta/test/theta_intersection_test.cpp @@ -237,4 +237,19 @@ TEST_CASE("theta intersection: seed mismatch", "[theta_intersection]") { REQUIRE_THROWS_AS(intersection.update(sketch), std::invalid_argument); } +TEST_CASE("theta intersection: get_live_bytes", "[theta_intersection]") { + theta_intersection intersection; + // before any update the intersection holds no table + REQUIRE(intersection.get_live_bytes() == 0); + + update_theta_sketch sketch = update_theta_sketch::builder().build(); + for (int i = 0; i < 100000; ++i) sketch.update(i); + intersection.update(sketch); + const size_t bytes = intersection.get_live_bytes(); + REQUIRE(bytes > 0); + REQUIRE(bytes % sizeof(uint64_t) == 0); + const size_t entries = bytes / sizeof(uint64_t); + REQUIRE((entries & (entries - 1)) == 0); +} + } /* namespace datasketches */ diff --git a/theta/test/theta_sketch_test.cpp b/theta/test/theta_sketch_test.cpp index 97c4f14e..f6534242 100644 --- a/theta/test/theta_sketch_test.cpp +++ b/theta/test/theta_sketch_test.cpp @@ -625,4 +625,28 @@ TEST_CASE("max serialized size", "[theta_sketch]") { REQUIRE(max_size_bytes == compact_theta_sketch::get_max_serialized_size_bytes(lg_k)); } +TEST_CASE("theta sketch: get_live_bytes", "[theta_sketch]") { + update_theta_sketch sketch = update_theta_sketch::builder().build(); + // a freshly built update sketch has an allocated hash table of 8-byte entries + const size_t empty_bytes = sketch.get_live_bytes(); + REQUIRE(empty_bytes > 0); + REQUIRE(empty_bytes % sizeof(uint64_t) == 0); + const size_t empty_entries = empty_bytes / sizeof(uint64_t); + REQUIRE((empty_entries & (empty_entries - 1)) == 0); // capacity is a power of two + + // the table only grows as distinct keys are inserted, and it grows past the initial size + size_t prev_bytes = empty_bytes; + bool non_decreasing = true; + for (int i = 0; i < 100000; ++i) { + sketch.update(i); + const size_t bytes = sketch.get_live_bytes(); + if (bytes < prev_bytes) non_decreasing = false; + prev_bytes = bytes; + } + REQUIRE(non_decreasing); + REQUIRE(prev_bytes > empty_bytes); + const size_t entries = prev_bytes / sizeof(uint64_t); + REQUIRE((entries & (entries - 1)) == 0); +} + } /* namespace datasketches */ diff --git a/theta/test/theta_union_test.cpp b/theta/test/theta_union_test.cpp index aef1c312..f51ce922 100644 --- a/theta/test/theta_union_test.cpp +++ b/theta/test/theta_union_test.cpp @@ -153,4 +153,19 @@ TEST_CASE("theta union: larger K", "[theta_union]") { REQUIRE(result2.get_estimate() == update_sketch3.get_estimate()); } +TEST_CASE("theta union: get_live_bytes", "[theta_union]") { + update_theta_sketch update_sketch = update_theta_sketch::builder().build(); + for (int i = 0; i < 100000; ++i) update_sketch.update(i); + + theta_union u = theta_union::builder().build(); + const size_t empty_bytes = u.get_live_bytes(); + u.update(update_sketch); + const size_t bytes = u.get_live_bytes(); + // unioning a large sketch grows the internal table past its initial footprint + REQUIRE(bytes > empty_bytes); + REQUIRE(bytes % sizeof(uint64_t) == 0); + const size_t entries = bytes / sizeof(uint64_t); + REQUIRE((entries & (entries - 1)) == 0); +} + } /* namespace datasketches */ diff --git a/tuple/include/tuple_intersection.hpp b/tuple/include/tuple_intersection.hpp index 12de7b73..3156a0a6 100644 --- a/tuple/include/tuple_intersection.hpp +++ b/tuple/include/tuple_intersection.hpp @@ -100,6 +100,13 @@ class tuple_intersection { */ bool has_result() const; + /** + * Returns the number of bytes currently allocated for this intersection's internal hash table. + * This is the exact live heap footprint right now, not an upper bound. + * @return the current allocated size of the internal hash table in bytes + */ + size_t get_live_bytes() const { return state_.get_live_bytes(); } + protected: State state_; }; diff --git a/tuple/include/tuple_sketch.hpp b/tuple/include/tuple_sketch.hpp index 7b636a78..5240e7f6 100644 --- a/tuple/include/tuple_sketch.hpp +++ b/tuple/include/tuple_sketch.hpp @@ -436,6 +436,14 @@ class update_tuple_sketch: public tuple_sketch { virtual const_iterator begin() const; virtual const_iterator end() const; + /** + * Returns the number of bytes currently allocated for this sketch's internal hash table. + * This is the exact live heap footprint of the retained entries right now, not an upper bound + * like get_serialized_size_bytes(). + * @return the current allocated size of the internal hash table in bytes + */ + size_t get_live_bytes() const { return map_.get_live_bytes(); } + protected: Policy policy_; tuple_map map_; diff --git a/tuple/include/tuple_union.hpp b/tuple/include/tuple_union.hpp index 71612b6d..24054f63 100644 --- a/tuple/include/tuple_union.hpp +++ b/tuple/include/tuple_union.hpp @@ -89,6 +89,13 @@ class tuple_union { */ void reset(); + /** + * Returns the number of bytes currently allocated for this union's internal hash table. + * This is the exact live heap footprint right now, not an upper bound. + * @return the current allocated size of the internal hash table in bytes + */ + size_t get_live_bytes() const { return state_.get_live_bytes(); } + protected: State state_; diff --git a/tuple/test/tuple_sketch_test.cpp b/tuple/test/tuple_sketch_test.cpp index 30c415f9..417bcdcc 100644 --- a/tuple/test/tuple_sketch_test.cpp +++ b/tuple/test/tuple_sketch_test.cpp @@ -386,4 +386,23 @@ TEST_CASE("tuple sketch: deserialize bounds-checks each entry key", "[tuple_sket std::out_of_range); } +TEST_CASE("tuple sketch: get_live_bytes", "[tuple_sketch]") { + auto sketch = update_tuple_sketch::builder().build(); + // a freshly built update sketch has an allocated hash table + const size_t empty_bytes = sketch.get_live_bytes(); + REQUIRE(empty_bytes > 0); + + // the table only grows as distinct keys are inserted, and it grows past the initial size + size_t prev_bytes = empty_bytes; + bool non_decreasing = true; + for (int i = 0; i < 100000; ++i) { + sketch.update(i, 1.0f); + const size_t bytes = sketch.get_live_bytes(); + if (bytes < prev_bytes) non_decreasing = false; + prev_bytes = bytes; + } + REQUIRE(non_decreasing); + REQUIRE(prev_bytes > empty_bytes); +} + } /* namespace datasketches */