From a65f94f7d7446e051b136935f936354e5fee6b59 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 5 Aug 2026 12:38:04 +0200 Subject: [PATCH 1/2] Add min bloom filter crate Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ab0460bc-ae55-4457-a8e0-00169d9c2692 --- README.md | 1 + crates/min-bloom-filter/Cargo.toml | 21 ++ crates/min-bloom-filter/README.md | 42 +++ .../benchmarks/performance.rs | 136 ++++++++ crates/min-bloom-filter/src/lib.rs | 317 ++++++++++++++++++ 5 files changed, 517 insertions(+) create mode 100644 crates/min-bloom-filter/Cargo.toml create mode 100644 crates/min-bloom-filter/README.md create mode 100644 crates/min-bloom-filter/benchmarks/performance.rs create mode 100644 crates/min-bloom-filter/src/lib.rs diff --git a/README.md b/README.md index daf2dc1..665a8fe 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ A collection of useful algorithms written in Rust. Currently contains: - [`sparse-ngrams`](crates/sparse-ngrams): fast sparse n-gram extraction from byte slices. Selects variable-length n-grams (2–8 bytes) deterministically using bigram frequency priorities, suitable for substring search indexes. - [`string-offsets`](crates/string-offsets): converts string positions between bytes, chars, UTF-16 code units, and line numbers. Useful when sending string indices across language boundaries. - [`casefold`](crates/casefold): a **fast** Unicode simple case-folding library backed by a **very compact** (~1.7 KB) paged-bitmap + run-length table. Folds whole strings at multiple GiB/s via a decode-free `simple_fold` that rewrites UTF-8 with little-endian byte arithmetic, beating a `HashMap` fold table by several × at ~10× less memory. +- [`min-bloom-filter`](crates/min-bloom-filter): a split-block Bloom filter with four-bit values, max insertion, and min retrieval. ## Background diff --git a/crates/min-bloom-filter/Cargo.toml b/crates/min-bloom-filter/Cargo.toml new file mode 100644 index 0000000..3447ad3 --- /dev/null +++ b/crates/min-bloom-filter/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "min-bloom-filter" +version = "0.1.0" +edition = "2024" +description = "A split-block Bloom filter whose four-bit counters support max insertion and min retrieval." +repository = "https://github.com/github/rust-gems" +license = "MIT" +keywords = ["bloom-filter", "probabilistic", "filter", "data-structure"] +categories = ["algorithms", "data-structures"] + +[lib] +bench = false + +[[bench]] +name = "performance" +path = "benchmarks/performance.rs" +harness = false + +[dev-dependencies] +criterion = "0.8" +sbbf-rs-safe = "0.3" diff --git a/crates/min-bloom-filter/README.md b/crates/min-bloom-filter/README.md new file mode 100644 index 0000000..bd86032 --- /dev/null +++ b/crates/min-bloom-filter/README.md @@ -0,0 +1,42 @@ +# min-bloom-filter + +A split-block Bloom filter that stores four-bit values instead of bits. + +Each 32-byte block contains eight `u32` words, and each word contains eight +nibbles. A hash selects one nibble per word. Inserting raises all eight selected +nibbles to the maximum of their current value and the inserted value; retrieval +returns their minimum. + +```rust +use min_bloom_filter::MinBloomFilter; + +let mut filter = MinBloomFilter::new(3_000_000, 0.01); +filter.insert(0x1234_5678_9abc_def0, 7); + +assert_eq!(filter.get(0x1234_5678_9abc_def0), 7); +``` + +Values must be in `0..=15`. Like a regular Bloom filter, collisions can produce +false positives: a key that was not inserted can retrieve a nonzero value. + +## Performance + +Criterion results on an Apple M4 Max for 3 million entries: + +| Target FPR | Memory | Insert | Retrieve | +|---|---:|---:|---:| +| 1% | 14.52 MB | 2.27 ns/entry | 1.42 ns/entry | +| 0.1% | 21.91 MB | 2.74 ns/entry | 1.58 ns/entry | + +For comparison, `sbbf-rs-safe` on the same hashes: + +| Target FPR | Insert | Contains | +|---|---:|---:| +| 1% | 1.50 ns/entry | 1.24 ns/entry | +| 0.1% | 1.55 ns/entry | 1.26 ns/entry | + +Run these benchmarks with: + +```console +cargo bench -p min-bloom-filter --bench performance +``` diff --git a/crates/min-bloom-filter/benchmarks/performance.rs b/crates/min-bloom-filter/benchmarks/performance.rs new file mode 100644 index 0000000..85e71d7 --- /dev/null +++ b/crates/min-bloom-filter/benchmarks/performance.rs @@ -0,0 +1,136 @@ +use std::hint::black_box; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use min_bloom_filter::MinBloomFilter; +use sbbf_rs_safe::Filter as Sbbf; + +const ENTRY_COUNT: usize = 3_000_000; +const RATES: [(&str, f64); 2] = [("1_percent", 0.01), ("0.1_percent", 0.001)]; + +fn hashes() -> Vec { + (0..ENTRY_COUNT) + .map(|index| splitmix64(index as u64)) + .collect() +} + +fn benchmark_insert(c: &mut Criterion) { + let hashes = hashes(); + let mut group = c.benchmark_group("insert_3m"); + group.sample_size(10); + group.throughput(Throughput::Elements(ENTRY_COUNT as u64)); + + for (name, rate) in RATES { + group.bench_with_input(BenchmarkId::new("fpr", name), &rate, |b, rate| { + b.iter_batched( + || MinBloomFilter::new(ENTRY_COUNT, *rate), + |mut filter| { + for (index, hash) in hashes.iter().copied().enumerate() { + filter.insert(black_box(hash), ((index % 15) + 1) as u8); + } + black_box(filter); + }, + criterion::BatchSize::LargeInput, + ); + }); + } + group.finish(); +} + +fn benchmark_get(c: &mut Criterion) { + let hashes = hashes(); + let mut group = c.benchmark_group("get_3m"); + group.sample_size(10); + group.throughput(Throughput::Elements(ENTRY_COUNT as u64)); + + for (name, rate) in RATES { + let mut filter = MinBloomFilter::new(ENTRY_COUNT, rate); + for (index, hash) in hashes.iter().copied().enumerate() { + filter.insert(hash, ((index % 15) + 1) as u8); + } + + group.bench_with_input(BenchmarkId::new("fpr", name), &filter, |b, filter| { + b.iter(|| { + let mut sum = 0_u64; + for hash in hashes.iter().copied() { + sum += u64::from(filter.get(black_box(hash))); + } + black_box(sum) + }); + }); + } + group.finish(); +} + +fn benchmark_sbbf_insert(c: &mut Criterion) { + let hashes = hashes(); + let mut group = c.benchmark_group("sbbf_insert_3m"); + group.sample_size(10); + group.throughput(Throughput::Elements(ENTRY_COUNT as u64)); + + for (name, rate) in RATES { + let bits_per_entry = bloom_bits_per_entry(rate); + group.bench_with_input( + BenchmarkId::new("fpr", name), + &bits_per_entry, + |b, bits_per_entry| { + b.iter_batched( + || Sbbf::new(*bits_per_entry, ENTRY_COUNT), + |mut filter| { + for hash in hashes.iter().copied() { + black_box(filter.insert_hash(black_box(hash))); + } + black_box(filter); + }, + criterion::BatchSize::LargeInput, + ); + }, + ); + } + group.finish(); +} + +fn benchmark_sbbf_contains(c: &mut Criterion) { + let hashes = hashes(); + let mut group = c.benchmark_group("sbbf_contains_3m"); + group.sample_size(10); + group.throughput(Throughput::Elements(ENTRY_COUNT as u64)); + + for (name, rate) in RATES { + let mut filter = Sbbf::new(bloom_bits_per_entry(rate), ENTRY_COUNT); + for hash in hashes.iter().copied() { + filter.insert_hash(hash); + } + + group.bench_with_input(BenchmarkId::new("fpr", name), &filter, |b, filter| { + b.iter(|| { + let mut matches = 0_u64; + for hash in hashes.iter().copied() { + matches += u64::from(filter.contains_hash(black_box(hash))); + } + black_box(matches) + }); + }); + } + group.finish(); +} + +fn bloom_bits_per_entry(false_positive_rate: f64) -> usize { + let probes = 8.0; + (-probes / (1.0 - false_positive_rate.powf(1.0 / probes)).ln()).ceil() as usize +} + +fn splitmix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} + +criterion_group!( + benches, + benchmark_insert, + benchmark_get, + benchmark_sbbf_insert, + benchmark_sbbf_contains +); +criterion_main!(benches); diff --git a/crates/min-bloom-filter/src/lib.rs b/crates/min-bloom-filter/src/lib.rs new file mode 100644 index 0000000..e42c1da --- /dev/null +++ b/crates/min-bloom-filter/src/lib.rs @@ -0,0 +1,317 @@ +//! A split-block Bloom filter with four-bit values. +//! +//! Every block consists of eight [`u32`] words. Each word contains eight +//! nibbles, and a hash selects one nibble in every word. Inserting a value +//! raises each selected nibble to the maximum of its current and inserted +//! values. Retrieving a hash returns the minimum of its eight selected +//! nibbles. + +const WORDS_PER_BLOCK: usize = 8; +const MAX_VALUE: u8 = 0x0f; + +// The salts are the constants from the Parquet split-block Bloom filter. +const SALT: [u32; WORDS_PER_BLOCK] = [ + 0x47b6_137b, + 0x4497_4d91, + 0x8824_ad5b, + 0xa2b7_289d, + 0x7054_95c7, + 0x2df1_424b, + 0x9efc_4947, + 0x5c6b_fb31, +]; + +/// A split-block min Bloom filter. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MinBloomFilter { + blocks: Box<[[u32; WORDS_PER_BLOCK]]>, +} + +impl MinBloomFilter { + /// Creates a filter sized for `expected_entries` and `false_positive_rate`. + /// + /// The rate must be finite and strictly between zero and one. The filter + /// uses the standard Bloom-filter sizing equation with eight probes, then + /// rounds up to a whole 32-byte split block. + #[must_use] + pub fn new(expected_entries: usize, false_positive_rate: f64) -> Self { + assert!( + expected_entries > 0, + "expected_entries must be greater than zero" + ); + assert!( + false_positive_rate.is_finite() + && false_positive_rate > 0.0 + && false_positive_rate < 1.0, + "false_positive_rate must be finite and between zero and one" + ); + + let probes = WORDS_PER_BLOCK as f64; + let bits_per_entry = -probes / (1.0 - false_positive_rate.powf(1.0 / probes)).ln(); + let total_bits = + (expected_entries as f64 * bits_per_entry * f64::from(MAX_VALUE.ilog2() + 1)).ceil() + as usize; + let bits_per_block = WORDS_PER_BLOCK * u32::BITS as usize; + let block_count = total_bits.div_ceil(bits_per_block).max(1); + + Self { + blocks: vec![[0; WORDS_PER_BLOCK]; block_count].into_boxed_slice(), + } + } + + /// Creates an empty filter with exactly `block_count` split blocks. + #[must_use] + pub fn with_block_count(block_count: usize) -> Self { + assert!(block_count > 0, "block_count must be greater than zero"); + Self { + blocks: vec![[0; WORDS_PER_BLOCK]; block_count].into_boxed_slice(), + } + } + + /// Raises the eight nibbles selected by `hash` to at least `value`. + /// + /// # Panics + /// + /// Panics when `value` is greater than 15. + #[inline] + pub fn insert(&mut self, hash: u64, value: u8) { + assert!(value <= MAX_VALUE, "value must fit in a nibble"); + + let block_index = block_index(self.blocks.len(), hash); + let block = &mut self.blocks[block_index]; + insert_block(block, hash as u32, value); + } + + /// Returns the minimum of the eight nibbles selected by `hash`. + #[inline] + #[must_use] + pub fn get(&self, hash: u64) -> u8 { + let block = &self.blocks[block_index(self.blocks.len(), hash)]; + get_block(block, hash as u32) + } + + /// Returns the number of 32-byte split blocks in this filter. + #[inline] + #[must_use] + pub const fn block_count(&self) -> usize { + self.blocks.len() + } + + /// Returns the filter's allocated data size in bytes. + #[inline] + #[must_use] + pub const fn len_bytes(&self) -> usize { + self.blocks.len() * size_of::<[u32; WORDS_PER_BLOCK]>() + } + + /// Returns `true` when all nibbles are zero. + #[must_use] + pub fn is_empty(&self) -> bool { + self.blocks.iter().flatten().all(|word| *word == 0) + } + + /// Resets every nibble to zero. + pub fn clear(&mut self) { + self.blocks.fill([0; WORDS_PER_BLOCK]); + } +} + +#[inline] +fn block_index(block_count: usize, hash: u64) -> usize { + (((hash >> 32) * block_count as u64) >> 32) as usize +} + +#[cfg(any(not(target_arch = "aarch64"), test))] +#[inline] +fn nibble_shift(hash: u32, salt: u32) -> u32 { + (hash.wrapping_mul(salt) >> 29) * 4 +} + +#[cfg(target_arch = "aarch64")] +#[inline] +fn insert_block(block: &mut [u32; WORDS_PER_BLOCK], hash: u32, value: u8) { + // SAFETY: NEON is mandatory on AArch64, and `block` points to eight valid u32 values. + unsafe { neon::insert(block.as_mut_ptr(), hash, value) } +} + +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn insert_block(block: &mut [u32; WORDS_PER_BLOCK], hash: u32, value: u8) { + for (word, salt) in block.iter_mut().zip(SALT) { + let shift = nibble_shift(hash, salt); + let current = ((*word >> shift) & u32::from(MAX_VALUE)) as u8; + if current < value { + *word = (*word & !(u32::from(MAX_VALUE) << shift)) | (u32::from(value) << shift); + } + } +} + +#[cfg(target_arch = "aarch64")] +#[inline] +fn get_block(block: &[u32; WORDS_PER_BLOCK], hash: u32) -> u8 { + // SAFETY: NEON is mandatory on AArch64, and `block` points to eight valid u32 values. + unsafe { neon::get(block.as_ptr(), hash) } +} + +#[cfg(not(target_arch = "aarch64"))] +#[inline] +fn get_block(block: &[u32; WORDS_PER_BLOCK], hash: u32) -> u8 { + block + .iter() + .zip(SALT) + .map(|(word, salt)| { + let shift = nibble_shift(hash, salt); + ((word >> shift) & u32::from(MAX_VALUE)) as u8 + }) + .min() + .unwrap_or_default() +} + +#[cfg(target_arch = "aarch64")] +mod neon { + use core::arch::aarch64::{ + int32x4_t, uint32x4_t, vandq_u32, vbicq_u32, vdupq_n_u32, vld1q_u32, vmaxq_u32, vminq_u32, + vminvq_u32, vmulq_u32, vnegq_s32, vorrq_u32, vreinterpretq_s32_u32, vshlq_u32, vshrq_n_u32, + vst1q_u32, + }; + + use super::{MAX_VALUE, SALT}; + + #[target_feature(enable = "neon")] + #[inline] + unsafe fn shifts(hash: u32) -> (uint32x4_t, uint32x4_t) { + unsafe { + let hash = vdupq_n_u32(hash); + let low = vmulq_u32(vld1q_u32(SALT.as_ptr()), hash); + let high = vmulq_u32(vld1q_u32(SALT.as_ptr().add(4)), hash); + let four = vdupq_n_u32(4); + ( + vmulq_u32(vshrq_n_u32(low, 29), four), + vmulq_u32(vshrq_n_u32(high, 29), four), + ) + } + } + + #[target_feature(enable = "neon")] + #[inline] + unsafe fn selected_values(words: uint32x4_t, shifts: int32x4_t) -> uint32x4_t { + vandq_u32( + vshlq_u32(words, vnegq_s32(shifts)), + vdupq_n_u32(u32::from(MAX_VALUE)), + ) + } + + #[target_feature(enable = "neon")] + #[inline] + pub unsafe fn insert(block: *mut u32, hash: u32, value: u8) { + unsafe { + let shifts = shifts(hash); + let low_shifts = vreinterpretq_s32_u32(shifts.0); + let high_shifts = vreinterpretq_s32_u32(shifts.1); + let low = vld1q_u32(block); + let high = vld1q_u32(block.add(4)); + let nibble_mask = vdupq_n_u32(u32::from(MAX_VALUE)); + let value = vdupq_n_u32(u32::from(value)); + + let low_values = vmaxq_u32(selected_values(low, low_shifts), value); + let high_values = vmaxq_u32(selected_values(high, high_shifts), value); + let low_mask = vshlq_u32(nibble_mask, low_shifts); + let high_mask = vshlq_u32(nibble_mask, high_shifts); + let low_values = vshlq_u32(low_values, low_shifts); + let high_values = vshlq_u32(high_values, high_shifts); + + vst1q_u32(block, vorrq_u32(vbicq_u32(low, low_mask), low_values)); + vst1q_u32( + block.add(4), + vorrq_u32(vbicq_u32(high, high_mask), high_values), + ); + } + } + + #[target_feature(enable = "neon")] + #[inline] + pub unsafe fn get(block: *const u32, hash: u32) -> u8 { + unsafe { + let shifts = shifts(hash); + let low = selected_values(vld1q_u32(block), vreinterpretq_s32_u32(shifts.0)); + let high = selected_values(vld1q_u32(block.add(4)), vreinterpretq_s32_u32(shifts.1)); + vminvq_u32(vminq_u32(low, high)) as u8 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_filter_returns_zero() { + let filter = MinBloomFilter::with_block_count(1); + + assert_eq!(filter.get(42), 0); + assert!(filter.is_empty()); + } + + #[test] + fn insert_and_retrieve_value() { + let mut filter = MinBloomFilter::with_block_count(16); + + filter.insert(42, 7); + + assert_eq!(filter.get(42), 7); + assert!(!filter.is_empty()); + } + + #[test] + fn insert_only_raises_values() { + let mut filter = MinBloomFilter::with_block_count(1); + + filter.insert(42, 11); + filter.insert(42, 3); + assert_eq!(filter.get(42), 11); + + filter.insert(42, 15); + assert_eq!(filter.get(42), 15); + } + + #[test] + fn retrieval_uses_minimum_selected_nibble() { + let mut filter = MinBloomFilter::with_block_count(1); + let hash = 42; + let shifts = SALT.map(|salt| nibble_shift(hash as u32, salt)); + + for (index, (word, shift)) in filter.blocks[0].iter_mut().zip(shifts).enumerate() { + *word = ((index as u32 + 3) & u32::from(MAX_VALUE)) << shift; + } + + assert_eq!(filter.get(hash), 3); + } + + #[test] + fn clear_resets_filter() { + let mut filter = MinBloomFilter::with_block_count(2); + filter.insert(42, 9); + + filter.clear(); + + assert_eq!(filter.get(42), 0); + assert!(filter.is_empty()); + } + + #[test] + fn sizing_meets_requested_rate() { + let entries = 3_000_000; + let one_percent = MinBloomFilter::new(entries, 0.01); + let tenth_percent = MinBloomFilter::new(entries, 0.001); + + assert!(tenth_percent.len_bytes() > one_percent.len_bytes()); + assert_eq!(one_percent.len_bytes() % 32, 0); + assert_eq!(tenth_percent.len_bytes() % 32, 0); + } + + #[test] + #[should_panic(expected = "value must fit in a nibble")] + fn rejects_values_larger_than_a_nibble() { + MinBloomFilter::with_block_count(1).insert(42, 16); + } +} From 51fbe9a91811b4cf61c1b7ff1824aa6f0ed57cd4 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 5 Aug 2026 12:59:40 +0200 Subject: [PATCH 2/2] Correct split-block filter sizing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ab0460bc-ae55-4457-a8e0-00169d9c2692 --- crates/min-bloom-filter/README.md | 74 ++++++++++++++++++-- crates/min-bloom-filter/examples/accuracy.rs | 70 ++++++++++++++++++ crates/min-bloom-filter/src/lib.rs | 44 ++++++++++-- 3 files changed, 177 insertions(+), 11 deletions(-) create mode 100644 crates/min-bloom-filter/examples/accuracy.rs diff --git a/crates/min-bloom-filter/README.md b/crates/min-bloom-filter/README.md index bd86032..097df4f 100644 --- a/crates/min-bloom-filter/README.md +++ b/crates/min-bloom-filter/README.md @@ -19,21 +19,87 @@ assert_eq!(filter.get(0x1234_5678_9abc_def0), 7); Values must be in `0..=15`. Like a regular Bloom filter, collisions can produce false positives: a key that was not inserted can retrieve a nonzero value. +## False-positive probability and sizing + +Let `n` be the number of inserted keys, `B` the number of blocks, and +`lambda = n / B` the average number of keys assigned to a block. Each block has +eight words with eight nibbles per word. + +For a block containing exactly `x` keys, the probability that a selected nibble +in one word is nonzero is: + +```text +q(x) = 1 - (7 / 8)^x +``` + +A false positive requires the selected nibble in all eight words to be nonzero: + +```text +P(FP | X = x) = (1 - (7 / 8)^x)^8 +``` + +Block occupancy is approximately Poisson distributed, `X ~ Poisson(lambda)`. +The filter therefore uses the split-block false-positive model: + +```text +P(FP) = sum(x = 0..infinity) [ + exp(-lambda) * lambda^x / x! * (1 - (7 / 8)^x)^8 +] +``` + +This differs from the ordinary Bloom-filter approximation +`(1 - exp(-lambda / 8))^8`, which substitutes mean occupancy for the random +block occupancy. Because false-positive probability is nonlinear, overloaded +blocks contribute more false positives than underloaded blocks compensate for. + +Numerically solving the split-block expression gives: + +| Target FPP | Nibbles/entry | Memory for 3M entries | +|---|---:|---:| +| 1% | 13.43 | 20.15 MB | +| 0.1% | 25.34 | 38.01 MB | + +Each nibble occupies four bits, so the byte size is +`n * nibbles_per_entry / 2`. + +For comparison, an optimally sized ordinary Bloom filter needs: + +| Target FPP | Bits/entry | Memory for 3M entries | +|---|---:|---:| +| 1% | 9.59 | 3.59 MB | +| 0.1% | 14.38 | 5.39 MB | + ## Performance Criterion results on an Apple M4 Max for 3 million entries: | Target FPR | Memory | Insert | Retrieve | |---|---:|---:|---:| -| 1% | 14.52 MB | 2.27 ns/entry | 1.42 ns/entry | -| 0.1% | 21.91 MB | 2.74 ns/entry | 1.58 ns/entry | +| 1% | 20.15 MB | 2.71 ns/entry | 1.63 ns/entry | +| 0.1% | 38.01 MB | 5.02 ns/entry | 3.08 ns/entry | For comparison, `sbbf-rs-safe` on the same hashes: | Target FPR | Insert | Contains | |---|---:|---:| -| 1% | 1.50 ns/entry | 1.24 ns/entry | -| 0.1% | 1.55 ns/entry | 1.26 ns/entry | +| 1% | 1.52 ns/entry | 1.23 ns/entry | +| 0.1% | 1.57 ns/entry | 1.26 ns/entry | + +The empirical nonzero false-positive rates over 10 million absent keys were +0.9941% and 0.0991%, respectively. + +## Value accuracy + +An accuracy evaluation inserts 3 million values drawn from geometric +distributions and then queries inserted and absent keys. With either ratio +(`1, 0.5, 0.25, ...` or `1, 0.2, 0.04, ...`), all 10 million queries for +inserted keys returned the exact value: no overestimation was observed. + +Run the evaluation with: + +```console +cargo run --release -p min-bloom-filter --example accuracy +``` Run these benchmarks with: diff --git a/crates/min-bloom-filter/examples/accuracy.rs b/crates/min-bloom-filter/examples/accuracy.rs new file mode 100644 index 0000000..cafeb0b --- /dev/null +++ b/crates/min-bloom-filter/examples/accuracy.rs @@ -0,0 +1,70 @@ +use min_bloom_filter::MinBloomFilter; + +const ENTRY_COUNT: usize = 3_000_000; +const QUERY_COUNT: usize = 10_000_000; +const RATES: [(&str, f64); 2] = [("1%", 0.01), ("0.1%", 0.001)]; +const GEOMETRIC_RATIOS: [f64; 2] = [0.5, 0.2]; + +fn main() { + println!( + "| Target FPP | Ratio | Nonzero FPP | Exact | Overestimated | Mean excess | Mean relative excess |" + ); + println!("|---|---:|---:|---:|---:|---:|---:|"); + + for (rate_name, rate) in RATES { + for ratio in GEOMETRIC_RATIOS { + let mut filter = MinBloomFilter::new(ENTRY_COUNT, rate); + for index in 0..ENTRY_COUNT { + filter.insert(hash(index), geometric_value(hash(index), ratio)); + } + + let mut false_positives = 0_usize; + let mut exact = 0_usize; + let mut overestimated = 0_usize; + let mut excess = 0_u64; + let mut relative_excess = 0.0; + + for index in 0..QUERY_COUNT { + let key = index % ENTRY_COUNT; + let key_hash = hash(key); + let true_value = geometric_value(key_hash, ratio); + let estimate = filter.get(key_hash); + if estimate == true_value { + exact += 1; + } else { + overestimated += 1; + excess += u64::from(estimate - true_value); + relative_excess += f64::from(estimate - true_value) / f64::from(true_value); + } + + false_positives += usize::from(filter.get(hash(ENTRY_COUNT + index)) > 0); + } + + println!( + "| {rate_name} | {ratio:.1} | {:.4}% | {:.4}% | {:.4}% | {:.4} | {:.4}% |", + false_positives as f64 / QUERY_COUNT as f64 * 100.0, + exact as f64 / QUERY_COUNT as f64 * 100.0, + overestimated as f64 / QUERY_COUNT as f64 * 100.0, + excess as f64 / QUERY_COUNT as f64, + relative_excess / QUERY_COUNT as f64 * 100.0, + ); + } + } +} + +fn geometric_value(hash: u64, ratio: f64) -> u8 { + let uniform = ((hash >> 11) as f64 + 0.5) / ((1_u64 << 53) as f64); + let level = (uniform.ln() / ratio.ln()).floor() as u32; + (level + 1).min(15) as u8 +} + +fn hash(index: usize) -> u64 { + splitmix64(index as u64) +} + +fn splitmix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} diff --git a/crates/min-bloom-filter/src/lib.rs b/crates/min-bloom-filter/src/lib.rs index e42c1da..7411a88 100644 --- a/crates/min-bloom-filter/src/lib.rs +++ b/crates/min-bloom-filter/src/lib.rs @@ -46,13 +46,8 @@ impl MinBloomFilter { "false_positive_rate must be finite and between zero and one" ); - let probes = WORDS_PER_BLOCK as f64; - let bits_per_entry = -probes / (1.0 - false_positive_rate.powf(1.0 / probes)).ln(); - let total_bits = - (expected_entries as f64 * bits_per_entry * f64::from(MAX_VALUE.ilog2() + 1)).ceil() - as usize; - let bits_per_block = WORDS_PER_BLOCK * u32::BITS as usize; - let block_count = total_bits.div_ceil(bits_per_block).max(1); + let block_count = + (expected_entries as f64 / mean_block_occupancy(false_positive_rate)).ceil() as usize; Self { blocks: vec![[0; WORDS_PER_BLOCK]; block_count].into_boxed_slice(), @@ -116,6 +111,39 @@ impl MinBloomFilter { } } +fn mean_block_occupancy(false_positive_rate: f64) -> f64 { + let mut low = 0.0; + let mut high = 64.0; + while split_block_false_positive_probability(high) < false_positive_rate { + high *= 2.0; + } + for _ in 0..64 { + let middle = (low + high) / 2.0; + if split_block_false_positive_probability(middle) < false_positive_rate { + low = middle; + } else { + high = middle; + } + } + low +} + +fn split_block_false_positive_probability(mean_occupancy: f64) -> f64 { + let mut probability = (-mean_occupancy).exp(); + let mut total = 0.0; + let mut occupancy = 0_u32; + loop { + let selected = 1.0 - (7.0_f64 / 8.0).powf(f64::from(occupancy)); + total += probability * selected.powi(WORDS_PER_BLOCK as i32); + + occupancy += 1; + probability *= mean_occupancy / f64::from(occupancy); + if probability < f64::EPSILON && occupancy as f64 > mean_occupancy { + return total; + } + } +} + #[inline] fn block_index(block_count: usize, hash: u64) -> usize { (((hash >> 32) * block_count as u64) >> 32) as usize @@ -304,6 +332,8 @@ mod tests { let one_percent = MinBloomFilter::new(entries, 0.01); let tenth_percent = MinBloomFilter::new(entries, 0.001); + assert_eq!(one_percent.len_bytes(), 20_149_216); + assert_eq!(tenth_percent.len_bytes(), 38_009_632); assert!(tenth_percent.len_bytes() > one_percent.len_bytes()); assert_eq!(one_percent.len_bytes() % 32, 0); assert_eq!(tenth_percent.len_bytes() % 32, 0);