Add min bloom filter crate

README.md

@@ -10,6 +10,7 @@ A collection of useful algorithms written in Rust. Currently contains:
 - [`sparse-ngrams`](crates/sparse-ngrams): fast sparse n-gram extraction from byte slices. Selects variable-length n-grams (2–8 bytes) deterministically using bigram frequency priorities, suitable for substring search indexes.
 - [`string-offsets`](crates/string-offsets): converts string positions between bytes, chars, UTF-16 code units, and line numbers. Useful when sending string indices across language boundaries.
 - [`casefold`](crates/casefold): a **fast** Unicode simple case-folding library backed by a **very compact** (~1.7 KB) paged-bitmap + run-length table. Folds whole strings at multiple GiB/s via a decode-free `simple_fold` that rewrites UTF-8 with little-endian byte arithmetic, beating a `HashMap` fold table by several × at ~10× less memory.
+- [`min-bloom-filter`](crates/min-bloom-filter): a split-block Bloom filter with four-bit values, max insertion, and min retrieval.
 
 ## Background
 

crates/min-bloom-filter/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "min-bloom-filter"
+version = "0.1.0"
+edition = "2024"
+description = "A split-block Bloom filter whose four-bit counters support max insertion and min retrieval."
+repository = "https://github.com/github/rust-gems"
+license = "MIT"
+keywords = ["bloom-filter", "probabilistic", "filter", "data-structure"]
+categories = ["algorithms", "data-structures"]
+
+[lib]
+bench = false
+
+[[bench]]
+name = "performance"
+path = "benchmarks/performance.rs"
+harness = false
+
+[dev-dependencies]
+criterion = "0.8"
+sbbf-rs-safe = "0.3"

crates/min-bloom-filter/README.md

@@ -0,0 +1,108 @@
+# min-bloom-filter
+
+A split-block Bloom filter that stores four-bit values instead of bits.
+
+Each 32-byte block contains eight `u32` words, and each word contains eight
+nibbles. A hash selects one nibble per word. Inserting raises all eight selected
+nibbles to the maximum of their current value and the inserted value; retrieval
+returns their minimum.
+
+```rust
+use min_bloom_filter::MinBloomFilter;
+
+let mut filter = MinBloomFilter::new(3_000_000, 0.01);
+filter.insert(0x1234_5678_9abc_def0, 7);
+
+assert_eq!(filter.get(0x1234_5678_9abc_def0), 7);
+```
+
+Values must be in `0..=15`. Like a regular Bloom filter, collisions can produce
+false positives: a key that was not inserted can retrieve a nonzero value.
+
+## False-positive probability and sizing
+
+Let `n` be the number of inserted keys, `B` the number of blocks, and
+`lambda = n / B` the average number of keys assigned to a block. Each block has
+eight words with eight nibbles per word.
+
+For a block containing exactly `x` keys, the probability that a selected nibble
+in one word is nonzero is:
+
+```text
+q(x) = 1 - (7 / 8)^x
+```
+
+A false positive requires the selected nibble in all eight words to be nonzero:
+
+```text
+P(FP | X = x) = (1 - (7 / 8)^x)^8
+```
+
+Block occupancy is approximately Poisson distributed, `X ~ Poisson(lambda)`.
+The filter therefore uses the split-block false-positive model:
+
+```text
+P(FP) = sum(x = 0..infinity) [
+    exp(-lambda) * lambda^x / x! * (1 - (7 / 8)^x)^8
+]
+```
+
+This differs from the ordinary Bloom-filter approximation
+`(1 - exp(-lambda / 8))^8`, which substitutes mean occupancy for the random
+block occupancy. Because false-positive probability is nonlinear, overloaded
+blocks contribute more false positives than underloaded blocks compensate for.
+
+Numerically solving the split-block expression gives:
+
+| Target FPP | Nibbles/entry | Memory for 3M entries |
+|---|---:|---:|
+| 1% | 13.43 | 20.15 MB |
+| 0.1% | 25.34 | 38.01 MB |
+
+Each nibble occupies four bits, so the byte size is
+`n * nibbles_per_entry / 2`.
+
+For comparison, an optimally sized ordinary Bloom filter needs:
+
+| Target FPP | Bits/entry | Memory for 3M entries |
+|---|---:|---:|
+| 1% | 9.59 | 3.59 MB |
+| 0.1% | 14.38 | 5.39 MB |
+
+## Performance
+
+Criterion results on an Apple M4 Max for 3 million entries:
+
+| Target FPR | Memory | Insert | Retrieve |
+|---|---:|---:|---:|
+| 1% | 20.15 MB | 2.71 ns/entry | 1.63 ns/entry |
+| 0.1% | 38.01 MB | 5.02 ns/entry | 3.08 ns/entry |
+
+For comparison, `sbbf-rs-safe` on the same hashes:
+
+| Target FPR | Insert | Contains |
+|---|---:|---:|
+| 1% | 1.52 ns/entry | 1.23 ns/entry |
+| 0.1% | 1.57 ns/entry | 1.26 ns/entry |
+
+The empirical nonzero false-positive rates over 10 million absent keys were
+0.9941% and 0.0991%, respectively.
+
+## Value accuracy
+
+An accuracy evaluation inserts 3 million values drawn from geometric
+distributions and then queries inserted and absent keys. With either ratio
+(`1, 0.5, 0.25, ...` or `1, 0.2, 0.04, ...`), all 10 million queries for
+inserted keys returned the exact value: no overestimation was observed.
+
+Run the evaluation with:
+
+```console
+cargo run --release -p min-bloom-filter --example accuracy
+```
+
+Run these benchmarks with:
+
+```console
+cargo bench -p min-bloom-filter --bench performance
+```

crates/min-bloom-filter/benchmarks/performance.rs

@@ -0,0 +1,136 @@
+use std::hint::black_box;
+
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use min_bloom_filter::MinBloomFilter;
+use sbbf_rs_safe::Filter as Sbbf;
+
+const ENTRY_COUNT: usize = 3_000_000;
+const RATES: [(&str, f64); 2] = [("1_percent", 0.01), ("0.1_percent", 0.001)];
+
+fn hashes() -> Vec<u64> {
+    (0..ENTRY_COUNT)
+        .map(|index| splitmix64(index as u64))
+        .collect()
+}
+
+fn benchmark_insert(c: &mut Criterion) {
+    let hashes = hashes();
+    let mut group = c.benchmark_group("insert_3m");
+    group.sample_size(10);
+    group.throughput(Throughput::Elements(ENTRY_COUNT as u64));
+
+    for (name, rate) in RATES {
+        group.bench_with_input(BenchmarkId::new("fpr", name), &rate, |b, rate| {
+            b.iter_batched(
+                || MinBloomFilter::new(ENTRY_COUNT, *rate),
+                |mut filter| {
+                    for (index, hash) in hashes.iter().copied().enumerate() {
+                        filter.insert(black_box(hash), ((index % 15) + 1) as u8);
+                    }
+                    black_box(filter);
+                },
+                criterion::BatchSize::LargeInput,
+            );
+        });
+    }
+    group.finish();
+}
+
+fn benchmark_get(c: &mut Criterion) {
+    let hashes = hashes();
+    let mut group = c.benchmark_group("get_3m");
+    group.sample_size(10);
+    group.throughput(Throughput::Elements(ENTRY_COUNT as u64));
+
+    for (name, rate) in RATES {
+        let mut filter = MinBloomFilter::new(ENTRY_COUNT, rate);
+        for (index, hash) in hashes.iter().copied().enumerate() {
+            filter.insert(hash, ((index % 15) + 1) as u8);
+        }
+
+        group.bench_with_input(BenchmarkId::new("fpr", name), &filter, |b, filter| {
+            b.iter(|| {
+                let mut sum = 0_u64;
+                for hash in hashes.iter().copied() {
+                    sum += u64::from(filter.get(black_box(hash)));
+                }
+                black_box(sum)
+            });
+        });
+    }
+    group.finish();
+}
+
+fn benchmark_sbbf_insert(c: &mut Criterion) {
+    let hashes = hashes();
+    let mut group = c.benchmark_group("sbbf_insert_3m");
+    group.sample_size(10);
+    group.throughput(Throughput::Elements(ENTRY_COUNT as u64));
+
+    for (name, rate) in RATES {
+        let bits_per_entry = bloom_bits_per_entry(rate);
+        group.bench_with_input(
+            BenchmarkId::new("fpr", name),
+            &bits_per_entry,
+            |b, bits_per_entry| {
+                b.iter_batched(
+                    || Sbbf::new(*bits_per_entry, ENTRY_COUNT),
+                    |mut filter| {
+                        for hash in hashes.iter().copied() {
+                            black_box(filter.insert_hash(black_box(hash)));
+                        }
+                        black_box(filter);
+                    },
+                    criterion::BatchSize::LargeInput,
+                );
+            },
+        );
+    }
+    group.finish();
+}
+
+fn benchmark_sbbf_contains(c: &mut Criterion) {
+    let hashes = hashes();
+    let mut group = c.benchmark_group("sbbf_contains_3m");
+    group.sample_size(10);
+    group.throughput(Throughput::Elements(ENTRY_COUNT as u64));
+
+    for (name, rate) in RATES {
+        let mut filter = Sbbf::new(bloom_bits_per_entry(rate), ENTRY_COUNT);
+        for hash in hashes.iter().copied() {
+            filter.insert_hash(hash);
+        }
+
+        group.bench_with_input(BenchmarkId::new("fpr", name), &filter, |b, filter| {
+            b.iter(|| {
+                let mut matches = 0_u64;
+                for hash in hashes.iter().copied() {
+                    matches += u64::from(filter.contains_hash(black_box(hash)));
+                }
+                black_box(matches)
+            });
+        });
+    }
+    group.finish();
+}
+
+fn bloom_bits_per_entry(false_positive_rate: f64) -> usize {
+    let probes = 8.0;
+    (-probes / (1.0 - false_positive_rate.powf(1.0 / probes)).ln()).ceil() as usize
+}
+
+fn splitmix64(mut value: u64) -> u64 {
+    value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
+    value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
+    value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
+    value ^ (value >> 31)
+}
+
+criterion_group!(
+    benches,
+    benchmark_insert,
+    benchmark_get,
+    benchmark_sbbf_insert,
+    benchmark_sbbf_contains
+);
+criterion_main!(benches);

crates/min-bloom-filter/examples/accuracy.rs

@@ -0,0 +1,70 @@
+use min_bloom_filter::MinBloomFilter;
+
+const ENTRY_COUNT: usize = 3_000_000;
+const QUERY_COUNT: usize = 10_000_000;
+const RATES: [(&str, f64); 2] = [("1%", 0.01), ("0.1%", 0.001)];
+const GEOMETRIC_RATIOS: [f64; 2] = [0.5, 0.2];
+
+fn main() {
+    println!(
+        "| Target FPP | Ratio | Nonzero FPP | Exact | Overestimated | Mean excess | Mean relative excess |"
+    );
+    println!("|---|---:|---:|---:|---:|---:|---:|");
+
+    for (rate_name, rate) in RATES {
+        for ratio in GEOMETRIC_RATIOS {
+            let mut filter = MinBloomFilter::new(ENTRY_COUNT, rate);
+            for index in 0..ENTRY_COUNT {
+                filter.insert(hash(index), geometric_value(hash(index), ratio));
+            }
+
+            let mut false_positives = 0_usize;
+            let mut exact = 0_usize;
+            let mut overestimated = 0_usize;
+            let mut excess = 0_u64;
+            let mut relative_excess = 0.0;
+
+            for index in 0..QUERY_COUNT {
+                let key = index % ENTRY_COUNT;
+                let key_hash = hash(key);
+                let true_value = geometric_value(key_hash, ratio);
+                let estimate = filter.get(key_hash);
+                if estimate == true_value {
+                    exact += 1;
+                } else {
+                    overestimated += 1;
+                    excess += u64::from(estimate - true_value);
+                    relative_excess += f64::from(estimate - true_value) / f64::from(true_value);
+                }
+
+                false_positives += usize::from(filter.get(hash(ENTRY_COUNT + index)) > 0);
+            }
+
+            println!(
+                "| {rate_name} | {ratio:.1} | {:.4}% | {:.4}% | {:.4}% | {:.4} | {:.4}% |",
+                false_positives as f64 / QUERY_COUNT as f64 * 100.0,
+                exact as f64 / QUERY_COUNT as f64 * 100.0,
+                overestimated as f64 / QUERY_COUNT as f64 * 100.0,
+                excess as f64 / QUERY_COUNT as f64,
+                relative_excess / QUERY_COUNT as f64 * 100.0,
+            );
+        }
+    }
+}
+
+fn geometric_value(hash: u64, ratio: f64) -> u8 {
+    let uniform = ((hash >> 11) as f64 + 0.5) / ((1_u64 << 53) as f64);
+    let level = (uniform.ln() / ratio.ln()).floor() as u32;
+    (level + 1).min(15) as u8
+}
+
+fn hash(index: usize) -> u64 {
+    splitmix64(index as u64)
+}
+
+fn splitmix64(mut value: u64) -> u64 {
+    value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
+    value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
+    value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
+    value ^ (value >> 31)
+}