summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Cargo.toml3
-rw-r--r--src/collector/facet_collector.rs16
-rw-r--r--src/collector/mod.rs11
-rw-r--r--src/common/bitset.rs12
-rw-r--r--src/compression/mod.rs10
-rw-r--r--src/datastruct/stacker/expull.rs10
-rw-r--r--src/datastruct/stacker/hashmap.rs67
-rw-r--r--src/fastfield/mod.rs13
-rw-r--r--src/lib.rs6
-rw-r--r--src/postings/mod.rs144
-rw-r--r--src/query/bitset/mod.rs16
-rw-r--r--src/query/phrase_query/phrase_scorer.rs9
-rw-r--r--src/query/union.rs10
-rw-r--r--src/store/mod.rs10
14 files changed, 217 insertions, 120 deletions
diff --git a/Cargo.toml b/Cargo.toml
index 847ff55..01c6eae 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,7 +40,7 @@ stable_deref_trait = "1.0.0"
rust-stemmers = "0.1.0"
downcast = { version="0.9" }
matches = "0.1"
-bitpacking = {path="../bitpacking"}
+bitpacking = "0.4"
[dev-dependencies]
rand = "0.3"
@@ -57,6 +57,7 @@ default = ["mmap"]
simd = ["bitpacking/simd"]
streamdict = []
mmap = ["fst/mmap", "atomicwrites"]
+unstable = ["simd"]
[badges]
travis-ci = { repository = "tantivy-search/tantivy" }
diff --git a/src/collector/facet_collector.rs b/src/collector/facet_collector.rs
index bc0c46c..494b56d 100644
--- a/src/collector/facet_collector.rs
+++ b/src/collector/facet_collector.rs
@@ -13,6 +13,7 @@ use termdict::TermStreamerBuilder;
use std::collections::BTreeSet;
use termdict::TermMerger;
use docset::SkipResult;
+use std::collections::btree_map;
use std::{usize, u64};
use std::iter::Peekable;
@@ -433,9 +434,7 @@ pub struct FacetCounts {
}
-use std::collections::btree_map;
-
-struct FacetChildIterator<'a> {
+pub struct FacetChildIterator<'a> {
underlying: btree_map::Range<'a, Facet, u64>,
}
@@ -501,7 +500,6 @@ impl FacetCounts {
#[cfg(test)]
mod tests {
- use test::Bencher;
use core::Index;
use schema::{Document, Facet, SchemaBuilder};
use query::AllQuery;
@@ -622,6 +620,14 @@ mod tests {
}
}
+}
+
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+ use test::Bencher;
+
#[bench]
fn bench_facet_collector(b: &mut Bencher) {
let mut schema_builder = SchemaBuilder::new();
@@ -652,4 +658,4 @@ mod tests {
searcher.search(&AllQuery, &mut facet_collector).unwrap();
});
}
-}
+} \ No newline at end of file
diff --git a/src/collector/mod.rs b/src/collector/mod.rs
index f905e52..660f878 100644
--- a/src/collector/mod.rs
+++ b/src/collector/mod.rs
@@ -89,7 +89,6 @@ impl<'a, C: Collector> Collector for &'a mut C {
pub mod tests {
use super::*;
- use test::Bencher;
use DocId;
use Score;
use core::SegmentReader;
@@ -186,6 +185,14 @@ pub mod tests {
}
}
+}
+
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+ use test::Bencher;
+
#[bench]
fn build_collector(b: &mut Bencher) {
b.iter(|| {
@@ -197,4 +204,4 @@ pub mod tests {
count_collector.count()
});
}
-}
+} \ No newline at end of file
diff --git a/src/common/bitset.rs b/src/common/bitset.rs
index baf68d0..76e3ad5 100644
--- a/src/common/bitset.rs
+++ b/src/common/bitset.rs
@@ -202,7 +202,6 @@ impl BitSet {
#[cfg(test)]
mod tests {
- extern crate test;
use tests;
use std::collections::HashSet;
use super::BitSet;
@@ -353,6 +352,14 @@ mod tests {
assert!(!bitset.contains(el));
}
}
+}
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+ use test;
+ use super::TinySet;
+ use super::BitSet;
#[bench]
fn bench_tinyset_pop(b: &mut test::Bencher) {
@@ -385,5 +392,4 @@ mod tests {
fn bench_bitset_initialize(b: &mut test::Bencher) {
b.iter(|| BitSet::with_max_value(1_000_000));
}
-
-}
+} \ No newline at end of file
diff --git a/src/compression/mod.rs b/src/compression/mod.rs
index c4f5294..dbf9de8 100644
--- a/src/compression/mod.rs
+++ b/src/compression/mod.rs
@@ -174,8 +174,6 @@ impl VIntDecoder for BlockDecoder {
pub mod tests {
use super::*;
- use tests;
- use test::Bencher;
#[test]
fn test_encode_sorted_block() {
@@ -264,6 +262,13 @@ pub mod tests {
}
}
}
+}
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+ use super::*;
+ use test::Bencher;
#[bench]
fn bench_compress(b: &mut Bencher) {
@@ -320,5 +325,4 @@ pub mod tests {
decoder.uncompress_vint_sorted(compressed, 0u32, NUM_INTS_BENCH_VINT);
});
}
-
}
diff --git a/src/datastruct/stacker/expull.rs b/src/datastruct/stacker/expull.rs
index 6dac184..eb1fe7a 100644
--- a/src/datastruct/stacker/expull.rs
+++ b/src/datastruct/stacker/expull.rs
@@ -101,7 +101,6 @@ mod tests {
use super::*;
use super::super::heap::Heap;
- use test::Bencher;
const NUM_STACK: usize = 10_000;
const STACK_SIZE: u32 = 1000;
@@ -124,6 +123,13 @@ mod tests {
}
}
+
+}
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+ use test::Bencher;
+
#[bench]
fn bench_push_vec(bench: &mut Bencher) {
bench.iter(|| {
@@ -158,4 +164,4 @@ mod tests {
heap.clear();
});
}
-}
+} \ No newline at end of file
diff --git a/src/datastruct/stacker/hashmap.rs b/src/datastruct/stacker/hashmap.rs
index f0c17d5..16bb4d6 100644
--- a/src/datastruct/stacker/hashmap.rs
+++ b/src/datastruct/stacker/hashmap.rs
@@ -131,6 +131,28 @@ impl QuadraticProbing {
}
}
+use std::slice;
+
+pub struct Iter<'a: 'b, 'b> {
+ hashmap: &'b TermHashMap<'a>,
+ inner: slice::Iter<'a, usize>
+}
+
+impl<'a, 'b> Iterator for Iter<'a, 'b> {
+ type Item = (&'b [u8], u32, UnorderedTermId);
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.inner
+ .next()
+ .cloned()
+ .map(move |bucket: usize| {
+ let kv = self.hashmap.table[bucket];
+ let (key, offset): (&'b [u8], u32) = self.hashmap.get_key_value(kv.key_value_addr);
+ (key, offset, bucket as UnorderedTermId)
+ })
+ }
+}
+
impl<'a> TermHashMap<'a> {
pub fn new(num_bucket_power_of_2: usize, heap: &'a Heap) -> TermHashMap<'a> {
let table_size = 1 << num_bucket_power_of_2;
@@ -165,12 +187,11 @@ impl<'a> TermHashMap<'a> {
};
}
- pub fn iter<'b: 'a>(&'b self) -> impl Iterator<Item = (&'a [u8], u32, UnorderedTermId)> + 'b {
- self.occupied.iter().cloned().map(move |bucket: usize| {
- let kv = self.table[bucket];
- let (key, offset) = self.get_key_value(kv.key_value_addr);
- (key, offset, bucket as UnorderedTermId)
- })
+ pub fn iter<'b: 'a>(&'b self) -> Iter<'a, 'b> {
+ Iter {
+ inner: self.occupied.iter(),
+ hashmap: &self
+ }
}
pub fn get_or_create<S: AsRef<[u8]>, V: HeapAllocable>(
@@ -202,13 +223,32 @@ impl<'a> TermHashMap<'a> {
}
}
+#[cfg(all(test, unstable))]
+mod bench {
+ use test::Bencher;
+ use super::murmurhash2::murmurhash2;
+
+ #[bench]
+ fn bench_murmurhash_2(b: &mut Bencher) {
+ let keys: Vec<&'static str> =
+ vec!["wer qwe qwe qwe ", "werbq weqweqwe2 ", "weraq weqweqwe3 "];
+ b.iter(|| {
+ keys.iter()
+ .map(|&s| s.as_bytes())
+ .map(murmurhash2)
+ .map(|h| h as u64)
+ .last()
+ .unwrap()
+ });
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
use super::super::heap::{Heap, HeapAllocable};
use super::murmurhash2::murmurhash2;
- use test::Bencher;
use std::collections::HashSet;
use super::split_memory;
@@ -292,18 +332,5 @@ mod tests {
assert_eq!(set.len(), 10_000);
}
- #[bench]
- fn bench_murmurhash_2(b: &mut Bencher) {
- let keys: Vec<&'static str> =
- vec!["wer qwe qwe qwe ", "werbq weqweqwe2 ", "weraq weqweqwe3 "];
- b.iter(|| {
- keys.iter()
- .map(|&s| s.as_bytes())
- .map(murmurhash2::murmurhash2)
- .map(|h| h as u64)
- .last()
- .unwrap()
- });
- }
}
diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs
index 5142163..9e24c17 100644
--- a/src/fastfield/mod.rs
+++ b/src/fastfield/mod.rs
@@ -134,8 +134,6 @@ mod tests {
use std::collections::HashMap;
use std::path::Path;
use super::*;
- use test;
- use test::Bencher;
lazy_static! {
static ref SCHEMA: Schema = {
@@ -409,6 +407,14 @@ mod tests {
}
}
+}
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+
+ use test::{self, Bencher};
+
#[bench]
fn bench_intfastfield_linear_veclookup(b: &mut Bencher) {
let permutation = generate_permutation();
@@ -502,4 +508,5 @@ mod tests {
});
}
}
-}
+
+} \ No newline at end of file
diff --git a/src/lib.rs b/src/lib.rs
index e18f14f..42acf1d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2,8 +2,7 @@
#![cfg_attr(feature = "cargo-clippy", allow(module_inception))]
#![cfg_attr(feature = "cargo-clippy", allow(inline_always))]
-#![cfg_attr(test, feature(test))]
-#![cfg_attr(test, feature(iterator_step_by))]
+#![cfg_attr(all(feature="unstable", test), feature(test))]
#![doc(test(attr(allow(unused_variables), deny(warnings))))]
#![allow(unknown_lints)]
#![allow(new_without_default)]
@@ -160,7 +159,8 @@ extern crate winapi;
#[cfg(test)]
extern crate rand;
-#[cfg(test)]
+
+#[cfg(all(test, feature="unstable"))]
extern crate test;
extern crate tinysegmenter;
diff --git a/src/postings/mod.rs b/src/postings/mod.rs
index ab33987..fca9b53 100644
--- a/src/postings/mod.rs
+++ b/src/postings/mod.rs
@@ -52,7 +52,6 @@ pub mod tests {
use std::iter;
use datastruct::stacker::Heap;
use schema::Field;
- use test::{self, Bencher};
use indexer::operation::AddOperation;
use tests;
use rand::{Rng, SeedableRng, XorShiftRng};
@@ -530,6 +529,80 @@ pub mod tests {
};
}
+ /// Wraps a given docset, and forward alls call but the
+ /// `.skip_next(...)`. This is useful to test that a specialized
+ /// implementation of `.skip_next(...)` is consistent
+ /// with the default implementation.
+ pub(crate) struct UnoptimizedDocSet<TDocSet: DocSet>(TDocSet);
+
+ impl<TDocSet: DocSet> UnoptimizedDocSet<TDocSet> {
+ pub fn wrap(docset: TDocSet) -> UnoptimizedDocSet<TDocSet> {
+ UnoptimizedDocSet(docset)
+ }
+ }
+
+ impl<TDocSet: DocSet> DocSet for UnoptimizedDocSet<TDocSet> {
+ fn advance(&mut self) -> bool {
+ self.0.advance()
+ }
+
+ fn doc(&self) -> DocId {
+ self.0.doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.0.size_hint()
+ }
+ }
+
+ impl<TScorer: Scorer> Scorer for UnoptimizedDocSet<TScorer> {
+ fn score(&mut self) -> Score {
+ self.0.score()
+ }
+ }
+
+ pub fn test_skip_against_unoptimized<F: Fn() -> Box<DocSet>>(
+ postings_factory: F,
+ targets: Vec<u32>,
+ ) {
+ for target in targets {
+ let mut postings_opt = postings_factory();
+ let mut postings_unopt = UnoptimizedDocSet::wrap(postings_factory());
+ let skip_result_opt = postings_opt.skip_next(target);
+ let skip_result_unopt = postings_unopt.skip_next(target);
+ assert_eq!(
+ skip_result_unopt, skip_result_opt,
+ "Failed while skipping to {}",
+ target
+ );
+ match skip_result_opt {
+ SkipResult::Reached => assert_eq!(postings_opt.doc(), target),
+ SkipResult::OverStep => assert!(postings_opt.doc() > target),
+ SkipResult::End => {
+ return;
+ }
+ }
+ while postings_opt.advance() {
+ assert!(postings_unopt.advance());
+ assert_eq!(
+ postings_opt.doc(),
+ postings_unopt.doc(),
+ "Failed while skipping to {}",
+ target
+ );
+ }
+ assert!(!postings_unopt.advance());
+ }
+ }
+
+}
+
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+ use test::{self, Bencher};
+
#[bench]
fn bench_segment_postings(b: &mut Bencher) {
let searcher = INDEX.searcher();
@@ -646,71 +719,4 @@ pub mod tests {
s
});
}
-
- /// Wraps a given docset, and forward alls call but the
- /// `.skip_next(...)`. This is useful to test that a specialized
- /// implementation of `.skip_next(...)` is consistent
- /// with the default implementation.
- pub(crate) struct UnoptimizedDocSet<TDocSet: DocSet>(TDocSet);
-
- impl<TDocSet: DocSet> UnoptimizedDocSet<TDocSet> {
- pub fn wrap(docset: TDocSet) -> UnoptimizedDocSet<TDocSet> {
- UnoptimizedDocSet(docset)
- }
- }
-
- impl<TDocSet: DocSet> DocSet for UnoptimizedDocSet<TDocSet> {
- fn advance(&mut self) -> bool {
- self.0.advance()
- }
-
- fn doc(&self) -> DocId {
- self.0.doc()
- }
-
- fn size_hint(&self) -> u32 {
- self.0.size_hint()
- }
- }
-
- impl<TScorer: Scorer> Scorer for UnoptimizedDocSet<TScorer> {
- fn score(&mut self) -> Score {
- self.0.score()
- }
- }
-
- pub fn test_skip_against_unoptimized<F: Fn() -> Box<DocSet>>(
- postings_factory: F,
- targets: Vec<u32>,
- ) {
- for target in targets {
- let mut postings_opt = postings_factory();
- let mut postings_unopt = UnoptimizedDocSet::wrap(postings_factory());
- let skip_result_opt = postings_opt.skip_next(target);
- let skip_result_unopt = postings_unopt.skip_next(target);
- assert_eq!(
- skip_result_unopt, skip_result_opt,
- "Failed while skipping to {}",
- target
- );
- match skip_result_opt {
- SkipResult::Reached => assert_eq!(postings_opt.doc(), target),
- SkipResult::OverStep => assert!(postings_opt.doc() > target),
- SkipResult::End => {
- return;
- }
- }
- while postings_opt.advance() {
- assert!(postings_unopt.advance());
- assert_eq!(
- postings_opt.doc(),
- postings_unopt.doc(),
- "Failed while skipping to {}",
- target
- );
- }
- assert!(!postings_unopt.advance());
- }
- }
-
-}
+} \ No newline at end of file
diff --git a/src/query/bitset/mod.rs b/src/query/bitset/mod.rs
index 53947f1..27be37a 100644
--- a/src/query/bitset/mod.rs
+++ b/src/query/bitset/mod.rs
@@ -124,9 +124,7 @@ mod tests {
use common::BitSet;
use docset::{DocSet, SkipResult};
use super::BitSetDocSet;
- extern crate test;
- use tests;
-
+
fn create_docbitset(docs: &[DocId], max_doc: DocId) -> BitSetDocSet {
let mut docset = BitSet::with_max_value(max_doc);
for &doc in docs {
@@ -219,6 +217,16 @@ mod tests {
}
}
+}
+
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+ use tests;
+ use test::{self, Bencher};
+ use super::BitSet;
+
#[bench]
fn bench_bitset_1pct_insert(b: &mut test::Bencher) {
use tests;
@@ -254,4 +262,4 @@ mod tests {
while docset.advance() {}
});
}
-}
+} \ No newline at end of file
diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs
index 7e45329..830626a 100644
--- a/src/query/phrase_query/phrase_scorer.rs
+++ b/src/query/phrase_query/phrase_scorer.rs
@@ -245,7 +245,6 @@ impl<TPostings: Postings> Scorer for PhraseScorer<TPostings> {
#[cfg(test)]
mod tests {
- use test::Bencher;
use super::{intersection_count, intersection};
@@ -270,6 +269,14 @@ mod tests {
test_intersection_sym(&[5, 7], &[1, 5, 10, 12], &[5]);
test_intersection_sym(&[1, 5, 6, 9, 10, 12], &[6, 8, 9, 12], &[6, 9, 12]);
}
+}
+
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+ use test::Bencher;
+ use super::{intersection_count, intersection};
#[bench]
fn bench_intersection_short(b: &mut Bencher) {
diff --git a/src/query/union.rs b/src/query/union.rs
index 36756a6..6552d1c 100644
--- a/src/query/union.rs
+++ b/src/query/union.rs
@@ -263,7 +263,6 @@ mod tests {
use super::Union;
use tests;
- use test::Bencher;
use DocId;
use std::collections::BTreeSet;
use super::HORIZON;
@@ -407,6 +406,14 @@ mod tests {
);
}
+
+}
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+ use test::Bencher;
+
+
#[bench]
fn bench_union_3_high(bench: &mut Bencher) {
let union_docset: Vec<Vec<DocId>> = vec![
@@ -442,5 +449,4 @@ mod tests {
while v.advance() {}
});
}
-
}
diff --git a/src/store/mod.rs b/src/store/mod.rs
index 152fe88..b9f7759 100644
--- a/src/store/mod.rs
+++ b/src/store/mod.rs
@@ -42,7 +42,6 @@ pub use self::writer::StoreWriter;
mod tests {
use super::*;
- use test::Bencher;
use std::path::Path;
use schema::{Schema, SchemaBuilder};
use schema::TextOptions;
@@ -105,6 +104,13 @@ mod tests {
}
}
+}
+
+#[cfg(all(test, feature="unstable"))]
+mod bench {
+
+ use test::Bencher;
+
#[bench]
#[cfg(feature="mmap")]
fn bench_store_encode(b: &mut Bencher) {
@@ -127,4 +133,4 @@ mod tests {
store.get(12).unwrap();
});
}
-}
+} \ No newline at end of file
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-01 15:53:08 +0000