path: root/src/core/segment_reader.rs

use common::CompositeFile;
use common::HasLen;
use core::InvertedIndexReader;
use core::Segment;
use core::SegmentComponent;
use core::SegmentId;
use core::SegmentMeta;
use error::TantivyError;
use fastfield::DeleteBitSet;
use fastfield::FacetReader;
use fastfield::FastFieldReader;
use fastfield::{self, FastFieldNotAvailableError};
use fastfield::{BytesFastFieldReader, FastValue, MultiValueIntFastFieldReader};
use fieldnorm::FieldNormReader;
use schema::Cardinality;
use schema::Document;
use schema::Field;
use schema::FieldType;
use schema::Schema;
use std::collections::HashMap;
use std::fmt;
use std::sync::Arc;
use std::sync::RwLock;
use store::StoreReader;
use termdict::TermDictionary;
use DocId;
use Result;

/// Entry point to access all of the datastructures of the `Segment`
///
/// - term dictionary
/// - postings
/// - store
/// - fast field readers
/// - field norm reader
///
/// The segment reader has a very low memory footprint,
/// as close to all of the memory data is mmapped.
///
///
/// TODO fix not decoding docfreq
#[derive(Clone)]
pub struct SegmentReader {
    inv_idx_reader_cache: Arc<RwLock<HashMap<Field, Arc<InvertedIndexReader>>>>,

    segment_id: SegmentId,
    segment_meta: SegmentMeta,

    termdict_composite: CompositeFile,
    postings_composite: CompositeFile,
    positions_composite: CompositeFile,
    positions_idx_composite: CompositeFile,
    fast_fields_composite: CompositeFile,
    fieldnorms_composite: CompositeFile,

    store_reader: StoreReader,
    delete_bitset_opt: Option<DeleteBitSet>,
    schema: Schema,
}

impl SegmentReader {
    /// Returns the highest document id ever attributed in
    /// this segment + 1.
    /// Today, `tantivy` does not handle deletes, so it happens
    /// to also be the number of documents in the index.
    pub fn max_doc(&self) -> DocId {
        self.segment_meta.max_doc()
    }

    /// Returns the number of documents.
    /// Deleted documents are not counted.
    ///
    /// Today, `tantivy` does not handle deletes so max doc and
    /// num_docs are the same.
    pub fn num_docs(&self) -> DocId {
        self.segment_meta.num_docs()
    }

    /// Returns the schema of the index this segment belongs to.
    pub fn schema(&self) -> &Schema {
        &self.schema
    }

    /// Return the number of documents that have been
    /// deleted in the segment.
    pub fn num_deleted_docs(&self) -> DocId {
        self.delete_bitset()
            .map(|delete_set| delete_set.len() as DocId)
            .unwrap_or(0u32)
    }

    /// Returns true iff some of the documents of the segment have been deleted.
    pub fn has_deletes(&self) -> bool {
        self.delete_bitset().is_some()
    }

    /// Accessor to a segment's fast field reader given a field.
    ///
    /// Returns the u64 fast value reader if the field
    /// is a u64 field indexed as "fast".
    ///
    /// Return a FastFieldNotAvailableError if the field is not
    /// declared as a fast field in the schema.
    ///
    /// # Panics
    /// May panic if the index is corrupted.
    pub fn fast_field_reader<Item: FastValue>(
        &self,
        field: Field,
    ) -> fastfield::Result<FastFieldReader<Item>> {
        let field_entry = self.schema.get_field_entry(field);
        if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::SingleValue)
        {
            self.fast_fields_composite
                .open_read(field)
                .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
                .map(FastFieldReader::open)
        } else {
            Err(FastFieldNotAvailableError::new(field_entry))
        }
    }

    pub(crate) fn fast_field_reader_with_idx<Item: FastValue>(
        &self,
        field: Field,
        idx: usize,
    ) -> fastfield::Result<FastFieldReader<Item>> {
        if let Some(ff_source) = self.fast_fields_composite.open_read_with_idx(field, idx) {
            Ok(FastFieldReader::open(ff_source))
        } else {
            let field_entry = self.schema.get_field_entry(field);
            Err(FastFieldNotAvailableError::new(field_entry))
        }
    }

    /// Accessor to the `MultiValueIntFastFieldReader` associated to a given `Field`.
    /// May panick if the field is not a multivalued fastfield of the type `Item`.
    pub fn multi_fast_field_reader<Item: FastValue>(
        &self,
        field: Field,
    ) -> fastfield::Result<MultiValueIntFastFieldReader<Item>> {
        let field_entry = self.schema.get_field_entry(field);
        if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::MultiValues)
        {
            let idx_reader = self.fast_field_reader_with_idx(field, 0)?;
            let vals_reader = self.fast_field_reader_with_idx(field, 1)?;
            Ok(MultiValueIntFastFieldReader::open(idx_reader, vals_reader))
        } else {
            Err(FastFieldNotAvailableError::new(field_entry))
        }
    }

    /// Accessor to the `BytesFastFieldReader` associated to a given `Field`.
    pub fn bytes_fast_field_reader(&self, field: Field) -> fastfield::Result<BytesFastFieldReader> {
        let field_entry = self.schema.get_field_entry(field);
        match field_entry.field_type() {
            &FieldType::Bytes => {}
            _ => return Err(FastFieldNotAvailableError::new(field_entry)),
        }
        let idx_reader = self.fast_fields_composite
            .open_read_with_idx(field, 0)
            .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
            .map(FastFieldReader::open)?;
        let values = self.fast_fields_composite
            .open_read_with_idx(field, 1)
            .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))?;
        Ok(BytesFastFieldReader::open(idx_reader, values))
    }

    /// Accessor to the `FacetReader` associated to a given `Field`.
    pub fn facet_reader(&self, field: Field) -> Result<FacetReader> {
        let field_entry = self.schema.get_field_entry(field);
        if field_entry.field_type() != &FieldType::HierarchicalFacet {
            return Err(TantivyError::InvalidArgument(format!(
                "The field {:?} is not a \
                 hierarchical facet.",
                field_entry
            )).into());
        }
        let term_ords_reader = self.multi_fast_field_reader(field)?;
        let termdict_source = self.termdict_composite.open_read(field).ok_or_else(|| {
            TantivyError::InvalidArgument(format!(
                "The field \"{}\" is a hierarchical \
                 but this segment does not seem to have the field term \
                 dictionary.",
                field_entry.name()
            ))
        })?;
        let termdict = TermDictionary::from_source(termdict_source);
        let facet_reader = FacetReader::new(term_ords_reader, termdict);
        Ok(facet_reader)
    }

    /// Accessor to the segment's `Field norms`'s reader.
    ///
    /// Field norms are the length (in tokens) of the fields.
    /// It is used in the computation of the [TfIdf]
    /// (https://fulmicoton.gitbooks.io/tantivy-doc/content/tfidf.html).
    ///
    /// They are simply stored as a fast field, serialized in
    /// the `.fieldnorm` file of the segment.
    pub fn get_fieldnorms_reader(&self, field: Field) -> FieldNormReader {
        if let Some(fieldnorm_source) = self.fieldnorms_composite.open_read(field) {
            FieldNormReader::open(fieldnorm_source)
        } else {
            let field_name = self.schema.get_field_name(field);
            let err_msg = format!(
                "Field norm not found for field {:?}. Was it market as indexed during indexing.",
                field_name
            );
            panic!(err_msg);
        }
    }

    /// Accessor to the segment's `StoreReader`.
    pub fn get_store_reader(&self) -> &StoreReader {
        &self.store_reader
    }

    /// Open a new segment for reading.
    pub fn open(segment: &Segment) -> Result<SegmentReader> {
        let termdict_source = segment.open_read(SegmentComponent::TERMS)?;
        let termdict_composite = CompositeFile::open(&termdict_source)?;

        let store_source = segment.open_read(SegmentComponent::STORE)?;
        let store_reader = StoreReader::from_source(store_source);

        let postings_source = segment.open_read(SegmentComponent::POSTINGS)?;
        let postings_composite = CompositeFile::open(&postings_source)?;

        let positions_composite = {
            if let Ok(source) = segment.open_read(SegmentComponent::POSITIONS) {
                CompositeFile::open(&source)?
            } else {
                CompositeFile::empty()
            }
        };

        let positions_idx_composite = {
            if let