use core::Segment;
use core::SegmentReader;
use core::SerializableSegment;
use docset::DocSet;
use error::Result;
use fastfield::DeleteBitSet;
use fastfield::FastFieldReader;
use fastfield::FastFieldSerializer;
use fieldnorm::FieldNormReader;
use fieldnorm::FieldNormsSerializer;
use fieldnorm::FieldNormsWriter;
use indexer::SegmentSerializer;
use itertools::Itertools;
use postings::InvertedIndexSerializer;
use postings::Postings;
use schema::{Field, Schema};
use store::StoreWriter;
use termdict::TermMerger;
use DocId;
use schema::FieldType;
use termdict::TermOrdinal;
use schema::Cardinality;
use std::collections::HashMap;
use fastfield::MultiValueIntFastFieldReader;
use std::cmp;
fn compute_total_num_tokens(readers: &[SegmentReader], field: Field) -> u64 {
let mut total_tokens = 0u64;
let mut count: [usize; 256] = [0; 256];
for reader in readers {
if reader.has_deletes() {
// if there are deletes, then we use an approximation
// using the fieldnorm
let fieldnorms_reader = reader.get_fieldnorms_reader(field);
for doc in 0..reader.max_doc() {
if !reader.is_deleted(doc)