diff options
Diffstat (limited to 'src/tokenizer/mod.rs')
| -rw-r--r-- | src/tokenizer/mod.rs | 12 |
1 files changed, 3 insertions, 9 deletions
diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index dd4dba3..fe50e6f 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -4,8 +4,7 @@ //! You must define in your schema which tokenizer should be used for //! each of your fields : //! -//! ``` -//! extern crate tantivy; +//! ```rust //! use tantivy::schema::*; //! //! # fn main() { @@ -65,8 +64,6 @@ //! For instance, the `en_stem` is defined as follows. //! //! ```rust -//! # extern crate tantivy; -//! //! use tantivy::tokenizer::*; //! //! # fn main() { @@ -80,8 +77,7 @@ //! Once your tokenizer is defined, you need to //! register it with a name in your index's [`TokenizerManager`](./struct.TokenizerManager.html). //! -//! ``` -//! # extern crate tantivy; +//! ```rust //! # use tantivy::schema::Schema; //! # use tantivy::tokenizer::*; //! # use tantivy::Index; @@ -101,8 +97,7 @@ //! //! # Example //! -//! ``` -//! extern crate tantivy; +//! ```rust //! use tantivy::schema::{Schema, IndexRecordOption, TextOptions, TextFieldIndexing}; //! use tantivy::tokenizer::*; //! use tantivy::Index; @@ -155,7 +150,6 @@ pub use self::simple_tokenizer::SimpleTokenizer; pub use self::stemmer::{Language, Stemmer}; pub use self::stop_word_filter::StopWordFilter; pub(crate) use self::token_stream_chain::TokenStreamChain; -pub(crate) use self::tokenizer::box_tokenizer; pub use self::tokenizer::BoxedTokenizer; pub use self::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer}; |