use categories;
use rusqlite::types::ToSql;
use rusqlite::NO_PARAMS;
use std::fmt::Write;
use rusqlite;
#[macro_use]
extern crate failure;
#[macro_use]
extern crate lazy_static;
use chrono::prelude::*;
use failure::ResultExt;
use rich_crate::Include;
use rich_crate::Origin;
use rich_crate::Repo;
use rich_crate::RichCrate;
use rich_crate::RichCrateVersion;
use render_readme::Renderer;
use rusqlite::*;
use std::cell::RefCell;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use parking_lot::Mutex;
use thread_local::ThreadLocal;
type FResult<T> = std::result::Result<T, failure::Error>;
mod schema;
mod stopwords;
use crate::stopwords::{COND_STOPWORDS, STOPWORDS};
pub struct CrateDb {
url: String,
conn: ThreadLocal<std::result::Result<RefCell<Connection>, rusqlite::Error>>,
exclusive_conn: Mutex<Option<Connection>>,
tag_synonyms: HashMap<Box<str>, (Box<str>, u8)>,
}
impl CrateDb {
/// Path to sqlite db file to create/update
pub fn new(path: impl AsRef<Path>) -> FResult<Self> {
let path = path.as_ref();
let tag_synonyms = fs::read_to_string(path.with_file_name("tag-synonyms.csv"))?;
let tag_synonyms = tag_synonyms.lines()
.filter(|l| !l.starts_with('#'))
.map(|l| {
let mut cols = l.splitn(3, ',');
let score: u8 = cols.next().unwrap().parse().unwrap();
let find = cols.next().unwrap();
let replace = cols.next().unwrap();
(find.into(), (replace.into(), score))
})
.collect();
Ok(Self {
tag_synonyms,
url: format!("file:{}?cache=shared", path.display()),
conn: ThreadLocal::new(),
exclusive_conn: Mutex::new(None),
})
}
#[inline]
fn with_connection<F, T>(&self, cb: F) -> FResult<T> where F: FnOnce(&mut Connection) -> FResult<T> {
let conn = self.conn.get_or(|| Box::new(self.connect().map(RefCell::new)));
match conn {
Ok(conn) => cb(&mut *conn.borrow_mut()),
Err(err) => bail!("{}", err),
}
}
#[inline]
fn with_tx<F, T>(&self, cb: F) -> FResult<T> where F: FnOnce(&Connection) -> FResult<T> {
let mut conn = self.exclusive_conn.lock();
let conn = conn.get_or_insert_with(|| self.connect().unwrap());
let tx = conn.transaction()?;
let res = cb(&tx)?;
tx.commit()?;
Ok(res)
}
fn connect(&self) -> std::result::Result<Connection, rusqlite::Error> {
let db = Self::db(&self.url)?;
db.execute_batch("
PRAGMA cache_size = 500000;
PRAGMA threads = 4;
PRAGMA synchronous = 0;
PRAGMA journal_mode = TRUNCATE;")?;
Ok(db)
}
pub fn latest_crate_update_timestamp(&self) -> FResult<Option<u32>> {
self.with_connection(|conn| {
let nope: [u8; 0] = [];
Ok(none_rows(conn.query_row("SELECT max(created) FROM crate_versions", nope.iter(), |row| row.get(0)))?)
})
}
/// Add data of the latest version of a crate to the index
/// Score is a ranking of a crate (0 = bad, 1 = great)
pub fn index_latest(&self, c: &RichCrateVersion, deps_stats: &[(&str, f32)], score: f64, (is_build, is_dev): (bool, bool)) -> FResult<()> {
let origin = c.origin().to_str();
let mut insert_keyword = KeywordInsert::new()?;
for (i, k) in