diff options
| author | Ellie Huxtable <e@elm.sh> | 2021-04-25 18:21:52 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-04-25 17:21:52 +0000 |
| commit | 156893d774b4da5b541fdbb08428f9ec392949a0 (patch) | |
| tree | 9185d94384aa62eb6eb099ddc4ca9408df6f90d1 | |
| parent | 4210e8de5a29eb389b753adf8df47d2c449a2eeb (diff) | |
Update docs, unify on SQLx, bugfixes (#40)
* Begin moving to sqlx for local too
* Stupid scanners should just have a nice cup of tea
Random internet shit searching for /.env or whatever
* Remove diesel and rusqlite fully
39 files changed, 846 insertions, 611 deletions
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1a8ac289..54bbbb4f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -38,7 +38,7 @@ jobs: override: true - name: Run cargo test - run: cargo test + run: cargo test --workspace clippy: runs-on: ubuntu-latest @@ -92,6 +92,7 @@ dependencies = [ "chrono", "chrono-english", "cli-table", + "crossbeam-channel", "directories", "eyre", "fork", @@ -100,11 +101,11 @@ dependencies = [ "itertools", "log", "pretty_env_logger", - "rusqlite", "serde 1.0.125", "serde_derive", "serde_json", "structopt", + "tabwriter", "termion", "tokio", "tui", @@ -132,13 +133,13 @@ dependencies = [ "rand 0.8.3", "reqwest", "rmp-serde", - "rusqlite", "rust-crypto", "serde 1.0.125", "serde_derive", "serde_json", "shellexpand", "sodiumoxide", + "sqlx", "tokio", "urlencoding", "uuid", @@ -607,18 +608,6 @@ dependencies = [ ] [[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] name = "fern" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1818,21 +1807,6 @@ dependencies = [ ] [[package]] -name = "rusqlite" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc783b7ddae608338003bac1fa00b6786a75a9675fbd8e87243ecfdea3f6ed2" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "memchr", - "smallvec", -] - -[[package]] name = "rust-argon2" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2116,6 +2090,7 @@ dependencies = [ "hmac", "itoa", "libc", + "libsqlite3-sys", "log", "md-5", "memchr", @@ -2235,6 +2210,15 @@ dependencies = [ ] [[package]] +name = "tabwriter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36205cfc997faadcc4b0b87aaef3fbedafe20d38d4959a7ca6ff803564051111" +dependencies = [ + "unicode-width", +] + +[[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -37,7 +37,6 @@ chrono-english = "0.1.4" cli-table = "0.4" base64 = "0.13.0" humantime = "2.1.0" +tabwriter = "1.2.1" +crossbeam-channel = "0.5.1" -[dependencies.rusqlite] -version = "0.25" -features = ["bundled"] @@ -21,7 +21,7 @@ FROM debian:buster-slim as runtime WORKDIR app ENV TZ=Etc/UTC -ENV RUST_LOG=info +ENV RUST_LOG=atuin::api=info ENV ATUIN_CONFIG_DIR=/config COPY --from=builder /app/target/release/atuin /usr/local/bin @@ -1,9 +1,7 @@ <h1 align="center"> - A'Tuin + Atuin </h1> -<blockquote align="center"> - Through the fathomless deeps of space swims the star turtle Great AโTuin, bearing on its back the four giant elephants who carry on their shoulders the mass of the Discworld. - </blockquote> +<em align="center">Magical shell history</em> <p align="center"> <a href="https://github.com/ellie/atuin/actions?query=workflow%3ARust"><img src="https://img.shields.io/github/workflow/status/ellie/atuin/Rust?style=flat-square" /></a> @@ -12,28 +10,42 @@ <a href="https://github.com/ellie/atuin/blob/main/LICENSE"><img src="https://img.shields.io/crates/l/atuin.svg?style=flat-square" /></a> </p> -A'Tuin manages and synchronizes your shell history! Instead of storing -everything in a text file (such as ~/.history), A'Tuin uses a sqlite database. -While being a little more complex, this allows for more functionality. - -As well as the expected command, A'Tuin stores - -- duration -- exit code -- working directory -- hostname -- time -- a unique session ID +- store shell history in a sqlite database +- back up e2e encrypted history to the cloud, and synchronize between machines +- log exit code, cwd, hostname, session, command duration, etc +- smart interactive history search to replace ctrl-r +- calculate statistics such as "most used command" +- old history file is not replaced + +## Documentation + +- [Quickstart](#quickstart) +- [Install](#install) +- [Import](docs/import.md) +- [Configuration](docs/config.md) +- [Searching history](docs/search.md) +- [Cloud history sync](docs/sync.md) +- [History stats](docs/stats.md) ## Supported Shells - zsh +# Quickstart + +``` +curl https://github.com/ellie/atuin/blob/main/install.sh | bash + +atuin register -u <USERNAME> -e <EMAIL> -p <PASSWORD> +atuin import auto +atuin sync +``` + ## Install ### AUR -A'Tuin is available on the [AUR](https://aur.archlinux.org/packages/atuin/) +Atuin is available on the [AUR](https://aur.archlinux.org/packages/atuin/) ``` yay -S atuin # or your AUR helper of choice @@ -41,19 +53,16 @@ yay -S atuin # or your AUR helper of choice ### With cargo -`atuin` needs a nightly version of Rust + Cargo! It's best to use -[rustup](https://rustup.rs/) for getting set up there. +It's best to use [rustup](https://rustup.rs/) to get setup with a Rust +toolchain, then you can run: ``` -rustup default nightly - cargo install atuin ``` ### From source ``` -rustup default nightly git clone https://github.com/ellie/atuin.git cd atuin cargo install --path . @@ -67,107 +76,9 @@ Once the binary is installed, the shell plugin requires installing. Add eval "$(atuin init)" ``` -to your `.zshrc`/`.bashrc`/whatever your shell uses. - -## Usage - -### History search - -By default A'Tuin will rebind ctrl-r and the up arrow to search your history. - -You can prevent this by putting - -``` -export ATUIN_BINDKEYS="false" -``` - -into your shell config. - -### Import history - -``` -atuin import auto # detect shell, then import - -or - -atuin import zsh # specify shell -``` - -### List history - -List all history - -``` -atuin history list -``` - -List history for the current directory - -``` -atuin history list --cwd - -atuin h l -c # alternative, shorter version -``` - -List history for the current session - -``` -atuin history list --session - -atuin h l -s # similarly short -``` - -### Stats - -A'Tuin can calculate statistics for a single day, and accepts "natural language" style date input, as well as absolute dates: - -``` -$ atuin stats day last friday - -+---------------------+------------+ -| Statistic | Value | -+---------------------+------------+ -| Most used command | git status | -+---------------------+------------+ -| Commands ran | 450 | -+---------------------+------------+ -| Unique commands ran | 213 | -+---------------------+------------+ - -$ atuin stats day 01/01/21 # also accepts absolute dates -``` - -It can also calculate statistics for all of known history: - -``` -$ atuin stats all - -+---------------------+-------+ -| Statistic | Value | -+---------------------+-------+ -| Most used command | ls | -+---------------------+-------+ -| Commands ran | 8190 | -+---------------------+-------+ -| Unique commands ran | 2996 | -+---------------------+-------+ -``` - -## Config - -A'Tuin is configurable via TOML. The file lives at ` ~/.config/atuin/config.toml`, -and looks like this: - -``` -[local] -dialect = "uk" # or us. sets the date format used by stats -server_address = "https://atuin.elliehuxtable.com/" # the server to sync with - -[local.db] -path = "~/.local/share/atuin/history.db" # the local database for history -``` +to your `.zshrc` ## ...what's with the name? -A'Tuin is named after "The Great A'Tuin", a giant turtle from Terry Pratchett's +Atuin is named after "The Great A'Tuin", a giant turtle from Terry Pratchett's Discworld series of books. diff --git a/atuin-client/Cargo.toml b/atuin-client/Cargo.toml index 4d3e9130..bd09ca42 100644 --- a/atuin-client/Cargo.toml +++ b/atuin-client/Cargo.toml @@ -37,6 +37,6 @@ tokio = { version = "1", features = ["full"] } async-trait = "0.1.49" urlencoding = "1.1.1" humantime = "2.1.0" -rusqlite= { version = "0.25", features = ["bundled"] } itertools = "0.10.0" shellexpand = "2" +sqlx = { version = "0.5", features = [ "runtime-tokio-rustls", "uuid", "chrono", "sqlite" ] } diff --git a/atuin-client/migrations/20210422143411_create_history.sql b/atuin-client/migrations/20210422143411_create_history.sql new file mode 100644 index 00000000..23c63a4f --- /dev/null +++ b/atuin-client/migrations/20210422143411_create_history.sql @@ -0,0 +1,16 @@ +-- Add migration script here +create table if not exists history ( + id text primary key, + timestamp text not null, + duration integer not null, + exit integer not null, + command text not null, + cwd text not null, + session text not null, + hostname text not null, + + unique(timestamp, cwd, command) +); + +create index if not exists idx_history_timestamp on history(timestamp); +create index if not exists idx_history_command on history(command); diff --git a/atuin-client/src/database.rs b/atuin-client/src/database.rs index 0855359b..754a0ecf 100644 --- a/atuin-client/src/database.rs +++ b/atuin-client/src/database.rs @@ -1,44 +1,48 @@ -use chrono::prelude::*; -use chrono::Utc; use std::path::Path; +use std::str::FromStr; + +use async_trait::async_trait; +use chrono::Utc; -use eyre::{eyre, Result}; +use eyre::Result; -use rusqlite::{params, Connection}; -use rusqlite::{Params, Transaction}; +use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePool, SqlitePoolOptions}; use super::history::History; +#[async_trait] pub trait Database { - fn save(&mut self, h: &History) -> Result<()>; - fn save_bulk(&mut self, h: &[History]) -> Result<()>; + async fn save(&mut self, h: &History) -> Result<()>; + async fn save_bulk(&mut self, h: &[History]) -> Result<()>; - fn load(&self, id: &str) -> Result<History>; - fn list(&self, max: Option<usize>, unique: bool) -> Result<Vec<History>>; - fn range(&self, from: chrono::DateTime<Utc>, to: chrono::DateTime<Utc>) - -> Result<Vec<History>>; + async fn load(&self, id: &str) -> Result<History>; + async fn list(&self, max: Option<usize>, unique: bool) -> Result<Vec<History>>; + async fn range( + &self, + from: chrono::DateTime<Utc>, + to: chrono::DateTime<Utc>, + ) -> Result<Vec<History>>; - fn query(&self, query: &str, params: impl Params) -> Result<Vec<History>>; - fn update(&self, h: &History) -> Result<()>; - fn history_count(&self) -> Result<i64>; + async fn update(&self, h: &History) -> Result<()>; + async fn history_count(&self) -> Result<i64>; - fn first(&self) -> Result<History>; - fn last(&self) -> Result<History>; - fn before(&self, timestamp: chrono::DateTime<Utc>, count: i64) -> Result<Vec<History>>; + async fn first(&self) -> Result<History>; + async fn last(&self) -> Result<History>; + async fn before(&self, timestamp: chrono::DateTime<Utc>, count: i64) -> Result<Vec<History>>; - fn prefix_search(&self, query: &str) -> Result<Vec<History>>; + async fn search(&self, limit: Option<i64>, query: &str) -> Result<Vec<History>>; - fn search(&self, cwd: Option<String>, exit: Option<i64>, query: &str) -> Result<Vec<History>>; + async fn query_history(&self, query: &str) -> Result<Vec<History>>; } // Intended for use on a developer machine and not a sync server. // TODO: implement IntoIterator pub struct Sqlite { - conn: Connection, + pool: SqlitePool, } impl Sqlite { - pub fn new(path: impl AsRef<Path>) -> Result<Self> { + pub async fn new(path: impl AsRef<Path>) -> Result<Self> { let path = path.as_ref(); debug!("opening sqlite database at {:?}", path); @@ -49,137 +53,106 @@ impl Sqlite { } } - let conn = Connection::open(path)?; + let opts = SqliteConnectOptions::from_str(path.as_os_str().to_str().unwrap())? + .journal_mode(SqliteJournalMode::Wal) + .create_if_missing(true); + + let pool = SqlitePoolOptions::new().connect_with(opts).await?; - Self::setup_db(&conn)?; + Self::setup_db(&pool).await?; - Ok(Self { conn }) + Ok(Self { pool }) } - fn setup_db(conn: &Connection) -> Result<()> { + async fn setup_db(pool: &SqlitePool) -> Result<()> { debug!("running sqlite database setup"); - conn.execute( - "create table if not exists history ( - id text primary key, - timestamp integer not null, - duration integer not null, - exit integer not null, - command text not null, - cwd text not null, - session text not null, - hostname text not null, - - unique(timestamp, cwd, command) - )", - [], - )?; - - conn.execute( - "create table if not exists history_encrypted ( - id text primary key, - data blob not null - )", - [], - )?; - - conn.execute( - "create index if not exists idx_history_timestamp on history(timestamp)", - [], - )?; - - conn.execute( - "create index if not exists idx_history_command on history(command)", - [], - )?; + sqlx::migrate!("./migrations").run(pool).await?; Ok(()) } - fn save_raw(tx: &Transaction, h: &History) -> Result<()> { - tx.execute( - "insert or ignore into history ( - id, - timestamp, - duration, - exit, - command, - cwd, - session, - hostname - ) values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - params![ - h.id, - h.timestamp.timestamp_nanos(), - h.duration, - h.exit, - h.command, - h.cwd, - h.session, - h.hostname - ], - )?; + async fn save_raw(tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, h: &History) -> Result<()> { + sqlx::query( + "insert or ignore into history(id, timestamp, duration, exit, command, cwd, session, hostname) + values(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + ) + .bind(h.id.as_str()) + .bind(h.timestamp.to_rfc3339()) + .bind(h.duration) + .bind(h.exit) + .bind(h.command.as_str()) + .bind(h.cwd.as_str()) + .bind(h.session.as_str()) + .bind(h.hostname.as_str()) + .execute(tx) + .await?; Ok(()) } } +#[async_trait] impl Database for Sqlite { - fn save(&mut self, h: &History) -> Result<()> { + async fn save(&mut self, h: &History) -> Result<()> { debug!("saving history to sqlite"); - let tx = self.conn.transaction()?; - Self::save_raw(&tx, h)?; - tx.commit()?; + let mut tx = self.pool.begin().await?; + Self::save_raw(&mut tx, h).await?; + tx.commit().await?; Ok(()) } - fn save_bulk(&mut self, h: &[History]) -> Result<()> { + async fn save_bulk(&mut self, h: &[History]) -> Result<()> { debug!("saving history to sqlite"); - let tx = self.conn.transaction()?; + let mut tx = self.pool.begin().await?; + for i in h { - Self::save_raw(&tx, i)? + Self::save_raw(&mut tx, i).await? } - tx.commit()?; + + tx.commit().await?; Ok(()) } - fn load(&self, id: &str) -> Result<History> { + async fn load(&self, id: &str) -> Result<History> { debug!("loading history item {}", id); - let history = self.query( - "select id, timestamp, duration, exit, command, cwd, session, hostname from history - where id = ?1 limit 1", - &[id], - )?; + let res = sqlx::query_as::<_, History>("select * from history where id = ?1") + .bind(id) + .fetch_one(&self.pool) + .await?; - if history.is_empty() { - return Err(eyre!("could not find history with id {}", id)); - } - - let history = history[0].clone(); - - Ok(history) + Ok(res) } - fn update(&self, h: &History) -> Result<()> { + async fn update(&self, h: &History) -> Result<()> { debug!("updating sqlite history"); - self.conn.execute( + sqlx::query( "update history set timestamp = ?2, duration = ?3, exit = ?4, command = ?5, cwd = ?6, session = ?7, hostname = ?8 where id = ?1", - params![h.id, h.timestamp.timestamp_nanos(), h.duration, h.exit, h.command, h.cwd, h.session, h.hostname], - )?; + ) + .bind(h.id.as_str()) + .bind(h.timestamp.to_rfc3339()) + .bind(h.duration) + .bind(h.exit) + .bind(h.command.as_str()) + .bind(h.cwd.as_str()) + .bind(h.session.as_str()) + .bind(h.hostname.as_str()) + .execute(&self.pool) + .await?; Ok(()) } // make a unique list, that only shows the *newest* version of things - fn list(&self, max: Option<usize>, unique: bool) -> Result<Vec<History>> { + async fn list(&self, max: Option<usize>, unique: bool) -> Result<Vec<History>> { debug!("listing history"); // very likely vulnerable to SQL injection @@ -208,144 +181,96 @@ impl Database for Sqlite { } ); - let history = self.query(query.as_str(), params![])?; + let res = sqlx::query_as::<_, History>(query.as_str()) + .fetch_all(&self.pool) + .await?; - Ok(history) + Ok(res) } - fn range( + async fn range( &self, from: chrono::DateTime<Utc>, to: chrono::DateTime<Utc>, ) -> Result<Vec<History>> { debug!("listing history from {:?} to {:?}", from, to); - let mut stmt = self.conn.prepare( - "SELECT * FROM history where timestamp >= ?1 and timestamp <= ?2 order by timestamp asc", - )?; - - let history_iter = stmt.query_map( - params![from.timestamp_nanos(), to.timestamp_nanos()], - |row| history_from_sqlite_row(None, row), - )?; + let res = sqlx::query_as::<_, History>( + "select * from history where timestamp >= ?1 and timestamp <= ?2 order by timestamp asc", + ) + .bind(from) + .bind(to) + .fetch_all(&self.pool) + .await?; - Ok(history_iter.filter_map(Result::ok).collect()) + Ok(res) } - fn first(&self) -> Result<History> { - let mut stmt = self - .conn - .prepare("SELECT * FROM history order by timestamp asc limit 1")?; - - let history = stmt.query_row(params![], |row| history_from_sqlite_row(None, row))?; + async fn first(&self) -> Result<History> { + let res = sqlx::query_as::<_, History>( + "select * from history where duration >= 0 order by timestamp asc limit 1", + ) + .fetch_one(&self.pool) + .await?; - Ok(history) + Ok(res) } - fn last(&self) -> Result<History> { - let mut stmt = self - .conn - .prepare("SELECT * FROM history where duration >= 0 order by timestamp desc limit 1")?; - - let history = stmt.query_row(params![], |row| history_from_sqlite_row(None, row))?; + async fn last(&self) -> Result<History> { + let res = sqlx::query_as::<_, History>( + "select * from history where duration >= 0 order by timestamp desc limit 1", + ) + .fetch_one(&self.pool) + .await?; - Ok(history) + Ok(res) } - fn before(&self, timestamp: chrono::DateTime<Utc>, count: i64) -> Result<Vec<History>> { - let mut stmt = self - .conn - .prepare("SELECT * FROM history where timestamp < ? order by timestamp desc limit ?")?; - - let history_iter = stmt.query_map(params![timestamp.timestamp_nanos(), count], |row| { - history_from_sqlite_row(None, row) - })?; + async fn before(&self, timestamp: chrono::DateTime<Utc>, count: i64) -> Result<Vec<History>> { + let res = sqlx::query_as::<_, History>( + "select * from history where timestamp < ?1 order by |