refactor server to allow pluggable db and tracing (#1036)

* refactor server to allow pluggable db and tracing

* clean up

* fix descriptions

* remove dependencies

33 files changed, 760 insertions, 888 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 16e67c25..a3ac81b1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -97,6 +97,7 @@ dependencies = [
  "atuin-client",
  "atuin-common",
  "atuin-server",
+ "atuin-server-postgres",
  "base64 0.21.0",
  "bitflags",
  "cassowary",
@@ -104,7 +105,6 @@ dependencies = [
  "clap",
  "clap_complete",
  "colored",
- "crossbeam-channel",
  "crossterm",
  "directories",
  "env_logger",
@@ -160,7 +160,6 @@ dependencies = [
  "serde_regex",
  "sha2",
  "shellexpand",
- "sodiumoxide",
  "sql-builder",
  "sqlx",
  "tokio",
@@ -187,6 +186,7 @@ dependencies = [
  "argon2",
  "async-trait",
  "atuin-common",
+ "atuin-server-database",
  "axum",
  "base64 0.21.0",
  "chrono",
@@ -200,14 +200,39 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "sodiumoxide",
- "sqlx",
  "tokio",
  "tower",
  "tower-http",
  "tracing",
  "uuid",
- "whoami",
+]
+
+[[package]]
+name = "atuin-server-database"
+version = "15.0.0"
+dependencies = [
+ "async-trait",
+ "atuin-common",
+ "chrono",
+ "chronoutil",
+ "eyre",
+ "serde",
+ "tracing",
+ "uuid",
+]
+
+[[package]]
+name = "atuin-server-postgres"
+version = "15.0.0"
+dependencies = [
+ "async-trait",
+ "atuin-common",
+ "atuin-server-database",
+ "chrono",
+ "futures-util",
+ "serde",
+ "sqlx",
+ "tracing",
 ]
 
 [[package]]
@@ -516,16 +541,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d0165d2900ae6778e36e80bbc4da3b5eefccee9ba939761f9c2882a5d9af3ff"
 
 [[package]]
-name = "crossbeam-channel"
-version = "0.5.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
-]
-
-[[package]]
 name = "crossbeam-queue"
 version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -632,15 +647,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "ed25519"
-version = "1.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e9c280362032ea4203659fc489832d0204ef09f247a0506f170dafcac08c369"
-dependencies = [
- "signature",
-]
-
-[[package]]
 name = "either"
 version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1176,18 +1182,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
 
 [[package]]
-name = "libsodium-sys"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b779387cd56adfbc02ea4a668e704f729be8d6a6abd2c27ca5ee537849a92fd"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "walkdir",
-]
-
-[[package]]
 name = "libsqlite3-sys"
 version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1876,15 +1870,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "same-file"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
-dependencies = [
- "winapi-util",
-]
-
-[[package]]
 name = "schannel"
 version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2072,12 +2057,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "signature"
-version = "1.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e90531723b08e4d6d71b791108faf51f03e1b4a7784f96b2b87f852ebc247228"
-
-[[package]]
 name = "slab"
 version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2103,18 +2082,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "sodiumoxide"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e26be3acb6c2d9a7aac28482586a7856436af4cfe7100031d219de2d2ecb0028"
-dependencies = [
- "ed25519",
- "libc",
- "libsodium-sys",
- "serde",
-]
-
-[[package]]
 name = "spin"
 version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2660,17 +2627,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
-name = "walkdir"
-version = "2.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
-dependencies = [
- "same-file",
- "winapi",
- "winapi-util",
-]
-
-[[package]]
 name = "want"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 652efb8a..00b0434e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,12 @@
 [workspace]
-members = ["atuin", "atuin-client", "atuin-server", "atuin-common"]
+members = [
+    "atuin",
+    "atuin-client",
+    "atuin-server",
+    "atuin-server-postgres",
+    "atuin-server-database",
+    "atuin-common",
+]
 
 [workspace.package]
 name = "atuin"
@@ -27,7 +34,6 @@ rand = { version = "0.8.5", features = ["std"] }
 semver = "1.0.14"
 serde = { version = "1.0.145", features = ["derive"] }
 serde_json = "1.0.86"
-sodiumoxide = "0.2.6"
 tokio = { version = "1", features = ["full"] }
 uuid = { version = "1.2", features = ["v4"] }
 whoami = "1.1.2"
diff --git a/atuin-client/Cargo.toml b/atuin-client/Cargo.toml
index fee3eb5f..770d7741 100644
--- a/atuin-client/Cargo.toml
+++ b/atuin-client/Cargo.toml
@@ -53,7 +53,6 @@ memchr = "2.5"
 
 # sync
 urlencoding = { version = "2.1.0", optional = true }
-sodiumoxide = { workspace = true, optional = true }
 reqwest = { workspace = true, optional = true }
 hex = { version = "0.4", optional = true }
 sha2 = { version = "0.10", optional = true }
diff --git a/atuin-server-database/Cargo.toml b/atuin-server-database/Cargo.toml
new file mode 100644
index 00000000..485b3246
--- /dev/null
+++ b/atuin-server-database/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "atuin-server-database"
+edition = "2021"
+description = "server database library for atuin"
+
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+
+[dependencies]
+atuin-common = { path = "../atuin-common", version = "15.0.0" }
+
+tracing = "0.1"
+chrono = { workspace = true }
+eyre = { workspace = true }
+uuid = { workspace = true }
+serde = { workspace = true }
+async-trait = { workspace = true }
+chronoutil = "0.2.3"
diff --git a/atuin-server/src/calendar.rs b/atuin-server-database/src/calendar.rs
index 7c05dce3..7c05dce3 100644
--- a/atuin-server/src/calendar.rs
+++ b/atuin-server-database/src/calendar.rs
diff --git a/atuin-server-database/src/lib.rs b/atuin-server-database/src/lib.rs
new file mode 100644
index 00000000..de33ba44
--- /dev/null
+++ b/atuin-server-database/src/lib.rs
@@ -0,0 +1,220 @@
+#![forbid(unsafe_code)]
+
+pub mod calendar;
+pub mod models;
+
+use std::{
+    collections::HashMap,
+    fmt::{Debug, Display},
+};
+
+use self::{
+    calendar::{TimePeriod, TimePeriodInfo},
+    models::{History, NewHistory, NewSession, NewUser, Session, User},
+};
+use async_trait::async_trait;
+use atuin_common::utils::get_days_from_month;
+use chrono::{Datelike, TimeZone};
+use chronoutil::RelativeDuration;
+use serde::{de::DeserializeOwned, Serialize};
+use tracing::instrument;
+
+#[derive(Debug)]
+pub enum DbError {
+    NotFound,
+    Other(eyre::Report),
+}
+
+impl Display for DbError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{self:?}")
+    }
+}
+
+impl std::error::Error for DbError {}
+
+pub type DbResult<T> = Result<T, DbError>;
+
+#[async_trait]
+pub trait Database: Sized + Clone + Send + Sync + 'static {
+    type Settings: Debug + Clone + DeserializeOwned + Serialize + Send + Sync + 'static;
+    async fn new(settings: &Self::Settings) -> DbResult<Self>;
+
+    async fn get_session(&self, token: &str) -> DbResult<Session>;
+    async fn get_session_user(&self, token: &str) -> DbResult<User>;
+    async fn add_session(&self, session: &NewSession) -> DbResult<()>;
+
+    async fn get_user(&self, username: &str) -> DbResult<User>;
+    async fn get_user_session(&self, u: &User) -> DbResult<Session>;
+    async fn add_user(&self, user: &NewUser) -> DbResult<i64>;
+    async fn delete_user(&self, u: &User) -> DbResult<()>;
+
+    async fn count_history(&self, user: &User) -> DbResult<i64>;
+    async fn count_history_cached(&self, user: &User) -> DbResult<i64>;
+
+    async fn delete_history(&self, user: &User, id: String) -> DbResult<()>;
+    async fn deleted_history(&self, user: &User) -> DbResult<Vec<String>>;
+
+    async fn count_history_range(
+        &self,
+        user: &User,
+        start: chrono::NaiveDateTime,
+        end: chrono::NaiveDateTime,
+    ) -> DbResult<i64>;
+
+    async fn list_history(
+        &self,
+        user: &User,
+        created_after: chrono::NaiveDateTime,
+        since: chrono::NaiveDateTime,
+        host: &str,
+        page_size: i64,
+    ) -> DbResult<Vec<History>>;
+
+    async fn add_history(&self, history: &[NewHistory]) -> DbResult<()>;
+
+    async fn oldest_history(&self, user: &User) -> DbResult<History>;
+
+    /// Count the history for a given year
+    #[instrument(skip_all)]
+    async fn count_history_year(&self, user: &User, year: i32) -> DbResult<i64> {
+        let start = chrono::Utc.ymd(year, 1, 1).and_hms_nano(0, 0, 0, 0);
+        let end = start + RelativeDuration::years(1);
+
+        let res = self
+            .count_history_range(user, start.naive_utc(), end.naive_utc())
+            .await?;
+        Ok(res)
+    }
+
+    /// Count the history for a given month
+    #[instrument(skip_all)]
+    async fn count_history_month(&self, user: &User, month: chrono::NaiveDate) -> DbResult<i64> {
+        let start = chrono::Utc
+            .ymd(month.year(), month.month(), 1)
+            .and_hms_nano(0, 0, 0, 0);
+
+        // ofc...
+        let end = if month.month() < 12 {
+            chrono::Utc
+                .ymd(month.year(), month.month() + 1, 1)
+                .and_hms_nano(0, 0, 0, 0)
+        } else {
+            chrono::Utc
+                .ymd(month.year() + 1, 1, 1)
+                .and_hms_nano(0, 0, 0, 0)
+        };
+
+        tracing::debug!("start: {}, end: {}", start, end);
+
+        let res = self
+            .count_history_range(user, start.naive_utc(), end.naive_utc())
+            .await?;
+        Ok(res)
+    }
+
+    /// Count the history for a given day
+    #[instrument(skip_all)]
+    async fn count_history_day(&self, user: &User, day: chrono::NaiveDate) -> DbResult<i64> {
+        let start = chrono::Utc
+            .ymd(day.year(), day.month(), day.day())
+            .and_hms_nano(0, 0, 0, 0);
+        let end = chrono::Utc
+            .ymd(day.year(), day.month(), day.day() + 1)
+            .and_hms_nano(0, 0, 0, 0);
+
+        let res = self
+            .count_history_range(user, start.naive_utc(), end.naive_utc())
+            .await?;
+        Ok(res)
+    }
+
+    #[instrument(skip_all)]
+    async fn calendar(
+        &self,
+        user: &User,
+        period: TimePeriod,
+        year: u64,
+        month: u64,
+    ) -> DbResult<HashMap<u64, TimePeriodInfo>> {
+        // TODO: Support different timezones. Right now we assume UTC and
+        // everything is stored as such. But it _should_ be possible to
+        // interpret the stored date with a different TZ
+
+        match period {
+            TimePeriod::YEAR => {
+                let mut ret = HashMap::new();
+                // First we need to work out how far back to calculate. Get the
+                // oldest history item
+                let oldest = self.oldest_history(user).await?.timestamp.year();
+                let current_year = chrono::Utc::now().year();
+
+                // All the years we need to get data for
+                // The upper bound is exclusive, so include current +1
+                let years = oldest..current_year + 1;
+
+                for year in years {
+                    let count = self.count_history_year(user, year).await?;
+
+                    ret.insert(
+                        year as u64,
+                        TimePeriodInfo {
+                            count: count as u64,
+                            hash: "".to_string(),
+                        },
+                    );
+                }
+
+                Ok(ret)
+            }
+
+            TimePeriod::MONTH => {
+                let mut ret = HashMap::new();
+
+                for month in 1..13 {
+                    let count = self
+                        .count_history_month(
+                            user,
+                            chrono::Utc.ymd(year as i32, month, 1).naive_utc(),
+                        )
+                        .await?;
+
+                    ret.insert(
+                        month as u64,
+                        TimePeriodInfo {
+                            count: count as u64,
+                            hash: "".to_string(),
+                        },
+                    );
+                }
+
+                Ok(ret)
+            }
+
+            TimePeriod::DAY => {
+                let mut ret = HashMap::new();
+
+                for day in 1..get_days_from_month(year as i32, month as u32) {
+                    let count = self
+                        .count_history_day(
+                            user,
+                            chrono::Utc
+                                .ymd(year as i32, month as u32, day as u32)
+                                .naive_utc(),
+                        )
+                        .await?;
+
+                    ret.insert(
+                        day as u64,
+                        TimePeriodInfo {
+                            count: count as u64,
+                            hash: "".to_string(),
+                        },
+                    );
+                }
+
+                Ok(ret)
+            }
+        }
+    }
+}
diff --git a/atuin-server/src/models.rs b/atuin-server-database/src/models.rs
index ee84f58a..a95ceba2 100644
--- a/atuin-server/src/models.rs
+++ b/atuin-server-database/src/models.rs
@@ -1,6 +1,5 @@
 use chrono::prelude::*;
 
-#[derive(sqlx::FromRow)]
 pub struct History {
     pub id: i64,
     pub client_id: String, // a client generated ID
@@ -22,7 +21,6 @@ pub struct NewHistory {
     pub data: String,
 }
 
-#[derive(sqlx::FromRow)]
 pub struct User {
     pub id: i64,
     pub username: String,
@@ -30,7 +28,6 @@ pub struct User {
     pub password: String,
 }
 
-#[derive(sqlx::FromRow)]
 pub struct Session {
     pub id: i64,
     pub user_id: i64,
diff --git a/atuin-server-postgres/Cargo.toml b/atuin-server-postgres/Cargo.toml
new file mode 100644
index 00000000..18864f6c
--- /dev/null
+++ b/atuin-server-postgres/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "atuin-server-postgres"
+edition = "2018"
+description = "server postgres database library for atuin"
+
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+
+[dependencies]
+atuin-common = { path = "../atuin-common", version = "15.0.0" }
+atuin-server-database = { path = "../atuin-server-database", version = "15.0.0" }
+
+tracing = "0.1"
+chrono = { workspace = true }
+serde = { workspace = true }
+sqlx = { workspace = true }
+async-trait = { workspace = true }
+futures-util = "0.3"
diff --git a/atuin-server/migrations/20210425153745_create_history.sql b/atuin-server-postgres/migrations/20210425153745_create_history.sql
index 2c2d17b0..2c2d17b0 100644
--- a/atuin-server/migrations/20210425153745_create_history.sql
+++ b/atuin-server-postgres/migrations/20210425153745_create_history.sql
diff --git a/atuin-server/migrations/20210425153757_create_users.sql b/atuin-server-postgres/migrations/20210425153757_create_users.sql
index a25dcced..a25dcced 100644
--- a/atuin-server/migrations/20210425153757_create_users.sql
+++ b/atuin-server-postgres/migrations/20210425153757_create_users.sql
diff --git a/atuin-server/migrations/20210425153800_create_sessions.sql b/atuin-server-postgres/migrations/20210425153800_create_sessions.sql
index c2fb6559..c2fb6559 100644
--- a/atuin-server/migrations/20210425153800_create_sessions.sql
+++ b/atuin-server-postgres/migrations/20210425153800_create_sessions.sql
diff --git a/atuin-server/migrations/20220419082412_add_count_trigger.sql b/atuin-server-postgres/migrations/20220419082412_add_count_trigger.sql
index dd1afa88..dd1afa88 100644
--- a/atuin-server/migrations/20220419082412_add_count_trigger.sql
+++ b/atuin-server-postgres/migrations/20220419082412_add_count_trigger.sql
diff --git a/atuin-server/migrations/20220421073605_fix_count_trigger_delete.sql b/atuin-server-postgres/migrations/20220421073605_fix_count_trigger_delete.sql
index 6198f300..6198f300 100644
--- a/atuin-server/migrations/20220421073605_fix_count_trigger_delete.sql
+++ b/atuin-server-postgres/migrations/20220421073605_fix_count_trigger_delete.sql
diff --git a/atuin-server/migrations/20220421174016_larger-commands.sql b/atuin-server-postgres/migrations/20220421174016_larger-commands.sql
index 0ac43433..0ac43433 100644
--- a/atuin-server/migrations/20220421174016_larger-commands.sql
+++ b/atuin-server-postgres/migrations/20220421174016_larger-commands.sql
diff --git a/atuin-server/migrations/20220426172813_user-created-at.sql b/atuin-server-postgres/migrations/20220426172813_user-created-at.sql
index a9138194..a9138194 100644
--- a/atuin-server/migrations/20220426172813_user-created-at.sql
+++ b/atuin-server-postgres/migrations/20220426172813_user-created-at.sql
diff --git a/atuin-server/migrations/20220505082442_create-events.sql b/atuin-server-postgres/migrations/20220505082442_create-events.sql
index 57e16ec7..57e16ec7 100644
--- a/atuin-server/migrations/20220505082442_create-events.sql
+++ b/atuin-server-postgres/migrations/20220505082442_create-events.sql
diff --git a/atuin-server/migrations/20220610074049_history-length.sql b/atuin-server-postgres/migrations/20220610074049_history-length.sql
index b1c23016..b1c23016 100644
--- a/atuin-server/migrations/20220610074049_history-length.sql
+++ b/atuin-server-postgres/migrations/20220610074049_history-length.sql
diff --git a/atuin-server/migrations/20230315220537_drop-events.sql b/atuin-server-postgres/migrations/20230315220537_drop-events.sql
index fe3cae17..fe3cae17 100644
--- a/atuin-server/migrations/20230315220537_drop-events.sql
+++ b/atuin-server-postgres/migrations/20230315220537_drop-events.sql
diff --git a/atuin-server/migrations/20230315224203_create-deleted.sql b/atuin-server-postgres/migrations/20230315224203_create-deleted.sql
index 9a9e6263..9a9e6263 100644
--- a/atuin-server/migrations/20230315224203_create-deleted.sql
+++ b/atuin-server-postgres/migrations/20230315224203_create-deleted.sql
diff --git a/atuin-server/migrations/20230515221038_trigger-delete-only.sql b/atuin-server-postgres/migrations/20230515221038_trigger-delete-only.sql
index 3d0bba52..3d0bba52 100644
--- a/atuin-server/migrations/20230515221038_trigger-delete-only.sql
+++ b/atuin-server-postgres/migrations/20230515221038_trigger-delete-only.sql
diff --git a/atuin-server-postgres/src/lib.rs b/atuin-server-postgres/src/lib.rs
new file mode 100644
index 00000000..0dc51daf
--- /dev/null
+++ b/atuin-server-postgres/src/lib.rs
@@ -0,0 +1,332 @@
+use async_trait::async_trait;
+use atuin_server_database::models::{History, NewHistory, NewSession, NewUser, Session, User};
+use atuin_server_database::{Database, DbError, DbResult};
+use futures_util::TryStreamExt;
+use serde::{Deserialize, Serialize};
+use sqlx::postgres::PgPoolOptions;
+
+use sqlx::Row;
+
+use tracing::instrument;
+use wrappers::{DbHistory, DbSession, DbUser};
+
+mod wrappers;
+
+#[derive(Clone)]
+pub struct Postgres {
+    pool: sqlx::Pool<sqlx::postgres::Postgres>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct PostgresSettings {
+    pub db_uri: String,
+}
+
+fn fix_error(error: sqlx::Error) -> DbError {
+    match error {
+        sqlx::Error::RowNotFound => DbError::NotFound,
+        error => DbError::Other(error.into()),
+    }
+}
+
+#[async_trait]
+impl Database for Postgres {
+    type Settings = PostgresSettings;
+    async fn new(settings: &PostgresSettings) -> DbResult<Self> {
+        let pool = PgPoolOptions::new()
+            .max_connections(100)
+            .connect(settings.db_uri.as_str())
+            .await
+            .map_err(fix_error)?;
+
+        sqlx::migrate!("./migrations")
+            .run(&pool)
+            .await
+            .map_err(|error| DbError::Other(error.into()))?;