Add new sync (#1093)

* Add record migration

* Add database functions for inserting history

No real tests yet :( I would like to avoid running postgres lol

* Add index handler, use UUIDs not strings

* Fix a bunch of tests, remove Option<Uuid>

* Add tests, all passing

* Working upload sync

* Record downloading works

* Sync download works

* Don't waste requests

* Use a page size for uploads, make it variable later

* Aaaaaand they're encrypted now too

* Add cek

* Allow reading tail across hosts

* Revert "Allow reading tail across hosts"

Not like that

This reverts commit 7b0c72e7e050c358172f9b53cbd21b9e44cf4931.

* Handle multiple shards properly

* format

* Format and make clippy happy

* use some fancy types (#1098)

* use some fancy types

* fmt

* Goodbye horrible tuple

* Update atuin-server-postgres/migrations/20230623070418_records.sql

Co-authored-by: Conrad Ludgate <conradludgate@gmail.com>

* fmt

* Sort tests too because time sucks

* fix features

---------

Co-authored-by: Conrad Ludgate <conradludgate@gmail.com>

4 files changed, 110 insertions, 0 deletions

diff --git a/atuin-server/src/handlers/mod.rs b/atuin-server/src/handlers/mod.rs
index 35d32f6f6..2bd782dba 100644
--- a/atuin-server/src/handlers/mod.rs
+++ b/atuin-server/src/handlers/mod.rs
@@ -2,6 +2,7 @@ use atuin_common::api::{ErrorResponse, IndexResponse};
 use axum::{response::IntoResponse, Json};
 
 pub mod history;
+pub mod record;
 pub mod status;
 pub mod user;
 
diff --git a/atuin-server/src/handlers/record.rs b/atuin-server/src/handlers/record.rs
new file mode 100644
index 000000000..0100c6936
--- /dev/null
+++ b/atuin-server/src/handlers/record.rs
@@ -0,0 +1,104 @@
+use axum::{extract::Query, extract::State, Json};
+use http::StatusCode;
+use serde::Deserialize;
+use tracing::{error, instrument};
+
+use super::{ErrorResponse, ErrorResponseStatus, RespExt};
+use crate::router::{AppState, UserAuth};
+use atuin_server_database::Database;
+
+use atuin_common::record::{EncryptedData, HostId, Record, RecordId, RecordIndex};
+
+#[instrument(skip_all, fields(user.id = user.id))]
+pub async fn post<DB: Database>(
+    UserAuth(user): UserAuth,
+    state: State<AppState<DB>>,
+    Json(records): Json<Vec<Record<EncryptedData>>>,
+) -> Result<(), ErrorResponseStatus<'static>> {
+    let State(AppState { database, settings }) = state;
+
+    tracing::debug!(
+        count = records.len(),
+        user = user.username,
+        "request to add records"
+    );
+
+    let too_big = records
+        .iter()
+        .any(|r| r.data.data.len() >= settings.max_record_size || settings.max_record_size == 0);
+
+    if too_big {
+        return Err(
+            ErrorResponse::reply("could not add records; record too large")
+                .with_status(StatusCode::BAD_REQUEST),
+        );
+    }
+
+    if let Err(e) = database.add_records(&user, &records).await {
+        error!("failed to add record: {}", e);
+
+        return Err(ErrorResponse::reply("failed to add record")
+            .with_status(StatusCode::INTERNAL_SERVER_ERROR));
+    };
+
+    Ok(())
+}
+
+#[instrument(skip_all, fields(user.id = user.id))]
+pub async fn index<DB: Database>(
+    UserAuth(user): UserAuth,
+    state: State<AppState<DB>>,
+) -> Result<Json<RecordIndex>, ErrorResponseStatus<'static>> {
+    let State(AppState {
+        database,
+        settings: _,
+    }) = state;
+
+    let record_index = match database.tail_records(&user).await {
+        Ok(index) => index,
+        Err(e) => {
+            error!("failed to get record index: {}", e);
+
+            return Err(ErrorResponse::reply("failed to calculate record index")
+                .with_status(StatusCode::INTERNAL_SERVER_ERROR));
+        }
+    };
+
+    Ok(Json(record_index))
+}
+
+#[derive(Deserialize)]
+pub struct NextParams {
+    host: HostId,
+    tag: String,
+    start: Option<RecordId>,
+    count: u64,
+}
+
+#[instrument(skip_all, fields(user.id = user.id))]
+pub async fn next<DB: Database>(
+    params: Query<NextParams>,
+    UserAuth(user): UserAuth,
+    state: State<AppState<DB>>,
+) -> Result<Json<Vec<Record<EncryptedData>>>, ErrorResponseStatus<'static>> {
+    let State(AppState {
+        database,
+        settings: _,
+    }) = state;
+    let params = params.0;
+
+    let records = match database
+        .next_records(&user, params.host, params.tag, params.start, params.count)
+        .await
+    {
+        Ok(records) => records,
+        Err(e) => {
+            error!("failed to get record index: {}", e);
+
+            return Err(ErrorResponse::reply("failed to calculate record index")
+                .with_status(StatusCode::INTERNAL_SERVER_ERROR));
+        }
+    };
+
+    Ok(Json(records))
+}
diff --git a/atuin-server/src/router.rs b/atuin-server/src/router.rs
index ec558e78c..7dc8a246b 100644
--- a/atuin-server/src/router.rs
+++ b/atuin-server/src/router.rs
@@ -71,6 +71,9 @@ pub fn router<DB: Database>(database: DB, settings: Settings<DB::Settings>) -> R
         .route("/sync/status", get(handlers::status::status))
         .route("/history", post(handlers::history::add))
         .route("/history", delete(handlers::history::delete))
+        .route("/record", post(handlers::record::post))
+        .route("/record", get(handlers::record::index))
+        .route("/record/next", get(handlers::record::next))
         .route("/user/:username", get(handlers::user::get))
         .route("/account", delete(handlers::user::delete))
         .route("/register", post(handlers::user::register))
diff --git a/atuin-server/src/settings.rs b/atuin-server/src/settings.rs
index fb5325d4b..7e447e9ef 100644
--- a/atuin-server/src/settings.rs
+++ b/atuin-server/src/settings.rs
@@ -12,6 +12,7 @@ pub struct Settings<DbSettings> {
     pub path: String,
     pub open_registration: bool,
     pub max_history_length: usize,
+    pub max_record_size: usize,
     pub page_size: i64,
     pub register_webhook_url: Option<String>,
     pub register_webhook_username: String,
@@ -39,6 +40,7 @@ impl<DbSettings: DeserializeOwned> Settings<DbSettings> {
             .set_default("port", 8888)?
             .set_default("open_registration", false)?
             .set_default("max_history_length", 8192)?
+            .set_default("max_record_size", 1024 * 1024 * 1024)? // pretty chonky
             .set_default("path", "")?
             .set_default("register_webhook_username", "")?
             .set_default("page_size", 1100)?