feat: Add xonsh history import (#1678)

* add importers for xonsh JSON files and SQLite db

* rustfmt xonsh importers

* remove env-dependent tests from xonsh importers

* pass xonsh_data_dir into path resolver instead of looking up in env

* review: run format

* review: fix clippy errors

---------

Co-authored-by: Ellie Huxtable <ellie@elliehuxtable.com>

3 files changed, 462 insertions, 0 deletions

diff --git a/atuin-client/src/import/mod.rs b/atuin-client/src/import/mod.rs
index 0c15c9dd2..c9d8c7987 100644
--- a/atuin-client/src/import/mod.rs
+++ b/atuin-client/src/import/mod.rs
@@ -13,6 +13,8 @@ pub mod fish;
 pub mod nu;
 pub mod nu_histdb;
 pub mod resh;
+pub mod xonsh;
+pub mod xonsh_sqlite;
 pub mod zsh;
 pub mod zsh_histdb;
 
diff --git a/atuin-client/src/import/xonsh.rs b/atuin-client/src/import/xonsh.rs
new file mode 100644
index 000000000..2269212f3
--- /dev/null
+++ b/atuin-client/src/import/xonsh.rs
@@ -0,0 +1,238 @@
+use std::env;
+use std::fs::{self, File};
+use std::path::{Path, PathBuf};
+
+use async_trait::async_trait;
+use directories::BaseDirs;
+use eyre::{eyre, Result};
+use serde::Deserialize;
+use time::OffsetDateTime;
+use uuid::timestamp::{context::NoContext, Timestamp};
+use uuid::Uuid;
+
+use super::{Importer, Loader};
+use crate::history::History;
+
+// Note: both HistoryFile and HistoryData have other keys present in the JSON, we don't
+// care about them so we leave them unspecified so as to avoid deserializing unnecessarily.
+#[derive(Debug, Deserialize)]
+struct HistoryFile {
+    data: HistoryData,
+}
+
+#[derive(Debug, Deserialize)]
+struct HistoryData {
+    sessionid: String,
+    cmds: Vec<HistoryCmd>,
+}
+
+#[derive(Debug, Deserialize)]
+struct HistoryCmd {
+    cwd: String,
+    inp: String,
+    rtn: Option<i64>,
+    ts: (f64, f64),
+}
+
+#[derive(Debug)]
+pub struct Xonsh {
+    // history is stored as a bunch of json files, one per session
+    sessions: Vec<HistoryData>,
+    hostname: String,
+}
+
+fn get_hist_dir(xonsh_data_dir: Option<String>) -> Result<PathBuf> {
+    // if running within xonsh, this will be available
+    if let Some(d) = xonsh_data_dir {
+        let mut path = PathBuf::from(d);
+        path.push("history_json");
+        return Ok(path);
+    }
+
+    // otherwise, fall back to default
+    let base = BaseDirs::new().ok_or_else(|| eyre!("Could not determine home directory"))?;
+
+    let hist_dir = base.data_dir().join("xonsh/history_json");
+    if hist_dir.exists() || cfg!(test) {
+        Ok(hist_dir)
+    } else {
+        Err(eyre!("Could not find xonsh history files"))
+    }
+}
+
+fn get_hostname() -> String {
+    format!(
+        "{}:{}",
+        env::var("ATUIN_HOST_NAME").unwrap_or_else(|_| whoami::hostname()),
+        env::var("ATUIN_HOST_USER").unwrap_or_else(|_| whoami::username()),
+    )
+}
+
+fn load_sessions(hist_dir: &Path) -> Result<Vec<HistoryData>> {
+    let mut sessions = vec![];
+    for entry in fs::read_dir(hist_dir)? {
+        let p = entry?.path();
+        let ext = p.extension().and_then(|e| e.to_str());
+        if p.is_file() && ext == Some("json") {
+            if let Some(data) = load_session(&p)? {
+                sessions.push(data);
+            }
+        }
+    }
+    Ok(sessions)
+}
+
+fn load_session(path: &Path) -> Result<Option<HistoryData>> {
+    let file = File::open(path)?;
+    // empty files are not valid json, so we can't deserialize them
+    if file.metadata()?.len() == 0 {
+        return Ok(None);
+    }
+
+    let mut hist_file: HistoryFile = serde_json::from_reader(file)?;
+
+    // if there are commands in this session, replace the existing UUIDv4
+    // with a UUIDv7 generated from the timestamp of the first command
+    if let Some(cmd) = hist_file.data.cmds.first() {
+        let seconds = cmd.ts.0.trunc() as u64;
+        let nanos = (cmd.ts.0.fract() * 1_000_000_000_f64) as u32;
+        let ts = Timestamp::from_unix(NoContext, seconds, nanos);
+        hist_file.data.sessionid = Uuid::new_v7(ts).to_string();
+    }
+    Ok(Some(hist_file.data))
+}
+
+#[async_trait]
+impl Importer for Xonsh {
+    const NAME: &'static str = "xonsh";
+
+    async fn new() -> Result<Self> {
+        let hist_dir = get_hist_dir(env::var("XONSH_DATA_DIR").ok())?;
+        let sessions = load_sessions(&hist_dir)?;
+        let hostname = get_hostname();
+        Ok(Xonsh { sessions, hostname })
+    }
+
+    async fn entries(&mut self) -> Result<usize> {
+        let total = self.sessions.iter().map(|s| s.cmds.len()).sum();
+        Ok(total)
+    }
+
+    async fn load(self, loader: &mut impl Loader) -> Result<()> {
+        for session in self.sessions {
+            for cmd in session.cmds {
+                let (start, end) = cmd.ts;
+                let ts_nanos = (start * 1_000_000_000_f64) as i128;
+                let timestamp = OffsetDateTime::from_unix_timestamp_nanos(ts_nanos)?;
+
+                let duration = (end - start) * 1_000_000_000_f64;
+
+                match cmd.rtn {
+                    Some(exit) => {
+                        let entry = History::import()
+                            .timestamp(timestamp)
+                            .duration(duration.trunc() as i64)
+                            .exit(exit)
+                            .command(cmd.inp.trim())
+                            .cwd(cmd.cwd)
+                            .session(session.sessionid.clone())
+                            .hostname(self.hostname.clone());
+                        loader.push(entry.build().into()).await?;
+                    }
+                    None => {
+                        let entry = History::import()
+                            .timestamp(timestamp)
+                            .duration(duration.trunc() as i64)
+                            .command(cmd.inp.trim())
+                            .cwd(cmd.cwd)
+                            .session(session.sessionid.clone())
+                            .hostname(self.hostname.clone());
+                        loader.push(entry.build().into()).await?;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use time::macros::datetime;
+
+    use super::*;
+
+    use crate::history::History;
+    use crate::import::tests::TestLoader;
+
+    #[test]
+    fn test_hist_dir_xonsh() {
+        let hist_dir = get_hist_dir(Some("/home/user/xonsh_data".to_string())).unwrap();
+        assert_eq!(
+            hist_dir,
+            PathBuf::from("/home/user/xonsh_data/history_json")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_import() {
+        let dir = PathBuf::from("tests/data/xonsh");
+        let sessions = load_sessions(&dir).unwrap();
+        let hostname = "box:user".to_string();
+        let xonsh = Xonsh { sessions, hostname };
+
+        let mut loader = TestLoader::default();
+        xonsh.load(&mut loader).await.unwrap();
+        // order in buf will depend on filenames, so sort by timestamp for consistency
+        loader.buf.sort_by_key(|h| h.timestamp);
+        for (actual, expected) in loader.buf.iter().zip(expected_hist_entries().iter()) {
+            assert_eq!(actual.timestamp, expected.timestamp);
+            assert_eq!(actual.command, expected.command);
+            assert_eq!(actual.cwd, expected.cwd);
+            assert_eq!(actual.exit, expected.exit);
+            assert_eq!(actual.duration, expected.duration);
+            assert_eq!(actual.hostname, expected.hostname);
+        }
+    }
+
+    fn expected_hist_entries() -> [History; 4] {
+        [
+            History::import()
+                .timestamp(datetime!(2024-02-6 04:17:59.478272256 +00:00:00))
+                .command("echo hello world!".to_string())
+                .cwd("/home/user/Documents/code/atuin".to_string())
+                .exit(0)
+                .duration(4651069)
+                .hostname("box:user".to_string())
+                .build()
+                .into(),
+            History::import()
+                .timestamp(datetime!(2024-02-06 04:18:01.70632832 +00:00:00))
+                .command("ls -l".to_string())
+                .cwd("/home/user/Documents/code/atuin".to_string())
+                .exit(0)
+                .duration(21288633)
+                .hostname("box:user".to_string())
+                .build()
+                .into(),
+            History::import()
+                .timestamp(datetime!(2024-02-06 17:41:31.142515968 +00:00:00))
+                .command("false".to_string())
+                .cwd("/home/user/Documents/code/atuin/atuin-client".to_string())
+                .exit(1)
+                .duration(10269403)
+                .hostname("box:user".to_string())
+                .build()
+                .into(),
+            History::import()
+                .timestamp(datetime!(2024-02-06 17:41:32.271584 +00:00:00))
+                .command("exit".to_string())
+                .cwd("/home/user/Documents/code/atuin/atuin-client".to_string())
+                .exit(0)
+                .duration(4259347)
+                .hostname("box:user".to_string())
+                .build()
+                .into(),
+        ]
+    }
+}
diff --git a/atuin-client/src/import/xonsh_sqlite.rs b/atuin-client/src/import/xonsh_sqlite.rs
new file mode 100644
index 000000000..8310c375c
--- /dev/null
+++ b/atuin-client/src/import/xonsh_sqlite.rs
@@ -0,0 +1,222 @@
+use std::env;
+use std::path::PathBuf;
+
+use async_trait::async_trait;
+use directories::BaseDirs;
+use eyre::{eyre, Result};
+use futures::TryStreamExt;
+use sqlx::{sqlite::SqlitePool, FromRow, Row};
+use time::OffsetDateTime;
+use uuid::timestamp::{context::NoContext, Timestamp};
+use uuid::Uuid;
+
+use super::{Importer, Loader};
+use crate::history::History;
+
+#[derive(Debug, FromRow)]
+struct HistDbEntry {
+    inp: String,
+    rtn: Option<i64>,
+    tsb: f64,
+    tse: f64,
+    cwd: String,
+    session_start: f64,
+}
+
+impl HistDbEntry {
+    fn into_hist_with_hostname(self, hostname: String) -> History {
+        let ts_nanos = (self.tsb * 1_000_000_000_f64) as i128;
+        let timestamp = OffsetDateTime::from_unix_timestamp_nanos(ts_nanos).unwrap();
+
+        let session_ts_seconds = self.session_start.trunc() as u64;
+        let session_ts_nanos = (self.session_start.fract() * 1_000_000_000_f64) as u32;
+        let session_ts = Timestamp::from_unix(NoContext, session_ts_seconds, session_ts_nanos);
+        let session_id = Uuid::new_v7(session_ts).to_string();
+        let duration = (self.tse - self.tsb) * 1_000_000_000_f64;
+
+        if let Some(exit) = self.rtn {
+            let imported = History::import()
+                .timestamp(timestamp)
+                .duration(duration.trunc() as i64)
+                .exit(exit)
+                .command(self.inp)
+                .cwd(self.cwd)
+                .session(session_id)
+                .hostname(hostname);
+            imported.build().into()
+        } else {
+            let imported = History::import()
+                .timestamp(timestamp)
+                .duration(duration.trunc() as i64)
+                .command(self.inp)
+                .cwd(self.cwd)
+                .session(session_id)
+                .hostname(hostname);
+            imported.build().into()
+        }
+    }
+}
+
+fn get_db_path(xonsh_data_dir: Option<String>) -> Result<PathBuf> {
+    // if running within xonsh, this will be available
+    if let Some(d) = xonsh_data_dir {
+        let mut path = PathBuf::from(d);
+        path.push("xonsh-history.sqlite");
+        return Ok(path);
+    }
+
+    // otherwise, fall back to default
+    let base = BaseDirs::new().ok_or_else(|| eyre!("Could not determine home directory"))?;
+
+    let hist_file = base.data_dir().join("xonsh/xonsh-history.sqlite");
+    if hist_file.exists() || cfg!(test) {
+        Ok(hist_file)
+    } else {
+        Err(eyre!(
+            "Could not find xonsh history db at: {}",
+            hist_file.to_string_lossy()
+        ))
+    }
+}
+
+fn get_hostname() -> String {
+    format!(
+        "{}:{}",
+        env::var("ATUIN_HOST_NAME").unwrap_or_else(|_| whoami::hostname()),
+        env::var("ATUIN_HOST_USER").unwrap_or_else(|_| whoami::username()),
+    )
+}
+
+#[derive(Debug)]
+pub struct XonshSqlite {
+    pool: SqlitePool,
+    hostname: String,
+}
+
+#[async_trait]
+impl Importer for XonshSqlite {
+    const NAME: &'static str = "xonsh_sqlite";
+
+    async fn new() -> Result<Self> {
+        let db_path = get_db_path(env::var("XONSH_DATA_DIR").ok())?;
+        let connection_str = db_path.to_str().ok_or_else(|| {
+            eyre!(
+                "Invalid path for SQLite database: {}",
+                db_path.to_string_lossy()
+            )
+        })?;
+
+        let pool = SqlitePool::connect(connection_str).await?;
+        let hostname = get_hostname();
+        Ok(XonshSqlite { pool, hostname })
+    }
+
+    async fn entries(&mut self) -> Result<usize> {
+        let query = "SELECT COUNT(*) FROM xonsh_history";
+        let row = sqlx::query(query).fetch_one(&self.pool).await?;
+        let count: u32 = row.get(0);
+        Ok(count as usize)
+    }
+
+    async fn load(self, loader: &mut impl Loader) -> Result<()> {
+        let query = r#"
+            SELECT inp, rtn, tsb, tse, cwd,
+            MIN(tsb) OVER (PARTITION BY sessionid) AS session_start
+            FROM xonsh_history
+            ORDER BY rowid
+        "#;
+
+        let mut entries = sqlx::query_as::<_, HistDbEntry>(query).fetch(&self.pool);
+
+        let mut count = 0;
+        while let Some(entry) = entries.try_next().await? {
+            let hist = entry.into_hist_with_hostname(self.hostname.clone());
+            loader.push(hist).await?;
+            count += 1;
+        }
+
+        println!("Loaded: {count}");
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use time::macros::datetime;
+
+    use super::*;
+
+    use crate::history::History;
+    use crate::import::tests::TestLoader;
+
+    #[test]
+    fn test_db_path_xonsh() {
+        let db_path = get_db_path(Some("/home/user/xonsh_data".to_string())).unwrap();
+        assert_eq!(
+            db_path,
+            PathBuf::from("/home/user/xonsh_data/xonsh-history.sqlite")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_import() {
+        let connection_str = "tests/data/xonsh-history.sqlite";
+        let xonsh_sqlite = XonshSqlite {
+            pool: SqlitePool::connect(connection_str).await.unwrap(),
+            hostname: "box:user".to_string(),
+        };
+
+        let mut loader = TestLoader::default();
+        xonsh_sqlite.load(&mut loader).await.unwrap();
+
+        for (actual, expected) in loader.buf.iter().zip(expected_hist_entries().iter()) {
+            assert_eq!(actual.timestamp, expected.timestamp);
+            assert_eq!(actual.command, expected.command);
+            assert_eq!(actual.cwd, expected.cwd);
+            assert_eq!(actual.exit, expected.exit);
+            assert_eq!(actual.duration, expected.duration);
+            assert_eq!(actual.hostname, expected.hostname);
+        }
+    }
+
+    fn expected_hist_entries() -> [History; 4] {
+        [
+            History::import()
+                .timestamp(datetime!(2024-02-6 17:56:21.130956288 +00:00:00))
+                .command("echo hello world!".to_string())
+                .cwd("/home/user/Documents/code/atuin".to_string())
+                .exit(0)
+                .duration(2628564)
+                .hostname("box:user".to_string())
+                .build()
+                .into(),
+            History::import()
+                .timestamp(datetime!(2024-02-06 17:56:28.190406144 +00:00:00))
+                .command("ls -l".to_string())
+                .cwd("/home/user/Documents/code/atuin".to_string())
+                .exit(0)
+                .duration(9371519)
+                .hostname("box:user".to_string())
+                .build()
+                .into(),
+            History::import()
+                .timestamp(datetime!(2024-02-06 17:56:46.989020928 +00:00:00))
+                .command("false".to_string())
+                .cwd("/home/user/Documents/code/atuin".to_string())
+                .exit(1)
+                .duration(17337560)
+                .hostname("box:user".to_string())
+                .build()
+                .into(),
+            History::import()
+                .timestamp(datetime!(2024-02-06 17:56:48.218384128 +00:00:00))
+                .command("exit".to_string())
+                .cwd("/home/user/Documents/code/atuin".to_string())
+                .exit(0)
+                .duration(4599094)
+                .hostname("box:user".to_string())
+                .build()
+                .into(),
+        ]
+    }
+}