From 29f4a93e300a2eca9af2f878bb4808e68ff32966 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 1 Jan 2024 14:12:20 +0000 Subject: feat: rework record sync for improved reliability So, to tell a story 1. We introduced the record sync, intended to be the new algorithm to sync history. 2. On top of this, I added the KV store. This was intended as a simple test of the record sync, and to see if people wanted that sort of functionality 3. History remained syncing via the old means, as while it had issues it worked more-or-less OK. And we are aware of its flaws 4. If KV syncing worked ok, history would be moved across KV syncing ran ok for 6mo or so, so I started to move across history. For several weeks, I ran a local fork of Atuin + the server that synced via records instead. The record store maintained ordering via a linked list, which was a mistake. It performed well in testing, but was really difficult to debug and reason about. So when a few small sync issues occured, they took an extremely long time to debug. This PR is huge, which I regret. It involves replacing the "parent" relationship that records once had (pointing to the previous record) with a simple index (generally referred to as idx). This also means we had to change the recordindex, which referenced "tails". Tails were the last item in the chain. Now that we use an "array" vs linked list, that logic was also replaced. And is much simpler :D Same for the queries that act on this data. ---- This isn't final - we still need to add 1. Proper server/client error handling, which has been lacking for a while 2. The actual history implementation on top This exists in a branch, just without deletions. Won't be much to add that, I just don't want to make this any larger than it already is The _only_ caveat here is that we basically lose data synced via the old record store. This is the KV data from before. It hasn't been deleted or anything, just no longer hooked up. So it's totally possible to write a migration script. I just need to do that. --- .github/DISCUSSION_TEMPLATE/support.yml | 84 ---- .gitignore | 1 + Cargo.lock | 324 ++++++-------- Cargo.toml | 14 +- Dockerfile | 2 +- README.md | 21 +- atuin-client/Cargo.toml | 1 + atuin-client/config.toml | 2 +- .../20231127090831_create-store.sql | 15 + atuin-client/src/api_client.rs | 89 +--- atuin-client/src/history.rs | 210 ++++++++- atuin-client/src/history/store.rs | 52 +++ atuin-client/src/kv.rs | 99 ++--- atuin-client/src/record/encryption.rs | 29 +- atuin-client/src/record/sqlite_store.rs | 260 ++++++----- atuin-client/src/record/store.rs | 35 +- atuin-client/src/record/sync.rs | 493 +++++++++++++-------- atuin-client/src/settings.rs | 22 +- atuin-common/Cargo.toml | 3 - atuin-common/src/api.rs | 11 - atuin-common/src/record.rs | 171 ++++--- atuin-common/src/utils.rs | 30 +- atuin-server-database/src/lib.rs | 6 +- .../migrations/20231202170508_create-store.sql | 15 + .../migrations/20231203124112_create-store-idx.sql | 2 + atuin-server-postgres/src/lib.rs | 98 ++-- atuin-server-postgres/src/wrappers.rs | 7 +- atuin-server/Cargo.toml | 5 - atuin-server/server.toml | 5 - atuin-server/src/handlers/mod.rs | 1 + atuin-server/src/handlers/record.rs | 106 +---- atuin-server/src/handlers/v0/mod.rs | 1 + atuin-server/src/handlers/v0/record.rs | 111 +++++ atuin-server/src/lib.rs | 75 +--- atuin-server/src/router.rs | 28 +- atuin-server/src/settings.rs | 54 +-- atuin/Cargo.toml | 2 +- atuin/src/command/client.rs | 24 +- atuin/src/command/client/history.rs | 25 +- atuin/src/command/client/record.rs | 63 +++ atuin/src/command/client/search.rs | 12 - atuin/src/command/client/sync.rs | 28 +- atuin/src/command/server.rs | 5 +- atuin/src/interactive/inspector.rs | 37 ++ atuin/src/shell/atuin.bash | 5 + atuin/tests/sync.rs | 5 +- docs/docs/advanced-install.md | 10 - docs/docs/guide/index.md | 14 +- docs/docs/self-hosting/self-hosting.md | 11 - install.sh | 2 +- 50 files changed, 1515 insertions(+), 1210 deletions(-) delete mode 100644 .github/DISCUSSION_TEMPLATE/support.yml create mode 100644 atuin-client/record-migrations/20231127090831_create-store.sql create mode 100644 atuin-client/src/history/store.rs create mode 100644 atuin-server-postgres/migrations/20231202170508_create-store.sql create mode 100644 atuin-server-postgres/migrations/20231203124112_create-store-idx.sql create mode 100644 atuin-server/src/handlers/v0/mod.rs create mode 100644 atuin-server/src/handlers/v0/record.rs create mode 100644 atuin/src/command/client/record.rs create mode 100644 atuin/src/interactive/inspector.rs diff --git a/.github/DISCUSSION_TEMPLATE/support.yml b/.github/DISCUSSION_TEMPLATE/support.yml deleted file mode 100644 index b6a6ae40..00000000 --- a/.github/DISCUSSION_TEMPLATE/support.yml +++ /dev/null @@ -1,84 +0,0 @@ -body: - - type: input - attributes: - label: Operating System - description: What operating system are you using? - placeholder: "Example: macOS Big Sur" - validations: - required: true - - - type: input - attributes: - label: Shell - description: What shell are you using? - placeholder: "Example: zsh 5.8.1" - validations: - required: true - - - type: dropdown - attributes: - label: Version - description: What version of atuin are you running? - multiple: false - options: # how often will I forget to update this? a lot. - - v17.0.0 (Default) - - v16.0.0 - - v15.0.0 - - v14.0.1 - - v14.0.0 - - v13.0.1 - - v13.0.0 - - v12.0.0 - - v11.0.0 - - v0.10.0 - - v0.9.1 - - v0.9.0 - - v0.8.1 - - v0.8.0 - - v0.7.2 - - v0.7.1 - - v0.7.0 - - v0.6.4 - - v0.6.3 - default: 0 - validations: - required: true - - - type: checkboxes - attributes: - label: Self hosted - description: Are you self hosting atuin server? - options: - - label: I am self hosting atuin server - - - type: checkboxes - attributes: - label: Search the issues - description: Did you search the issues and discussions for your problem? - options: - - label: I checked that someone hasn't already asked about the same issue - required: true - - - type: textarea - attributes: - label: Behaviour - description: "Please describe the issue - what you expected to happen, what actually happened" - - - type: textarea - attributes: - label: Logs - description: "If possible, please include logs from atuin, especially if you self host the server - ATUIN_LOG=debug" - - - type: textarea - attributes: - label: Extra information - description: "Anything else you'd like to add?" - - - type: checkboxes - attributes: - label: Code of Conduct - description: The Code of Conduct helps create a safe space for everyone. We require - that everyone agrees to it. - options: - - label: I agree to follow this project's [Code of Conduct](https://github.com/atuinsh/atuin/blob/main/CODE_OF_CONDUCT.md) - required: true diff --git a/.gitignore b/.gitignore index 17c0b070..64caee2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store /target */target .env diff --git a/Cargo.lock b/Cargo.lock index 440f3024..adae9140 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,9 +57,9 @@ checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -77,30 +77,30 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -109,12 +109,6 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" -[[package]] -name = "arc-swap" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" - [[package]] name = "argon2" version = "0.5.2" @@ -135,7 +129,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -147,12 +141,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" - [[package]] name = "atomic-write-file" version = "0.1.2" @@ -186,7 +174,7 @@ dependencies = [ "fuzzy-matcher", "indicatif", "interim", - "itertools 0.12.0", + "itertools", "log", "ratatui", "rpassword", @@ -221,7 +209,7 @@ dependencies = [ "generic-array", "hex", "interim", - "itertools 0.12.0", + "itertools", "lazy_static", "log", "memchr", @@ -242,6 +230,7 @@ dependencies = [ "shellexpand", "sql-builder", "sqlx", + "thiserror", "time", "tokio", "typed-builder", @@ -255,10 +244,8 @@ name = "atuin-common" version = "17.1.0" dependencies = [ "eyre", - "lazy_static", "pretty_assertions", "rand", - "semver", "serde", "sqlx", "time", @@ -275,20 +262,15 @@ dependencies = [ "atuin-common", "atuin-server-database", "axum", - "axum-server", "base64 0.21.5", "config", "eyre", "fs-err", "http", - "hyper", - "hyper-rustls", "metrics", "metrics-exporter-prometheus", "rand", "reqwest", - "rustls", - "rustls-pemfile", "semver", "serde", "serde_json", @@ -383,26 +365,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "axum-server" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447f28c85900215cc1bea282f32d4a2f22d55c5a300afdfbc661c8d6a632e063" -dependencies = [ - "arc-swap", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "pin-project-lite", - "rustls", - "rustls-pemfile", - "tokio", - "tokio-rustls", - "tower-service", -] - [[package]] name = "backtrace" version = "0.3.69" @@ -581,9 +543,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" dependencies = [ "clap_builder", "clap_derive", @@ -591,9 +553,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" dependencies = [ "anstream", "anstyle", @@ -619,7 +581,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -661,10 +623,11 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "colored" -version = "2.1.0" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" dependencies = [ + "is-terminal", "lazy_static", "windows-sys 0.48.0", ] @@ -698,9 +661,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.6" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "core-foundation" @@ -744,21 +707,22 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", "memoffset 0.9.0", + "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ "cfg-if", "crossbeam-utils", @@ -766,9 +730,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] @@ -860,7 +824,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -876,11 +840,10 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ - "powerfmt", "serde", ] @@ -924,11 +887,11 @@ dependencies = [ [[package]] name = "directories" -version = "5.0.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" dependencies = [ - "dirs-sys", + "dirs-sys 0.3.7", ] [[package]] @@ -937,7 +900,18 @@ version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys", + "dirs-sys 0.4.1", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", ] [[package]] @@ -1070,9 +1044,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "eyre" -version = "0.6.11" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6267a1fa6f59179ea4afc8e50fd8612a3cc60bc858f786ff877a4a8cb042799" +checksum = "80f656be11ddf91bd709454d15d5bd896fbaf4cc3314e69349e4d1569f5b46cd" dependencies = [ "indenter", "once_cell", @@ -1215,7 +1189,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -1372,9 +1346,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.12.4" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" dependencies = [ "hmac", ] @@ -1390,11 +1364,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -1410,9 +1384,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", @@ -1476,9 +1450,7 @@ dependencies = [ "futures-util", "http", "hyper", - "log", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", ] @@ -1601,20 +1573,11 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" -dependencies = [ - "either", -] - [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" @@ -1636,9 +1599,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libm" @@ -1710,7 +1673,7 @@ dependencies = [ "proc-macro2", "quote", "regex-syntax 0.6.29", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -1811,9 +1774,9 @@ dependencies = [ [[package]] name = "metrics-exporter-prometheus" -version = "0.12.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" +checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" dependencies = [ "base64 0.21.5", "hyper", @@ -1835,7 +1798,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -1882,9 +1845,9 @@ checksum = "1269a17ac308ae0b906ec1b0ff8062fd0c82f18cc2956faa367302ec3380f4e8" [[package]] name = "mio" -version = "0.8.10" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "log", @@ -1935,15 +1898,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "nu-ansi-term" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c073d3c1930d0751774acf49e66653acecb416c3a54c6ec095a9b11caddb5a68" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "num" version = "0.2.1" @@ -2104,9 +2058,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opaque-debug" @@ -2250,7 +2204,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -2311,15 +2265,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - -[[package]] -name = "powerfmt" -version = "0.2.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" [[package]] name = "ppv-lite86" @@ -2411,7 +2359,7 @@ dependencies = [ "cassowary", "crossterm", "indoc", - "itertools 0.11.0", + "itertools", "lru", "paste", "strum", @@ -2549,9 +2497,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" dependencies = [ "cc", "getrandom", @@ -2645,9 +2593,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" dependencies = [ "bitflags 2.4.1", "errno", @@ -2658,12 +2606,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.6", "rustls-webpki", "sct", ] @@ -2695,7 +2643,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.7", + "ring 0.17.6", "untrusted 0.9.0", ] @@ -2748,9 +2696,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "salsa20" @@ -2782,7 +2730,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.7", + "ring 0.17.6", "untrusted 0.9.0", ] @@ -2817,22 +2765,22 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -3036,11 +2984,11 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" dependencies = [ - "itertools 0.12.0", + "itertools", "nom", "unicode_categories", ] @@ -3295,7 +3243,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -3317,9 +3265,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.41" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -3377,22 +3325,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -3407,15 +3355,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07" dependencies = [ "deranged", "itoa", "libc", "num_threads", - "powerfmt", "serde", "time-core", "time-macros", @@ -3423,15 +3370,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451" dependencies = [ "time-core", ] @@ -3472,9 +3419,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -3497,7 +3444,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -3611,7 +3558,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -3626,9 +3573,9 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.2.0" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" dependencies = [ "log", "once_cell", @@ -3642,7 +3589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "matchers", - "nu-ansi-term 0.46.0", + "nu-ansi-term", "once_cell", "regex", "sharded-slab", @@ -3653,11 +3600,11 @@ dependencies = [ [[package]] name = "tracing-tree" -version = "0.3.0" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65139ecd2c3f6484c3b99bc01c77afe21e95473630747c7aca525e78b0666675" +checksum = "2ec6adcab41b1391b08a308cc6302b79f8095d1673f6947c2dc65ffb028b0b2d" dependencies = [ - "nu-ansi-term 0.49.0", + "nu-ansi-term", "tracing-core", "tracing-log", "tracing-subscriber", @@ -3679,28 +3626,28 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typed-builder" -version = "0.18.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47c0496149861b7c95198088cbf36645016b1a0734cf350c50e2a38e070f38a" +checksum = "7fe83c85a85875e8c4cb9ce4a890f05b23d38cd0d47647db7895d3d2a79566d2" dependencies = [ "typed-builder-macro", ] [[package]] name = "typed-builder-macro" -version = "0.18.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982ee4197351b5c9782847ef5ec1fdcaf50503fb19d68f9771adae314e72b492" +checksum = "29a3151c41d0b13e3d011f98adc24434560ef06673a155a6c7f66b9879eecce2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -3711,9 +3658,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" @@ -3799,7 +3746,6 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ - "atomic", "getrandom", "serde", ] @@ -3858,7 +3804,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", "wasm-bindgen-shared", ] @@ -3892,7 +3838,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4299,22 +4245,22 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.31" +version = "0.7.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.31" +version = "0.7.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] [[package]] @@ -4334,5 +4280,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.39", ] diff --git a/Cargo.toml b/Cargo.toml index b245d259..334629f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,20 +29,24 @@ time = { version = "0.3", features = [ ] } clap = { version = "4.0.18", features = ["derive"] } config = { version = "0.13", default-features = false, features = ["toml"] } -directories = "5.0.1" +directories = "4" eyre = "0.6" fs-err = "2.9" interim = { version = "0.1.0", features = ["time"] } -itertools = "0.12.0" +itertools = "0.11.0" rand = { version = "0.8.5", features = ["std"] } semver = "1.0.20" -serde = { version = "1.0.193", features = ["derive"] } +# https://github.com/serde-rs/serde/issues/2538 +# I don't trust dtolnay with our user's builds. especially as we +# have things like encryption keys +serde = { version = "1.0.145, <=1.0.171", features = ["derive"] } serde_json = "1.0.108" tokio = { version = "1", features = ["full"] } -uuid = { version = "1.3", features = ["v4", "v7", "serde"] } +uuid = { version = "1.3", features = ["v4", "serde"] } whoami = "1.1.2" -typed-builder = "0.18.0" +typed-builder = "0.15.0" pretty_assertions = "1.3.0" +thiserror = "1.0" [workspace.dependencies.reqwest] version = "0.11" diff --git a/Dockerfile b/Dockerfile index 8897e96a..a126a253 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lukemathwalker/cargo-chef:latest-rust-1.74.1-buster AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.74.0-buster AS chef WORKDIR app FROM chef AS planner diff --git a/README.md b/README.md index b016077a..a793a243 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -#

+

- Text changing depending on mode. Light: 'So light!' Dark: 'So dark!' + Atuin logo

@@ -19,7 +19,6 @@ - Arm CI sponsored by Actuated

@@ -108,11 +107,7 @@ This will sign you up for the default sync server, hosted by me. Everything is e Read more below for offline-only usage, or for hosting your own server. ``` -# bash/zsh/etc -bash <(curl --proto '=https' --tlsv1.2 -sSf https://setup.atuin.sh) - -# fish -bash (curl --proto '=https' --tlsv1.2 -sSf https://setup.atuin.sh | psub) +bash <(curl https://raw.githubusercontent.com/atuinsh/atuin/main/install.sh) atuin register -u -e atuin import auto @@ -217,16 +212,6 @@ pacman -S atuin And then follow [the shell setup](#shell-plugin) -### Xbps - -Atuin is available in the Void Linux [repository](https://github.com/void-linux/void-packages/tree/master/srcpkgs/atuin): - -``` -sudo xbps-install atuin -``` - -And then follow [the shell setup](#shell-plugin) - ### Termux Atuin is available in the Termux package repository: diff --git a/atuin-client/Cargo.toml b/atuin-client/Cargo.toml index 951b3274..f03ef4d2 100644 --- a/atuin-client/Cargo.toml +++ b/atuin-client/Cargo.toml @@ -48,6 +48,7 @@ rmp = { version = "0.8.11" } typed-builder = { workspace = true } tokio = { workspace = true } semver = { workspace = true } +thiserror = { workspace = true } futures = "0.3" crypto_secretbox = "0.1.1" generic-array = { version = "0.14", features = ["serde"] } diff --git a/atuin-client/config.toml b/atuin-client/config.toml index 511662cc..2e333b92 100644 --- a/atuin-client/config.toml +++ b/atuin-client/config.toml @@ -39,7 +39,7 @@ # filter_mode = "global" ## With workspace filtering enabled, Atuin will filter for commands executed -## in any directory within a git repository tree (default: false) +## in any directory within a git repositiry tree (default: false) # workspaces = false ## which filter mode to use when atuin is invoked from a shell up-key binding diff --git a/atuin-client/record-migrations/20231127090831_create-store.sql b/atuin-client/record-migrations/20231127090831_create-store.sql new file mode 100644 index 00000000..53d78860 --- /dev/null +++ b/atuin-client/record-migrations/20231127090831_create-store.sql @@ -0,0 +1,15 @@ +-- Add migration script here +create table if not exists store ( + id text primary key, -- globally unique ID + + idx integer, -- incrementing integer ID unique per (host, tag) + host text not null, -- references the host row + tag text not null, + + timestamp integer not null, + version text not null, + data blob not null, + cek blob not null +); + +create unique index record_uniq ON store(host, tag, idx); diff --git a/atuin-client/src/api_client.rs b/atuin-client/src/api_client.rs index ae8df5ad..359b0464 100644 --- a/atuin-client/src/api_client.rs +++ b/atuin-client/src/api_client.rs @@ -5,19 +5,16 @@ use std::time::Duration; use eyre::{bail, Result}; use reqwest::{ header::{HeaderMap, AUTHORIZATION, USER_AGENT}, - Response, StatusCode, Url, + StatusCode, Url, }; +use atuin_common::record::{EncryptedData, HostId, Record, RecordIdx}; use atuin_common::{ api::{ AddHistoryRequest, CountResponse, DeleteHistoryRequest, ErrorResponse, IndexResponse, LoginRequest, LoginResponse, RegisterResponse, StatusResponse, SyncHistoryResponse, }, - record::RecordIndex, -}; -use atuin_common::{ - api::{ATUIN_CARGO_VERSION, ATUIN_HEADER_VERSION, ATUIN_VERSION}, - record::{EncryptedData, HostId, Record, RecordId}, + record::RecordStatus, }; use semver::Version; use time::format_description::well_known::Rfc3339; @@ -55,15 +52,10 @@ pub async fn register( let resp = client .post(url) .header(USER_AGENT, APP_USER_AGENT) - .header(ATUIN_HEADER_VERSION, ATUIN_CARGO_VERSION) .json(&map) .send() .await?; - if !ensure_version(&resp)? { - bail!("could not register user due to version mismatch"); - } - if !resp.status().is_success() { let error = resp.json::().await?; bail!("failed to register user: {}", error.reason); @@ -84,10 +76,6 @@ pub async fn login(address: &str, req: LoginRequest) -> Result { .send() .await?; - if !ensure_version(&resp)? { - bail!("could not login due to version mismatch"); - } - if resp.status() != reqwest::StatusCode::OK { let error = resp.json::().await?; bail!("invalid login details: {}", error.reason); @@ -118,31 +106,6 @@ pub async fn latest_version() -> Result { Ok(version) } -pub fn ensure_version(response: &Response) -> Result { - let version = response.headers().get(ATUIN_HEADER_VERSION); - - let version = if let Some(version) = version { - match version.to_str() { - Ok(v) => Version::parse(v), - Err(e) => bail!("failed to parse server version: {:?}", e), - } - } else { - // if there is no version header, then the newest this server can possibly be is 17.1.0 - Version::parse("17.1.0") - }?; - - // If the client is newer than the server - if version.major < ATUIN_VERSION.major { - println!("Atuin version mismatch! In order to successfully sync, the server needs to run a newer version of Atuin"); - println!("Client: {}", ATUIN_CARGO_VERSION); - println!("Server: {}", version); - - return Ok(false); - } - - Ok(true) -} - impl<'a> Client<'a> { pub fn new( sync_addr: &'a str, @@ -153,9 +116,6 @@ impl<'a> Client<'a> { let mut headers = HeaderMap::new(); headers.insert(AUTHORIZATION, format!("Token {session_token}").parse()?); - // used for semver server check - headers.insert(ATUIN_HEADER_VERSION, ATUIN_CARGO_VERSION.parse()?); - Ok(Client { sync_addr, client: reqwest::Client::builder() @@ -173,10 +133,6 @@ impl<'a> Client<'a> { let resp = self.client.get(url).send().await?; - if !ensure_version(&resp)? { - bail!("could not sync due to version mismatch"); - } - if resp.status() != StatusCode::OK { bail!("failed to get count (are you logged in?)"); } @@ -192,10 +148,6 @@ impl<'a> Client<'a> { let resp = self.client.get(url).send().await?; - if !ensure_version(&resp)? { - bail!("could not sync due to version mismatch"); - } - if resp.status() != StatusCode::OK { bail!("failed to get status (are you logged in?)"); } @@ -279,24 +231,22 @@ impl<'a> Client<'a> { &self, host: HostId, tag: String, - start: Option, + start: RecordIdx, count: u64, ) -> Result>> { + debug!( + "fetching record/s from host {}/{}/{}", + host.0.to_string(), + tag, + start + ); + let url = format!( - "{}/record/next?host={}&tag={}&count={}", - self.sync_addr, host.0, tag, count + "{}/record/next?host={}&tag={}&count={}&start={}", + self.sync_addr, host.0, tag, count, start ); - let mut url = Url::parse(url.as_str())?; - - if let Some(start) = start { - url.set_query(Some( - format!( - "host={}&tag={}&count={}&start={}", - host.0, tag, count, start.0 - ) - .as_str(), - )); - } + + let url = Url::parse(url.as_str())?; let resp = self.client.get(url).send().await?; @@ -305,18 +255,15 @@ impl<'a> Client<'a> { Ok(records) } - pub async fn record_index(&self) -> Result { + pub async fn record_status(&self) -> Result { let url = format!("{}/record", self.sync_addr); let url = Url::parse(url.as_str())?; let resp = self.client.get(url).send().await?; - - if !ensure_version(&resp)? { - bail!("could not sync records due to version mismatch"); - } - let index = resp.json().await?; + debug!("got remote index {:?}", index); + Ok(index) } diff --git a/atuin-client/src/history.rs b/atuin-client/src/history.rs index fbcb169c..fdd7649c 100644 --- a/atuin-client/src/history.rs +++ b/atuin-client/src/history.rs @@ -1,12 +1,21 @@ +use rmp::decode::ValueReadError; +use rmp::{decode::Bytes, Marker}; use std::env; +use atuin_common::record::DecryptedData; use atuin_common::utils::uuid_v7; + +use eyre::{bail, eyre, Result}; use regex::RegexSet; use crate::{secrets::SECRET_PATTERNS, settings::Settings}; use time::OffsetDateTime; mod builder; +pub mod store; + +const HISTORY_VERSION: &str = "v0"; +const _HISTORY_TAG: &str = "history"; /// Client-side history entry. /// @@ -81,6 +90,108 @@ impl History { } } + pub fn serialize(&self) -> Result { + // This is pretty much the same as what we used for the old history, with one difference - + // it uses integers for timestamps rather than a string format. + + use rmp::encode; + + let mut output = vec![]; + + // write the version + encode::write_u16(&mut output, 0)?; + // INFO: ensure this is updated when adding new fields + encode::write_array_len(&mut output, 9)?; + + encode::write_str(&mut output, &self.id)?; + encode::write_u64(&mut output, self.timestamp.unix_timestamp_nanos() as u64)?; + encode::write_sint(&mut output, self.duration)?; + encode::write_sint(&mut output, self.exit)?; + encode::write_str(&mut output, &self.command)?; + encode::write_str(&mut output, &self.cwd)?; + encode::write_str(&mut output, &self.session)?; + encode::write_str(&mut output, &self.hostname)?; + + match self.deleted_at { + Some(d) => encode::write_u64(&mut output, d.unix_timestamp_nanos() as u64)?, + None => encode::write_nil(&mut output)?, + } + + Ok(DecryptedData(output)) + } + + fn deserialize_v0(bytes: &[u8]) -> Result { + use rmp::decode; + + fn error_report(err: E) -> eyre::Report { + eyre!("{err:?}") + } + + let mut bytes = Bytes::new(bytes); + + let version = decode::read_u16(&mut bytes).map_err(error_report)?; + + if version != 0 { + bail!("expected decoding v0 record, found v{version}"); + } + + let nfields = decode::read_array_len(&mut bytes).map_err(error_report)?; + + if nfields != 9 { + bail!("cannot decrypt history from a different version of Atuin"); + } + + let bytes = bytes.remaining_slice(); + let (id, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; + + let mut bytes = Bytes::new(bytes); + let timestamp = decode::read_u64(&mut bytes).map_err(error_report)?; + let duration = decode::read_int(&mut bytes).map_err(error_report)?; + let exit = decode::read_int(&mut bytes).map_err(error_report)?; + + let bytes = bytes.remaining_slice(); + let (command, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; + let (cwd, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; + let (session, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; + let (hostname, bytes) = decode::read_str_from_slice(bytes).map_err(error_report)?; + + // if we have more fields, try and get the deleted_at + let mut bytes = Bytes::new(bytes); + + let (deleted_at, bytes) = match decode::read_u64(&mut bytes) { + Ok(unix) => (Some(unix), bytes.remaining_slice()), + // we accept null here + Err(ValueReadError::TypeMismatch(Marker::Null)) => (None, bytes.remaining_slice()), + Err(err) => return Err(error_report(err)), + }; + + if !bytes.is_empty() { + bail!("trailing bytes in encoded history. malformed") + } + + Ok(History { + id: id.to_owned(), + timestamp: OffsetDateTime::from_unix_timestamp_nanos(timestamp as i128)?, + duration, + exit, + command: command.to_owned(), + cwd: cwd.to_owned(), + session: session.to_owned(), + hostname: hostname.to_owned(), + deleted_at: deleted_at + .map(|t| OffsetDateTime::from_unix_timestamp_nanos(t as i128)) + .transpose()?, + }) + } + + pub fn deserialize(bytes: &[u8], version: &str) -> Result { + match version { + HISTORY_VERSION => Self::deserialize_v0(bytes), + + _ => bail!("unknown version {version:?}"), + } + } + /// Builder for a history entry that is imported from shell history. /// /// The only two required fields are `timestamp` and `command`. @@ -202,8 +313,9 @@ impl History { #[cfg(test)] mod tests { use regex::RegexSet; + use time::macros::datetime; - use crate::settings::Settings; + use crate::{history::HISTORY_VERSION, settings::Settings}; use super::History; @@ -274,4 +386,100 @@ mod tests { assert!(stripe_key.should_save(&settings)); } + + #[test] + fn test_serialize_deserialize() { + let bytes = [ + 205, 0, 0, 153, 217, 32, 54, 54, 100, 49, 54, 99, 98, 101, 101, 55, 99, 100, 52, 55, + 53, 51, 56, 101, 53, 99, 53, 98, 56, 98, 52, 52, 101, 57, 48, 48, 54, 101, 207, 23, 99, + 98, 117, 24, 210, 246, 128, 206, 2, 238, 210, 240, 0, 170, 103, 105, 116, 32, 115, 116, + 97, 116, 117, 115, 217, 42, 47, 85, 115, 101, 114, 115, 47, 99, 111, 110, 114, 97, 100, + 46, 108, 117, 100, 103, 97, 116, 101, 47, 68, 111, 99, 117, 109, 101, 110, 116, 115, + 47, 99, 111, 100, 101, 47, 97, 116, 117, 105, 110, 217, 32, 98, 57, 55, 100, 57, 97, + 51, 48, 54, 102, 50, 55, 52, 52, 55, 51, 97, 50, 48, 51, 100, 50, 101, 98, 97, 52, 49, + 102, 57, 52, 53, 55, 187, 102, 118, 102, 103, 57, 51, 54, 99, 48, 107, 112, 102, 58, + 99, 111, 110, 114, 97, 100, 46, 108, 117, 100, 103, 97, 116, 101, 192, + ]; + + let history = History { + id: "66d16cbee7cd47538e5c5b8b44e9006e".to_owned(), + timestamp: datetime!(2023-05-28 18:35:40.633872 +00:00), + duration: 49206000, + exit: 0, + command: "git status".to_owned(), + cwd: "/Users/conrad.ludgate/Documents/code/atuin".to_owned(), + session: "b97d9a306f274473a203d2eba41f9457".to_owned(), + hostname: "fvfg936c0kpf:conrad.ludgate".to_owned(), + deleted_at: None, + }; + + let serialized = history.serialize().expect("failed to serialize history"); + assert_eq!(serialized.0, bytes); + + let deserialized = History::deserialize(&serialized.0, HISTORY_VERSION) + .expect("failed to deserialize history"); + assert_eq!(history, deserialized); + + // test the snapshot too + let deserialized = + History::deserialize(&bytes, HISTORY_VERSION).expect("failed to deserialize history"); + assert_eq!(history, deserialized); + } + + #[test] + fn test_serialize_deserialize_deleted() { + let history = History { + id: "66d16cbee7cd47538e5c5b8b44e9006e".to_owned(), + timestamp: datetime!(2023-05-28 18:35:40.633872 +00:00), + duration: 49206000, + exit: 0, + command: "git status".to_owned(), + cwd: "/Users/conrad.ludgate/Documents/code/atuin".to_owned(), + session: "b97d9a306f274473a203d2eba41f9457".to_owned(), + hostname: "fvfg936c0kpf:conrad.ludgate".to_owned(), + deleted_at: Some(datetime!(2023-11-19 20:18 +00:00)), + }; + + let serialized = history.serialize().expect("failed to serialize history"); + + let deserialized = History::deserialize(&serialized.0, HISTORY_VERSION) + .expect("failed to deserialize history"); + + assert_eq!(history, deserialized); + } + + #[test] + fn test_serialize_deserialize_version() { + // v0 + let bytes_v0 = [ + 205, 0, 0, 153, 217, 32, 54, 54, 100, 49, 54, 99, 98, 101, 101, 55, 99, 100, 52, 55, + 53, 51, 56, 101, 53, 99, 53, 98, 56, 98, 52, 52, 101, 57, 48, 48, 54, 101, 207, 23, 99, + 98, 117, 24, 210, 246, 128, 206, 2, 238, 210, 240, 0, 170, 103, 105, 116, 32, 115, 116, + 97, 116, 117, 115, 217, 42, 47, 85, 115, 101, 114, 115, 47, 99, 111, 110, 114, 97, 100, + 46, 108, 117, 100, 103, 97, 116, 101, 47, 68, 111, 99, 117, 109, 101, 110, 116, 115, + 47, 99, 111, 100, 101, 47, 97, 116, 117, 105, 110, 217, 32, 98, 57, 55, 100, 57, 97, + 51, 48, 54, 102, 50, 55, 52, 52, 55, 51, 97, 50, 48, 51, 100, 50, 101, 98, 97, 52, 49, + 102, 57, 52, 53, 55, 187, 102, 118, 102, 103, 57, 51, 54, 99, 48, 107, 112, 102, 58, + 99, 111, 110, 114, 97, 100, 46, 108, 117, 100, 103, 97, 116, 101, 192, + ]; + + // some other version + let bytes_v1 = [ + 205, 1, 0, 153, 217, 32, 54, 54, 100, 49, 54, 99, 98, 101, 101, 55, 99, 100, 52, 55, + 53, 51, 56, 101, 53, 99, 53, 98, 56, 98, 52, 52, 101, 57, 48, 48, 54, 101, 207, 23, 99, + 98, 117, 24, 210, 246, 128, 206, 2, 238, 210, 240, 0, 170, 103, 105, 116, 32, 115, 116, + 97, 116, 117, 115, 217, 42, 47, 85, 115, 101, 114, 115, 47, 99, 111, 110, 114, 97, 100, + 46, 108, 117, 100, 103, 97, 116, 101, 47, 68, 111, 99, 117, 109, 101, 110, 116, 115, + 47, 99, 111, 100, 101, 47, 97, 116, 117, 105, 110, 217, 32, 98, 57, 55, 100, 57, 97, + 51, 48, 54, 102, 50, 55, 52, 52, 55, 51, 97, 50, 48, 51, 100, 50, 101, 98, 97, 52, 49, + 102, 57, 52, 53, 55, 187, 102, 118, 102, 103, 57, 51, 54, 99, 48, 107, 112, 102, 58, + 99, 111, 110, 114, 97, 100, 46, 108, 117, 100, 103, 97, 116, 101, 192, + ]; + + let deserialized = History::deserialize(&bytes_v0, HISTORY_VERSION); + assert!(deserialized.is_ok()); + + let deserialized = History::deserialize(&bytes_v1, HISTORY_VERSION); + assert!(deserialized.is_err()); + } } diff --git a/atuin-client/src/history/store.rs b/atuin-client/src/history/store.rs new file mode 100644 index 00000000..1f242063 --- /dev/null +++ b/atuin-client/src/history/store.rs @@ -0,0 +1,52 @@ +use eyre::Result; + +use crate::record::sqlite_store::SqliteStore; +use atuin_common::record::HostId; + +use super::History; + +#[derive(Debug)] +pub struct HistoryStore { + pub store: SqliteStore, + pub host_id: HostId, + pub encryption_key: [u8; 32], +} + +impl HistoryStore { + pub fn new(store: SqliteStore, host_id: HostId, encryption_key: [u8; 32]) -> Self { + HistoryStore { + store, + host_id, + encryption_key, + } + } + + pub async fn push(&self, _history: &History) -> Result<()> { + Ok(()) + /* + * will continue this in another PR + * + * + let bytes = history.serialize()?; + let id = self + .store + .last(self.host_id, HISTORY_TAG) + .await? + .map_or(0, |p| p.idx + 1); + + let record = Record::builder() + .host(Host::new(self.host_id)) + .version(HISTORY_VERSION.to_string()) + .tag(HISTORY_TAG.to_string()) + .idx(id) + .data(bytes) + .build(); + + self.store + .push(&record.encrypt::(&self.encryption_key)) + .await?; + + Ok(()) + */ + } +} diff --git a/atuin-client/src/kv.rs b/atuin-client/src/kv.rs index 1ca6b5e8..b9c87858 100644 --- a/atuin-client/src/kv.rs +++ b/atuin-client/src/kv.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use atuin_common::record::{DecryptedData, HostId}; +use atuin_common::record::{DecryptedData, Host, HostId}; use eyre::{bail, ensure, eyre, Result}; use serde::Deserialize; @@ -111,13 +111,16 @@ impl KvStore { let bytes = record.serialize()?; - let parent = store.tail(host_id, KV_TAG).await?.map(|entry| entry.id); + let idx = store + .last(host_id, KV_TAG) + .await? + .map_or(0, |entry| entry.idx + 1); let record = atuin_common::record::Record::builder() - .host(host_id) + .host(Host::new(host_id)) .version(KV_VERSION.to_string()) .tag(KV_TAG.to_string()) - .parent(parent) + .idx(idx) .data(bytes) .build(); @@ -137,43 +140,18 @@ impl KvStore { namespace: &str, key: &str, ) -> Result> { - // Currently, this is O(n). When we have an actual KV store, it can be better - // Just a poc for now! + // TODO: don't rebuild every time... + let map = self.build_kv(store, encryption_key).await?; - // iterate records to find the value we want - // start at the end, so we get the most recent version - let tails = store.tag_tails(KV_TAG).await?; + let res = map.get(namespace); - if tails.is_empty() { - return Ok(None); - } - - // first, decide on a record. - // try getting the newest first - // we always need a way of deciding the "winner" of a write - // TODO(ellie): something better than last-write-wins, what if two write at the same time? - let mut record = tails.iter().max_by_key(|r| r.timestamp).unwrap().clone(); - - loop { - let decrypted = match record.version.as_str() { - KV_VERSION => record.decrypt::(encryption_key)?, - version => bail!("unknown version {version:?}"), - }; - - let kv = KvRecord::deserialize(&decrypted.data, &decrypted.version)?; - if kv.key == key && kv.namespace == namespace { - return Ok(Some(kv)); - } + if let Some(ns) = res { + let value = ns.get(key); - if let Some(parent) = decrypted.parent { - record = store.get(parent).await?; - } else { - break; - } + Ok(value.cloned()) + } else { + Ok(None) } - - // if we get here, then... we didn't find the record with that key :( - Ok(None) } // Build a kv map out of the linked list kv store @@ -184,32 +162,31 @@ impl KvStore { &self, store: &impl Store, encryption_key: &[u8; 32], - ) -> Result>> { + ) -> Result>> { let mut map = BTreeMap::new(); - let tails = store.tag_tails(KV_TAG).await?; - - if tails.is_empty() { - return Ok(map); - } - let mut record = tails.iter().max_by_key(|r| r.timestamp).unwrap().clone(); + // TODO: maybe don't load the entire tag into memory to build the kv + // we can be smart about it and only load values since the last build + // or, iterate/paginate + let tagged = store.all_tagged(KV_TAG).await?; - loop { + // iterate through all tags and play each KV record at a time + // this is "last write wins" + // probably good enough for now, but revisit in future + for record in tagged { let decrypted = match record.version.as_str() { KV_VERSION => record.decrypt::(encryption_key)?, version => bail!("unknown version {version:?}"), }; - let kv = KvRecord::deserialize(&decrypted.data, &decrypted.version)?; + let kv = KvRecord::deserialize(&decrypted.data, KV_VERSION)?; + println!("{:?}", kv); - let ns = map.entry(kv.namespace).or_insert_with(BTreeMap::new); - ns.entry(kv.key).or_insert_with(|| kv.value); + let ns = map + .entry(kv.namespace.clone()) + .or_insert_with(BTreeMap::new); - if let Some(parent) = decrypted.parent { - record = store.get(parent).await?; - } else { - break; - } + ns.insert(kv.key.clone(), kv); } Ok(map) @@ -261,19 +238,27 @@ mod tests { let map = kv.build_kv(&store, &key).await.unwrap(); assert_eq!( - map.get("test-kv") + *map.get("test-kv") .expect("map namespace not set") .get("foo") .expect("map key not set"), - "bar" + KvRecord { + namespace: String::from("test-kv"), + key: String::from("foo"), + value: String::from("bar") + } ); assert_eq!( - map.get("test-kv") + *map.get("test-kv") .expect("map namespace not set") .get("1") .expect("map key not set"), - "2" + KvRecord { + namespace: String::from("test-kv"), + key: String::from("1"), + value: String::from("2") + } ); } } diff --git a/atuin-client/src/record/encryption.rs b/atuin-client/src/record/encryption.rs index 3074a9c2..c2cdaa6a 100644 --- a/atuin-client/src/record/encryption.rs +++ b/atuin-client/src/record/encryption.rs @@ -1,5 +1,5 @@ use atuin_common::record::{ - AdditionalData, DecryptedData, EncryptedData, Encryption, HostId, RecordId, + AdditionalData, DecryptedData, EncryptedData, Encryption, HostId, RecordId, RecordIdx, }; use base64::{engine::general_purpose, Engine}; use eyre::{ensure, Context, Result}; @@ -170,10 +170,10 @@ struct AtuinFooter { #[derive(Debug, Copy, Clone, Serialize)] struct Assertions<'a> { id: &'a RecordId, + idx: &'a RecordIdx, version: &'a str, tag: &'a str, host: &'a HostId, - parent: Option<&'a RecordId>, } impl<'a> From> for Assertions<'a> { @@ -183,7 +183,7 @@ impl<'a> From> for Assertions<'a> { version: ad.version, tag: ad.tag, host: ad.host, - parent: ad.parent, + idx: ad.idx, } } } @@ -196,7 +196,10 @@ impl Assertions<'_> { #[cfg(test)] mod tests { - use atuin_common::{record::Record, utils::uuid_v7}; + use atuin_common::{ + record::{Host, Record}, + utils::uuid_v7, + }; use super::*; @@ -209,7 +212,7 @@ mod tests { version: "v0", tag: "kv", host: &HostId(uuid_v7()), - parent: None, + idx: &0, }; let data = DecryptedData(vec![1, 2, 3, 4]); @@ -228,7 +231,7 @@ mod tests { version: "v0", tag: "kv", host: &HostId(uuid_v7()), - parent: None, + idx: &0, }; let data = DecryptedData(vec![1, 2, 3, 4]); @@ -252,7 +255,7 @@ mod tests { version: "v0", tag: "kv", host: &HostId(uuid_v7()), - parent: None, + idx: &0, }; let data = DecryptedData(vec![1, 2, 3, 4]); @@ -270,7 +273,7 @@ mod tests { version: "v0", tag: "kv", host: &HostId(uuid_v7()), - parent: None, + idx: &0, }; let data = DecryptedData(vec![1, 2, 3, 4]); @@ -294,7 +297,7 @@ mod tests { version: "v0", tag: "kv", host: &HostId(uuid_v7()), - parent: None, + idx: &0, }; let data = DecryptedData(vec![1, 2, 3, 4]); @@ -323,9 +326,10 @@ mod tests { .id(RecordId(uuid_v7())) .version("v0".to_owned()) .tag("kv".to_owned()) - .host(HostId(uuid_v7())) + .host(Host::new(HostId(uuid_v7()))) .timestamp(1687244806000000) .data(DecryptedData(vec![1, 2, 3, 4])) + .idx(0) .build(); let encry