diff options
| author | Kornel <kornel@geekhood.net> | 2020-04-10 14:40:04 +0100 |
|---|---|---|
| committer | Kornel <kornel@geekhood.net> | 2020-04-10 14:40:30 +0100 |
| commit | 2c2e95f854dcac1eaab9e9dcb5aacee94cf92dc8 (patch) | |
| tree | a22d08903aacd551cc91a95507db03ec692b331f | |
| parent | ab582c45f206f62030b9d95dd8bf2150348fb030 (diff) | |
Stream datadump
| -rw-r--r-- | datadump/Cargo.toml | 3 | ||||
| -rw-r--r-- | datadump/src/main.rs | 5 | ||||
| -rw-r--r-- | server/Cargo.toml | 2 |
3 files changed, 7 insertions, 3 deletions
diff --git a/datadump/Cargo.toml b/datadump/Cargo.toml index 1ec6d75..6fd6454 100644 --- a/datadump/Cargo.toml +++ b/datadump/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadump" -version = "0.1.0" +version = "0.1.1" authors = ["Kornel <kornel@geekhood.net>"] edition = "2018" @@ -13,3 +13,4 @@ serde_derive = "1.0.104" chrono = "0.4.10" kitchen_sink = { path = "../kitchen_sink", version = "0.9" } tokio = { version = "0.2", features = ["rt-threaded", "macros"] } +reqwest = { version = "0.10.4", features = ["blocking"] } diff --git a/datadump/src/main.rs b/datadump/src/main.rs index 3ec6c4a..de2cd92 100644 --- a/datadump/src/main.rs +++ b/datadump/src/main.rs @@ -21,7 +21,10 @@ type BoxErr = Box<dyn std::error::Error + Sync + Send>; async fn main() -> Result<(), BoxErr> { tokio::runtime::Handle::current().spawn(async move { let handle = tokio::runtime::Handle::current(); - let mut a = Archive::new(Decoder::new(BufReader::new(File::open("db-dump.tar.gz")?))?); + /// I can't be bothered to make async stream adapter to make async body impl Read + let res = reqwest::blocking::get("https://static.crates.io/db-dump.tar.gz")?; + let res = BufReader::with_capacity(256_000_000, res); + let mut a = Archive::new(Decoder::new(res)?); let ksink = KitchenSink::new_default().await?; tokio::task::block_in_place(move || { diff --git a/server/Cargo.toml b/server/Cargo.toml index 1002d36..7a39ee5 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "crates-server" -version = "0.13.1" +version = "0.13.2" authors = ["Kornel <kornel@geekhood.net>"] edition = "2018" description = "Crates.rs web server" |