From 2c2e95f854dcac1eaab9e9dcb5aacee94cf92dc8 Mon Sep 17 00:00:00 2001 From: Kornel Date: Fri, 10 Apr 2020 14:40:04 +0100 Subject: Stream datadump --- datadump/Cargo.toml | 3 ++- datadump/src/main.rs | 5 ++++- server/Cargo.toml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/datadump/Cargo.toml b/datadump/Cargo.toml index 1ec6d75..6fd6454 100644 --- a/datadump/Cargo.toml +++ b/datadump/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datadump" -version = "0.1.0" +version = "0.1.1" authors = ["Kornel "] edition = "2018" @@ -13,3 +13,4 @@ serde_derive = "1.0.104" chrono = "0.4.10" kitchen_sink = { path = "../kitchen_sink", version = "0.9" } tokio = { version = "0.2", features = ["rt-threaded", "macros"] } +reqwest = { version = "0.10.4", features = ["blocking"] } diff --git a/datadump/src/main.rs b/datadump/src/main.rs index 3ec6c4a..de2cd92 100644 --- a/datadump/src/main.rs +++ b/datadump/src/main.rs @@ -21,7 +21,10 @@ type BoxErr = Box; async fn main() -> Result<(), BoxErr> { tokio::runtime::Handle::current().spawn(async move { let handle = tokio::runtime::Handle::current(); - let mut a = Archive::new(Decoder::new(BufReader::new(File::open("db-dump.tar.gz")?))?); + /// I can't be bothered to make async stream adapter to make async body impl Read + let res = reqwest::blocking::get("https://static.crates.io/db-dump.tar.gz")?; + let res = BufReader::with_capacity(256_000_000, res); + let mut a = Archive::new(Decoder::new(res)?); let ksink = KitchenSink::new_default().await?; tokio::task::block_in_place(move || { diff --git a/server/Cargo.toml b/server/Cargo.toml index 1002d36..7a39ee5 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "crates-server" -version = "0.13.1" +version = "0.13.2" authors = ["Kornel "] edition = "2018" description = "Crates.rs web server" -- cgit v1.2.3