From 4b2ecab8aa60b11c4eef37e306a863be4679899d Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Thu, 1 Jul 2021 10:23:01 +0200 Subject: Refactor: Move helper fn to in outermost scope Signed-off-by: Matthias Beyer --- src/commands/source.rs | 86 +++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) (limited to 'src/commands') diff --git a/src/commands/source.rs b/src/commands/source.rs index 3e0167d..36cf352 100644 --- a/src/commands/source.rs +++ b/src/commands/source.rs @@ -214,6 +214,49 @@ pub async fn download( repo: Repository, progressbars: ProgressBars, ) -> Result<()> { + async fn perform_download(source: &SourceEntry, bar: &indicatif::ProgressBar) -> Result<()> { + trace!("Creating: {:?}", source); + let file = source.create().await.with_context(|| { + anyhow!( + "Creating source file destination: {}", + source.path().display() + ) + })?; + + let mut file = tokio::io::BufWriter::new(file); + let response = match reqwest::get(source.url().as_ref()).await { + Ok(resp) => resp, + Err(e) => { + bar.finish_with_message(format!("Failed: {}", source.url())); + return Err(e).with_context(|| anyhow!("Downloading '{}'", source.url())) + } + }; + + if let Some(len) = response.content_length() { + bar.set_length(len); + } + + let mut stream = reqwest::get(source.url().as_ref()).await?.bytes_stream(); + let mut bytes_written = 0; + while let Some(bytes) = stream.next().await { + let bytes = bytes?; + file.write_all(bytes.as_ref()).await?; + bytes_written += bytes.len(); + + bar.inc(bytes.len() as u64); + if let Some(len) = response.content_length() { + bar.set_message(format!("Downloading {} ({}/{} bytes)", source.url(), bytes_written, len)); + } else { + bar.set_message(format!("Downloading {} ({} bytes)", source.url(), bytes_written)); + } + } + + file.flush() + .await + .map_err(Error::from) + .map(|_| ()) + } + let force = matches.is_present("force"); let cache = PathBuf::from(config.source_cache_root()); let sc = SourceCache::new(cache); @@ -260,49 +303,6 @@ pub async fn download( if source_path_exists && !force { Err(anyhow!("Source exists: {}", source.path().display())) } else { - async fn perform_download(source: &SourceEntry, bar: &indicatif::ProgressBar) -> Result<()> { - trace!("Creating: {:?}", source); - let file = source.create().await.with_context(|| { - anyhow!( - "Creating source file destination: {}", - source.path().display() - ) - })?; - - let mut file = tokio::io::BufWriter::new(file); - let response = match reqwest::get(source.url().as_ref()).await { - Ok(resp) => resp, - Err(e) => { - bar.finish_with_message(format!("Failed: {}", source.url())); - return Err(e).with_context(|| anyhow!("Downloading '{}'", source.url())) - } - }; - - if let Some(len) = response.content_length() { - bar.set_length(len); - } - - let mut stream = reqwest::get(source.url().as_ref()).await?.bytes_stream(); - let mut bytes_written = 0; - while let Some(bytes) = stream.next().await { - let bytes = bytes?; - file.write_all(bytes.as_ref()).await?; - bytes_written += bytes.len(); - - bar.inc(bytes.len() as u64); - if let Some(len) = response.content_length() { - bar.set_message(format!("Downloading {} ({}/{} bytes)", source.url(), bytes_written, len)); - } else { - bar.set_message(format!("Downloading {} ({} bytes)", source.url(), bytes_written)); - } - } - - file.flush() - .await - .map_err(Error::from) - .map(|_| ()) - } - if source_path_exists /* && force is implied by 'if' above*/ { if let Err(e) = source.remove_file().await { bar.finish_with_message(format!("Failed to remove existing file: {}", source.path().display())); -- cgit v1.2.3 From d2cc4a734a808afc4a2fc0f8661d65090e8ccffb Mon Sep 17 00:00:00 2001 From: Matthias Beyer Date: Thu, 1 Jul 2021 10:30:22 +0200 Subject: Fix: Follow HTTP redirects This patch fixes the reqwest GET call by building a Client and a Request object configured to follow HTTP redirects (10 by now, maybe this will later need to be configurable). Signed-off-by: Matthias Beyer --- src/commands/source.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src/commands') diff --git a/src/commands/source.rs b/src/commands/source.rs index 36cf352..3ad7f5d 100644 --- a/src/commands/source.rs +++ b/src/commands/source.rs @@ -224,7 +224,16 @@ pub async fn download( })?; let mut file = tokio::io::BufWriter::new(file); - let response = match reqwest::get(source.url().as_ref()).await { + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::limited(10)) + .build() + .context("Building HTTP client failed")?; + + let request = client.get(source.url().as_ref()) + .build() + .with_context(|| anyhow!("Building request for {} failed", source.url().as_ref()))?; + + let response = match client.execute(request).await { Ok(resp) => resp, Err(e) => { bar.finish_with_message(format!("Failed: {}", source.url())); -- cgit v1.2.3