From 0cf5be505470b2bcaf837d93239283be1f9a4a1e Mon Sep 17 00:00:00 2001 From: Sameer Puri <11097096+sameer@users.noreply.github.com> Date: Mon, 31 Dec 2018 11:46:50 -0500 Subject: Implement capability to add files/directories by path to IPFS --- ipfs-api/Cargo.toml | 1 + ipfs-api/src/client.rs | 81 +++++++++++++++++++++++++++++++++++++++++++++ ipfs-api/src/lib.rs | 1 + ipfs-cli/src/command/add.rs | 14 ++++---- ipfs-cli/src/command/mod.rs | 3 +- 5 files changed, 93 insertions(+), 7 deletions(-) diff --git a/ipfs-api/Cargo.toml b/ipfs-api/Cargo.toml index 53cb91e..3904dc1 100644 --- a/ipfs-api/Cargo.toml +++ b/ipfs-api/Cargo.toml @@ -27,6 +27,7 @@ serde_urlencoded = "0.5" tokio = "0.1" tokio-codec = "0.1" tokio-io = "0.1" +walkdir = "2.2" [dev-dependencies] tokio-timer = "0.2" diff --git a/ipfs-api/src/client.rs b/ipfs-api/src/client.rs index 92946ea..430612e 100644 --- a/ipfs-api/src/client.rs +++ b/ipfs-api/src/client.rs @@ -9,6 +9,7 @@ use futures::{ stream::{self, Stream}, Future, IntoFuture, + future, }; use header::TRAILER; use http::uri::InvalidUri; @@ -24,6 +25,7 @@ use response::{self, Error}; use serde::{Deserialize, Serialize}; use serde_json; use std::io::Read; +use std::path::{Path, PathBuf}; use tokio_codec::{Decoder, FramedRead}; /// A response returned by the HTTP client. @@ -351,6 +353,85 @@ impl IpfsClient { self.request(&request::Add, Some(form)) } + /// Add a path to Ipfs. Can be a file or directory. + /// A hard limit of 128 open file descriptors is set such + /// that any small additional files are stored in-memory. + /// + /// # Examples + /// + /// ```no_run + /// # extern crate ipfs_api; + /// # + /// use ipfs_api::IpfsClient; + /// + /// # fn main() { + /// let client = IpfsClient::default(); + /// let path = "./src"; + /// let req = client.add_path(path); + /// # } + /// ``` + /// + #[inline] + pub fn add_path

(&self, path: P) -> AsyncResponse + where + P: AsRef, + { + let mut form = multipart::Form::default(); + + let prefix = path.as_ref().parent(); + + let mut paths_to_add: Vec<(PathBuf, u64)> = vec![]; + + for path in walkdir::WalkDir::new(path.as_ref()) { + match path { + Ok(entry) => { + if entry.file_type().is_file() { + let file_size = + entry.metadata().map(|metadata| metadata.len()).unwrap_or(0); + paths_to_add.push((entry.path().to_path_buf(), file_size)); + } + } + Err(err) => { + return Box::new(future::err(Error::Io(err.into()))); + } + } + } + + paths_to_add.sort_unstable_by(|(_, a), (_, b)| a.cmp(b).reverse()); + + let mut it = 0; + const FILE_DESCRIPTOR_LIMIT: usize = 127; + + for (path, file_size) in paths_to_add { + let file = std::fs::File::open(&path); + if file.is_err() { + return Box::new(future::err(file.unwrap_err().into())); + } + let file_name = match prefix { + Some(prefix) => path.strip_prefix(prefix).unwrap(), + None => path.as_path(), + } + .to_string_lossy(); + + if it < FILE_DESCRIPTOR_LIMIT { + form.add_reader_file("path", file.unwrap(), file_name); + it += 1; + } else { + let mut buf = Vec::with_capacity(file_size as usize); + if let Err(err) = file.unwrap().read_to_end(&mut buf) { + return Box::new(future::err(err.into())); + } + form.add_reader_file("path", std::io::Cursor::new(buf), file_name); + } + } + + Box::new( + self.request_stream_json(&request::Add, Some(form)) + .collect() + .map(|mut responses: Vec| responses.pop().unwrap()), + ) + } + /// Returns the current ledger for a peer. /// /// # Examples diff --git a/ipfs-api/src/lib.rs b/ipfs-api/src/lib.rs index 2a89b32..4314cbd 100644 --- a/ipfs-api/src/lib.rs +++ b/ipfs-api/src/lib.rs @@ -97,6 +97,7 @@ extern crate serde_urlencoded; extern crate tokio; extern crate tokio_codec; extern crate tokio_io; +extern crate walkdir; pub use client::IpfsClient; pub use request::{KeyType, Logger, LoggingLevel, ObjectTemplate}; diff --git a/ipfs-cli/src/command/add.rs b/ipfs-cli/src/command/add.rs index 4382231..52505ef 100644 --- a/ipfs-cli/src/command/add.rs +++ b/ipfs-cli/src/command/add.rs @@ -7,9 +7,9 @@ // use clap::App; -use command::{verify_file, CliCommand, EXPECTED_FILE}; +use command::{CliCommand, EXPECTED_FILE}; use futures::Future; -use std::fs::File; +use std::path::Path; pub struct Command; @@ -20,17 +20,19 @@ impl CliCommand for Command { clap_app!( @subcommand add => (about: "Add file to IPFS") - (@arg INPUT: +required {verify_file} "File to add") + (@arg INPUT: +required "File to add") + (@arg recursive: -r --recursive "Add directory paths recursively. Default: false") ) } handle!( (args, client) => { let path = args.value_of("INPUT").unwrap(); - let file = File::open(path).expect(EXPECTED_FILE); - + if !args.is_present("recursive") && Path::new(path).is_dir() { + panic!(EXPECTED_FILE); + } client - .add(file) + .add_path(path) .map(|response| { println!(); println!(" name : {}", response.name); diff --git a/ipfs-cli/src/command/mod.rs b/ipfs-cli/src/command/mod.rs index 3f6d4b3..969f872 100644 --- a/ipfs-cli/src/command/mod.rs +++ b/ipfs-cli/src/command/mod.rs @@ -6,6 +6,7 @@ // copied, modified, or distributed except according to those terms. // +use std::path::Path; use clap::{App, ArgMatches}; use futures::Future; use ipfs_api::IpfsClient; @@ -18,7 +19,7 @@ pub const EXPECTED_FILE: &str = "expected to read input file"; /// Verifies that a path points to a file that exists, and not a directory. /// -pub fn verify_file(path: String) -> Result<(), String> { +pub fn verify_file

(path: P) -> Result<(), String> where P: AsRef { match fs::metadata(path) { Ok(ref metadata) if metadata.is_file() => Ok(()), Ok(_) => Err("file must not be a directory".into()), -- cgit v1.2.3