Implement capability to add files/directories by path to IPFS

5 files changed, 93 insertions, 7 deletions

diff --git a/ipfs-api/Cargo.toml b/ipfs-api/Cargo.toml
index 53cb91e..3904dc1 100644
--- a/ipfs-api/Cargo.toml
+++ b/ipfs-api/Cargo.toml
@@ -27,6 +27,7 @@ serde_urlencoded          = "0.5"
 tokio                     = "0.1"
 tokio-codec               = "0.1"
 tokio-io                  = "0.1"
+walkdir                   = "2.2"
 
 [dev-dependencies]
 tokio-timer               = "0.2"
diff --git a/ipfs-api/src/client.rs b/ipfs-api/src/client.rs
index 92946ea..430612e 100644
--- a/ipfs-api/src/client.rs
+++ b/ipfs-api/src/client.rs
@@ -9,6 +9,7 @@
 use futures::{
     stream::{self, Stream},
     Future, IntoFuture,
+    future,
 };
 use header::TRAILER;
 use http::uri::InvalidUri;
@@ -24,6 +25,7 @@ use response::{self, Error};
 use serde::{Deserialize, Serialize};
 use serde_json;
 use std::io::Read;
+use std::path::{Path, PathBuf};
 use tokio_codec::{Decoder, FramedRead};
 
 /// A response returned by the HTTP client.
@@ -351,6 +353,85 @@ impl IpfsClient {
         self.request(&request::Add, Some(form))
     }
 
+    /// Add a path to Ipfs. Can be a file or directory.
+    /// A hard limit of 128 open file descriptors is set such
+    /// that any small additional files are stored in-memory.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # extern crate ipfs_api;
+    /// #
+    /// use ipfs_api::IpfsClient;
+    ///
+    /// # fn main() {
+    /// let client = IpfsClient::default();
+    /// let path = "./src";
+    /// let req = client.add_path(path);
+    /// # }
+    /// ```
+    ///
+    #[inline]
+    pub fn add_path<P>(&self, path: P) -> AsyncResponse<response::AddResponse>
+    where
+        P: AsRef<Path>,
+    {
+        let mut form = multipart::Form::default();
+
+        let prefix = path.as_ref().parent();
+
+        let mut paths_to_add: Vec<(PathBuf, u64)> = vec![];
+
+        for path in walkdir::WalkDir::new(path.as_ref()) {
+            match path {
+                Ok(entry) => {
+                    if entry.file_type().is_file() {
+                        let file_size =
+                            entry.metadata().map(|metadata| metadata.len()).unwrap_or(0);
+                        paths_to_add.push((entry.path().to_path_buf(), file_size));
+                    }
+                }
+                Err(err) => {
+                    return Box::new(future::err(Error::Io(err.into())));
+                }
+            }
+        }
+
+        paths_to_add.sort_unstable_by(|(_, a), (_, b)| a.cmp(b).reverse());
+
+        let mut it = 0;
+        const FILE_DESCRIPTOR_LIMIT: usize = 127;
+
+        for (path, file_size) in paths_to_add {
+            let file = std::fs::File::open(&path);
+            if file.is_err() {
+                return Box::new(future::err(file.unwrap_err().into()));
+            }
+            let file_name = match prefix {
+                Some(prefix) => path.strip_prefix(prefix).unwrap(),
+                None => path.as_path(),
+            }
+            .to_string_lossy();
+
+            if it < FILE_DESCRIPTOR_LIMIT {
+                form.add_reader_file("path", file.unwrap(), file_name);
+                it += 1;
+            } else {
+                let mut buf = Vec::with_capacity(file_size as usize);
+                if let Err(err) = file.unwrap().read_to_end(&mut buf) {
+                    return Box::new(future::err(err.into()));
+                }
+                form.add_reader_file("path", std::io::Cursor::new(buf), file_name);
+            }
+        }
+
+        Box::new(
+            self.request_stream_json(&request::Add, Some(form))
+                .collect()
+                .map(|mut responses: Vec<response::AddResponse>| responses.pop().unwrap()),
+        )
+    }
+
     /// Returns the current ledger for a peer.
     ///
     /// # Examples
diff --git a/ipfs-api/src/lib.rs b/ipfs-api/src/lib.rs
index 2a89b32..4314cbd 100644
--- a/ipfs-api/src/lib.rs
+++ b/ipfs-api/src/lib.rs
@@ -97,6 +97,7 @@ extern crate serde_urlencoded;
 extern crate tokio;
 extern crate tokio_codec;
 extern crate tokio_io;
+extern crate walkdir;
 
 pub use client::IpfsClient;
 pub use request::{KeyType, Logger, LoggingLevel, ObjectTemplate};
diff --git a/ipfs-cli/src/command/add.rs b/ipfs-cli/src/command/add.rs
index 4382231..52505ef 100644
--- a/ipfs-cli/src/command/add.rs
+++ b/ipfs-cli/src/command/add.rs
@@ -7,9 +7,9 @@
 //
 
 use clap::App;
-use command::{verify_file, CliCommand, EXPECTED_FILE};
+use command::{CliCommand, EXPECTED_FILE};
 use futures::Future;
-use std::fs::File;
+use std::path::Path;
 
 pub struct Command;
 
@@ -20,17 +20,19 @@ impl CliCommand for Command {
         clap_app!(
             @subcommand add =>
                 (about: "Add file to IPFS")
-                (@arg INPUT: +required {verify_file} "File to add")
+                (@arg INPUT: +required "File to add")
+                (@arg recursive: -r --recursive "Add directory paths recursively. Default: false")
         )
     }
 
     handle!(
         (args, client) => {
             let path = args.value_of("INPUT").unwrap();
-            let file = File::open(path).expect(EXPECTED_FILE);
-
+            if !args.is_present("recursive") && Path::new(path).is_dir() {
+                panic!(EXPECTED_FILE);
+            }
             client
-                .add(file)
+                .add_path(path)
                 .map(|response| {
                     println!();
                     println!("  name    : {}", response.name);
diff --git a/ipfs-cli/src/command/mod.rs b/ipfs-cli/src/command/mod.rs
index 3f6d4b3..969f872 100644
--- a/ipfs-cli/src/command/mod.rs
+++ b/ipfs-cli/src/command/mod.rs
@@ -6,6 +6,7 @@
 // copied, modified, or distributed except according to those terms.
 //
 
+use std::path::Path;
 use clap::{App, ArgMatches};
 use futures::Future;
 use ipfs_api::IpfsClient;
@@ -18,7 +19,7 @@ pub const EXPECTED_FILE: &str = "expected to read input file";
 
 /// Verifies that a path points to a file that exists, and not a directory.
 ///
-pub fn verify_file(path: String) -> Result<(), String> {
+pub fn verify_file<P>(path: P) -> Result<(), String> where P: AsRef<Path> {
     match fs::metadata(path) {
         Ok(ref metadata) if metadata.is_file() => Ok(()),
         Ok(_) => Err("file must not be a directory".into()),