Rename testing crate to tools, and add README

9 files changed, 605 insertions, 0 deletions

diff --git a/tools/Cargo.toml b/tools/Cargo.toml
new file mode 100644
index 00000000..e855132f
--- /dev/null
+++ b/tools/Cargo.toml
@@ -0,0 +1,31 @@
+[package]
+name = "tools"
+version = "0.4.1"
+authors = ["Manos Pitsidianakis <el13635@mail.ntua.gr>"]
+workspace = ".."
+edition = "2018"
+
+[[bin]]
+name = "emailparse"
+path = "src/email_parse.rs"
+
+[[bin]]
+name = "mboxparse"
+path = "src/mboxparse.rs"
+
+[[bin]]
+name = "imapshell"
+path = "src/imapshell.rs"
+
+[[bin]]
+name = "smtp_conn"
+path = "src/smtp_conn.rs"
+
+[dependencies]
+melib = { path = "../melib", version = "*", features = ["debug-tracing", "unicode_algorithms"] }
+
+[features]
+default = []
+
+# Print tracing logs as meli runs
+debug-tracing = []
diff --git a/tools/README.md b/tools/README.md
new file mode 100644
index 00000000..de3e1b5f
--- /dev/null
+++ b/tools/README.md
@@ -0,0 +1,46 @@
+# meli tools
+
+This crate holds a collection of small binaries used for meli development. Of note is `imapshell` which opens a shell to an IMAP server for the user to play with.
+
+## imapshell
+
+```shell
+cd tools/
+cargo build --bin imapshell
+# Usage: imap_conn server_hostname server_username server_password server_port
+rlwrap ./target/debug/imapshell "mail.domain.tld" "epilys@domain.tld" "hunter2" 143
+```
+
+Example session:
+
+First, the IMAP connections performs its own non-interactive setup:
+
+```text
+[2020-08-27 17:11:33]["main"] melib/src/backends/imap/connection.rs:459_25: sent: M1 CAPABILITY
+[2020-08-27 17:11:33]["main"] melib/src/backends/imap/connection.rs:408_33: &ret[last_line_idx..] = "M1 OK Pre-login capabilities listed, post-login capabilities have more.\r\n"
+[2020-08-27 17:11:33]["main"] melib/src/backends/imap/connection.rs:459_25: sent: M2 LOGIN "epilys@domain.tld" "hunter2"
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/connection.rs:459_25: sent: M3 CAPABILITY
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/connection.rs:408_33: &ret[last_line_idx..] = "M3 OK Capability completed (0.000 + 0.120 + 0.119 secs).\r\n"
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/connection.rs:459_25: sent: M4 ENABLE CONDSTORE
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/connection.rs:408_33: &ret[last_line_idx..] = "M4 OK Enabled (0.000 + 0.120 + 0.119 secs).\r\n"
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/protocol_parser.rs:261_9: &val = "M4 OK Enabled (0.000 + 0.120 + 0.119 secs).\r\n"
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/connection.rs:459_25: sent: M5 COMPRESS DEFLATE
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/connection.rs:408_33: &ret[last_line_idx..] = "M5 OK Begin compression (0.000 + 0.127 + 0.126 secs).\r\n"
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/protocol_parser.rs:261_9: &val = "M5 OK Begin compression (0.000 + 0.127 + 0.126 secs).\r\n"
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/protocol_parser.rs:261_9: &val = "M5 OK Begin compression (0.000 + 0.127 + 0.126 secs).\r\n"
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/connection.rs:459_25: sent: M6 NOOP
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/connection.rs:408_33: &ret[last_line_idx..] = "M6 OK NOOP completed (0.000 + 0.000 secs).\r\n"
+[2020-08-27 17:11:34]["main"] melib/src/backends/imap/protocol_parser.rs:261_9: &val = "M6 OK NOOP completed (0.000 + 0.000 secs).\r\n"
+```
+
+Then, input is read line by line and sent to the server. You don't have to prefix the commands with a unique ID, that is taken care of by the tool. Example command and reply:
+
+```text
+LIST "" "INBOX"
+[2020-08-27 17:14:53]["main"] melib/src/backends/imap/connection.rs:717_9: send_command
+[2020-08-27 17:14:53]["main"] melib/src/backends/imap/connection.rs:437_9: stream send_command()
+[2020-08-27 17:14:53]["main"] melib/src/backends/imap/connection.rs:459_25: sent: M8 LIST "" "INBOX"
+[2020-08-27 17:14:53]["main"] melib/src/backends/imap/connection.rs:729_13: send_command ok
+[2020-08-27 17:14:53]["main"] melib/src/backends/imap.rs:1241_21: out: * LIST (\HasChildren) "." INBOX
+M8 OK List completed (0.000 + 0.000 secs).
+```
diff --git a/tools/src/email_parse.rs b/tools/src/email_parse.rs
new file mode 100644
index 00000000..aa240a0b
--- /dev/null
+++ b/tools/src/email_parse.rs
@@ -0,0 +1,33 @@
+extern crate melib;
+use melib::Result;
+use melib::*;
+
+/// Parses e-mail from files and prints the debug information of the parsed `Envelope`
+///
+/// # Example invocation
+/// ```sh
+/// ./emailparse /path/to/email [/path/to/email2 /path/to/email3 ..]"
+/// ```
+
+fn main() -> Result<()> {
+    if std::env::args().len() == 1 {
+        eprintln!("Usage: ./emailparse /path/to/email [/path/to/email2 /path/to/email3 ..]");
+        std::process::exit(1);
+    }
+
+    for i in std::env::args().skip(1) {
+        println!("Path is {}", i);
+        let filename = std::path::PathBuf::from(&i);
+
+        if filename.exists() && filename.is_file() {
+            let buffer = std::fs::read_to_string(&filename)
+                .expect(&format!("Something went wrong reading the file {}", i,));
+            let env = Envelope::from_bytes(&buffer.as_bytes(), None).expect("Couldn't parse email");
+            println!("Env is {:#?}", env);
+            println!("{:?}", env.body_bytes(buffer.as_bytes()));
+        } else {
+            println!("{} is not a valid file.", i);
+        }
+    }
+    Ok(())
+}
diff --git a/tools/src/embed.rs b/tools/src/embed.rs
new file mode 100644
index 00000000..e9183023
--- /dev/null
+++ b/tools/src/embed.rs
@@ -0,0 +1,7 @@
+extern crate ui;
+use ui::terminal::embed::create_pty;
+
+fn main() -> std::io::Result<()> {
+    create_pty().unwrap();
+    Ok(())
+}
diff --git a/tools/src/imapshell.rs b/tools/src/imapshell.rs
new file mode 100644
index 00000000..4a10a9db
--- /dev/null
+++ b/tools/src/imapshell.rs
@@ -0,0 +1,57 @@
+extern crate melib;
+
+use melib::backends::ImapType;
+use melib::{futures, smol, Result};
+use melib::{AccountSettings, BackendEventConsumer};
+
+/// Opens an interactive shell on an IMAP server. Suggested use is with rlwrap(1)
+///
+/// # Example invocation:
+/// ```sh
+/// ./imapshell server_hostname server_username server_password server_port");
+/// ```
+///
+/// `danger_accept_invalid_certs` is turned on by default, so no certificate validation is performed.
+
+fn main() -> Result<()> {
+    let mut args = std::env::args().skip(1).collect::<Vec<String>>();
+    if args.len() != 4 {
+        eprintln!("Usage: imapshell server_hostname server_username server_password server_port");
+        std::process::exit(1);
+    }
+
+    let (a, b, c, d) = (
+        std::mem::replace(&mut args[0], String::new()),
+        std::mem::replace(&mut args[1], String::new()),
+        std::mem::replace(&mut args[2], String::new()),
+        std::mem::replace(&mut args[3], String::new()),
+    );
+    let set = AccountSettings {
+        extra: [
+            ("server_hostname".to_string(), a),
+            ("server_username".to_string(), b),
+            ("server_password".to_string(), c),
+            ("server_port".to_string(), d),
+            (
+                "danger_accept_invalid_certs".to_string(),
+                "true".to_string(),
+            ),
+        ]
+        .iter()
+        .cloned()
+        .collect(),
+        ..Default::default()
+    };
+    let mut imap = ImapType::new(
+        &set,
+        Box::new(|_| true),
+        BackendEventConsumer::new(std::sync::Arc::new(|_, _| ())),
+    )?;
+
+    std::thread::spawn(|| smol::run(futures::future::pending::<()>()));
+    (imap.as_any_mut())
+        .downcast_mut::<ImapType>()
+        .unwrap()
+        .shell();
+    Ok(())
+}
diff --git a/tools/src/linebreak.rs b/tools/src/linebreak.rs
new file mode 100644
index 00000000..ec93603c
--- /dev/null
+++ b/tools/src/linebreak.rs
@@ -0,0 +1,161 @@
+extern crate melib;
+use melib::Result;
+
+extern crate text_processing;
+use text_processing::line_break::*;
+
+fn cost(i: usize, j: usize, width: usize, minima: &Vec<usize>, offsets: &Vec<usize>) -> usize {
+    let w = offsets[j] - offsets[i] + j - i - 1;
+    if w > width {
+        return 65536 * (w - width);
+    }
+    minima[i] + (width - w) * (width - w)
+}
+
+fn smawk(
+    rows: &mut Vec<usize>,
+    columns: &mut Vec<usize>,
+    minima: &mut Vec<usize>,
+    breaks: &mut Vec<usize>,
+    width: usize,
+    offsets: &Vec<usize>,
+) {
+    let mut stack = Vec::new();
+    let mut i = 0;
+    while i < rows.len() {
+        if stack.len() > 0 {
+            let c = columns[stack.len() - 1];
+            if cost(*stack.iter().last().unwrap(), c, width, minima, offsets)
+                < cost(rows[i], c, width, minima, offsets)
+            {
+                if stack.len() < columns.len() {
+                    stack.push(rows[i]);
+                }
+                i += 1;
+            } else {
+                stack.pop();
+            }
+        } else {
+            stack.push(rows[i]);
+            i += 1;
+        }
+    }
+    let rows = &mut stack;
+    if columns.len() > 1 {
+        let mut odd_columns = columns.iter().skip(1).step_by(2).cloned().collect();
+        smawk(rows, &mut odd_columns, minima, breaks, width, offsets);
+        for (i, o) in odd_columns.into_iter().enumerate() {
+            columns[2 * i + 1] = o;
+        }
+    }
+    let mut i = 0;
+    let mut j = 0;
+    while j < columns.len() {
+        let end = if j + 1 < columns.len() {
+            breaks[columns[j + 1]]
+        } else {
+            *rows.iter().last().unwrap()
+        };
+        let c = cost(rows[i], columns[j], width, minima, offsets);
+        if c < minima[columns[j]] {
+            minima[columns[j]] = c;
+            breaks[columns[j]] = rows[i];
+        }
+        if rows[i] < end {
+            i += 1;
+        } else {
+            j += 2;
+        }
+    }
+}
+
+fn linear(text: &str, width: usize) -> Vec<String> {
+    let mut words = Vec::new();
+    let breaks = LineBreakCandidateIter::new(text).collect::<Vec<(usize, LineBreakCandidate)>>();
+    {
+        let mut prev = 0;
+        for b in breaks {
+            if &text[prev..b.0] != "\n" {
+                words.push(text[prev..b.0].trim_end_matches("\n"));
+                if text[prev..b.0].ends_with("\n") {
+                    words.push(" ");
+                }
+            }
+            prev = b.0;
+        }
+        if &text[prev..] != "\n" {
+            words.push(text[prev..].trim_end_matches("\n"));
+        }
+    }
+    let count = words.len();
+    let mut minima = vec![std::usize::MAX - 1; count + 1];
+    minima[0] = 0;
+    let mut offsets = Vec::with_capacity(words.len());
+    offsets.push(0);
+    for w in words.iter() {
+        offsets.push(offsets.iter().last().unwrap() + w.len());
+    }
+
+    let mut breaks = vec![0; count + 1];
+
+    let mut n = count + 1;
+    let mut i = 1;
+    let mut offset = 0;
+    loop {
+        let r = std::cmp::min(n, 2 * i);
+        let edge = i + offset;
+        smawk(
+            &mut (offset..edge).collect(),
+            &mut (edge..(r + offset)).collect(),
+            &mut minima,
+            &mut breaks,
+            width,
+            &offsets,
+        );
+        let x = minima[r - 1 + offset];
+        let mut for_was_broken = false;
+        for j in i..(r - 1) {
+            let y = cost(j + offset, r - 1 + offset, width, &minima, &offsets);
+            if y <= x {
+                n -= j;
+                i = 1;
+                offset += j;
+                for_was_broken = true;
+                break;
+            }
+        }
+
+        if !for_was_broken || i >= (r - 1) {
+            if r == n {
+                break;
+            }
+            i *= 2;
+        }
+    }
+    let mut lines = Vec::new();
+    let mut j = count;
+    while j > 0 {
+        let mut line = String::new();
+        for i in breaks[j]..j {
+            line.push_str(words[i]);
+        }
+        lines.push(line);
+        j = breaks[j];
+    }
+    lines.reverse();
+    lines
+}
+
+fn main() -> Result<()> {
+    let text = std::fs::read_to_string(std::env::args().nth(1).unwrap())?;
+    let paragraphs = text.split("\n\n").collect::<Vec<&str>>();
+    for (i, p) in paragraphs.iter().enumerate() {
+        for l in linear(&p, 72) {
+            println!("{}", l.trim());
+        }
+        if i + 1 < paragraphs.len() {
+            println!("");
+        }
+    }
+    Ok(())
+}
diff --git a/tools/src/linebreak1.rs b/tools/src/linebreak1.rs
new file mode 100644
index 00000000..f5abceaa
--- /dev/null
+++ b/tools/src/linebreak1.rs
@@ -0,0 +1,150 @@
+extern crate melib;
+use melib::Result;
+use melib::StackVec;
+use std::str::FromStr;
+
+extern crate text_processing;
+use text_processing::line_break::*;
+use text_processing::Graphemes;
+
+fn cost(i: usize, j: usize, width: usize, minima: &Vec<usize>, offsets: &Vec<usize>) -> usize {
+    let w = offsets[j] + j - offsets[i] - i - 1;
+    if w > width {
+        return 65536 * (w - width);
+    }
+    minima[i] + (width - w) * (width - w)
+}
+
+fn smawk(
+    rows: &mut StackVec<usize>,
+    columns: &mut StackVec<usize>,
+    minima: &mut Vec<usize>,
+    breaks: &mut Vec<usize>,
+    width: usize,
+    offsets: &Vec<usize>,
+) {
+    let mut stack = StackVec::new();
+    let mut i = 0;
+    while i < rows.len() {
+        if stack.len() > 0 {
+            let c = columns[stack.len() - 1];
+            if cost(*stack.iter().last().unwrap(), c, width, minima, offsets)
+                < cost(rows[i], c, width, minima, offsets)
+            {
+                if stack.len() < columns.len() {
+                    stack.push(rows[i]);
+                }
+                i += 1;
+            } else {
+                stack.pop();
+            }
+        } else {
+            stack.push(rows[i]);
+            i += 1;
+        }
+    }
+    let rows = &mut stack;
+    if columns.len() > 1 {
+        let mut odd_columns = columns.iter().skip(1).step_by(2).cloned().collect();
+        smawk(rows, &mut odd_columns, minima, breaks, width, offsets);
+        for (i, o) in odd_columns.into_iter().enumerate() {
+            columns.set(2 * i + 1, o);
+        }
+    }
+    let mut i = 0;
+    let mut j = 0;
+    while j < columns.len() {
+        let end = if j + 1 < columns.len() {
+            breaks[columns[j + 1]]
+        } else {
+            *rows.iter().last().unwrap()
+        };
+        let c = cost(rows[i], columns[j], width, minima, offsets);
+        if c < minima[columns[j]] {
+            minima[columns[j]] = c;
+            breaks[columns[j]] = rows[i];
+        }
+        if rows[i] < end {
+            i += 1;
+        } else {
+            j += 2;
+        }
+    }
+}
+
+fn linear(text: &str, width: usize) -> Vec<String> {
+    let mut words = text.split_whitespace().collect::<Vec<&str>>();
+    let breaks = LineBreakCandidateIter::new(text).collect::<Vec<(usize, LineBreakCandidate)>>();
+    let count = words.len();
+    let mut minima = vec![std::usize::MAX - 1; count + 1];
+    minima[0] = 0;
+    let mut offsets = Vec::with_capacity(words.len());
+    offsets.push(0);
+    for w in words.iter() {
+        offsets.push(offsets.iter().last().unwrap() + w.grapheme_len());
+    }
+
+    let mut breaks = vec![0; count + 1];
+
+    let mut n = count + 1;
+    let mut i = 1;
+    let mut offset = 0;
+    loop {
+        let r = std::cmp::min(n, 2 * i);
+        let edge = i + offset;
+        smawk(
+            &mut (offset..edge).collect(),
+            &mut (edge..(r + offset)).collect(),
+            &mut minima,
+            &mut breaks,
+            width,
+            &offsets,
+        );
+        let x = minima[r - 1 + offset];
+        let mut for_was_broken = false;
+        for j in i..(r - 1) {
+            let y = cost(j + offset, r - 1 + offset, width, &minima, &offsets);
+            if y <= x {
+                n -= j;
+                i = 1;
+                offset += j;
+                for_was_broken = true;
+                break;
+            }
+        }
+
+        if !for_was_broken {
+            if r == n {
+                break;
+            }
+            i *= 2;
+        }
+    }
+    let mut lines = Vec::new();
+    let mut j = count;
+    while j > 0 {
+        let mut line = words[breaks[j]..j].join(" ");
+        lines.push(line);
+        j = breaks[j];
+    }
+    lines.reverse();
+    lines
+}
+
+fn main() -> Result<()> {
+    let text = std::fs::read_to_string(std::env::args().nth(1).unwrap())?;
+    let width = usize::from_str(&std::env::args().nth(2).unwrap()).unwrap();
+    //let paragraphs = text.split("\n\n").collect::<Vec<&str>>();
+    for _ in 0..(width - 1) {
+        print!(" ");
+    }
+    println!("|");
+    for l in linear(&text, width) {
+        println!("{}", l);
+    }
+    for _ in 0..(width - 1) {
+        print!(" ");
+    }
+    println!("|");
+    Ok(())
+}
diff --git a/tools/src/mboxparse.rs b/tools/src/mboxparse.rs
new file mode 100644
index 00000000..1abb8c56
--- /dev/null
+++ b/tools/src/mboxparse.rs
@@ -0,0 +1,70 @@
+/*
+ * meli - mboxparse.rs
+ *
+ * Copyright 2020 Manos Pitsidianakis
+ *
+ * This file is part of meli.
+ *
+ * meli is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * meli is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with meli. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+extern crate melib;
+use melib::Result;
+use melib::*;
+
+/// Parses e-mail from files and prints the debug information of the parsed `Envelope`
+///
+/// # Example invocation
+/// ```sh
+/// ./mboxparse /path/to/mbox"
+/// ```
+
+fn main() -> Result<()> {
+    if std::env::args().len() == 1 {
+        eprintln!("Usage: ./mboxparse /path/to/mbox");
+        std::process::exit(1);
+    }
+
+    for i in std::env::args().skip(1) {
+        println!("Path is {}", i);
+        let filename = std::path::PathBuf::from(&i);
+
+        if filename.exists() && filename.is_file() {
+            let buffer = std::fs::read_to_string(&filename)
+                .expect(&format!("Something went wrong reading the file {}", i,));
+            let res =
+                melib::backends::mbox::mbox_parse(Default::default(), buffer.as_bytes(), 0, None);
+            match res {
+                Ok((_, v)) => {
+                    println!("{} envelopes parsed", v.len());
+                }
+                Err(melib::nom::Err::Error(err)) => {
+                    println!(
+                        "Error in parsing {:?}",
+                        unsafe { std::str::from_utf8_unchecked(err.0) }
+                            .chars()
+                            .take(150)
+                            .collect::<String>()
+                    );
+                }
+                Err(err) => {
+                    println!("Error in parsing {:?}", err);
+                }
+            }
+        } else {
+            println!("{} is not a valid file.", i);
+        }
+    }
+    Ok(())
+}
diff --git a/tools/src/smtp_conn.rs b/tools/src/smtp_conn.rs
new file mode 100644
index 00000000..d7195372
--- /dev/null
+++ b/tools/src/smtp_conn.rs
@@ -0,0 +1,50 @@
+extern crate melib;
+
+use melib::futures;
+use melib::smol;
+use melib::smtp::*;
+use melib::Result;
+
+/// Opens an interactive shell on an IMAP server. Suggested use is with rlwrap(1)
+///
+/// # Example invocation:
+/// ```sh
+/// ./imap_conn server_hostname server_username server_password server_port");
+/// ```
+///
+/// `danger_accept_invalid_certs` is turned on by default, so no certificate validation is performed.
+
+fn main() -> Result<()> {
+    let conf = SmtpServerConf {
+        hostname: "smtp1.ntua.gr".into(),
+        port: 587,
+        security: SmtpSecurity::StartTLS {
+            danger_accept_invalid_certs: false,
+        },
+        extensions: SmtpExtensionSupport::default(),
+        auth: SmtpAuth::Auto {
+            username: "el13635".into(),
+            password: Password::CommandEval(
+                "gpg2 --no-tty -q -d ~/.passwords/msmtp/ntua.gpg".into(),
+            ),
+            require_auth: true,
+        },
+        envelope_from: String::new(),
+    };
+    for _ in 0..1 {
+        std::thread::spawn(|| smol::run(futures::future::pending::<()>()));
+    }
+
+    let mut conn = futures::executor::block_on(SmtpConnection::new_connection(conf)).unwrap();
+    futures::executor::block_on(conn.mail_transaction(
+         r##"To: pr.birch@gmail.com
+Auto-Submitted: auto-generated
+Subject: Fwd: *** SMTP TEST #2 information ***
+From: Manos <el13635@mail.ntua.gr>
+Message-Id: <E1hSjnr-0003fN-RL2@postretch>
+Date: Mon, 13 Jul 2020 15:02:15 +0300
+
+postretch : May 20 18:02:00 : epilys : user NOT in sudoers ; TTY=pts/13 ; PWD=/tmp/db-project ; USER=postgres ; COMMAND=/usr/bin/dropdb Prescriptions-R-X"##,
+    )).unwrap();
+    Ok(())
+}