summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichael Aaron Murphy <mmstickman@gmail.com>2017-01-03 20:58:56 -0500
committerMichael Aaron Murphy <mmstickman@gmail.com>2017-01-03 21:11:31 -0500
commit7fe661161a97f73e4efbb534b319a507e55ae142 (patch)
tree267894afc03963861d2e45f84994ef12e3833924
parent4dfbb81b36da89e6a29eb759faef77195cf6e61e (diff)
Implement Shebang Parameter
This will enable utilizing the parallel application as an interpreter.
-rw-r--r--README.md57
-rw-r--r--TODO.md2
-rw-r--r--src/arguments/man.rs9
-rw-r--r--src/arguments/mod.rs27
4 files changed, 67 insertions, 28 deletions
diff --git a/README.md b/README.md
index bd74c72..25041ef 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,9 @@ See the [to-do list](https://github.com/mmstick/parallel/blob/master/TODO.md) fo
## Benchmark Comparison to GNU Parallel
### GNU Parallel
+
+#### Printing 1 to 10,000 in parallel
+
```
~/D/parallel (master) $ seq 1 10000 | time -v /usr/bin/parallel echo > /dev/null
Command being timed: "/usr/bin/parallel echo"
@@ -42,6 +45,8 @@ See the [to-do list](https://github.com/mmstick/parallel/blob/master/TODO.md) fo
Exit status: 0
```
+#### Cat the contents of every binary in /usr/bin
+
```
~/D/parallel (master) $ time -v /usr/bin/parallel cat ::: /usr/bin/* > /dev/null
User time (seconds): 71.71
@@ -72,26 +77,28 @@ See the [to-do list](https://github.com/mmstick/parallel/blob/master/TODO.md) fo
It's highly recommend to compile Parallel with MUSL instead of glibc, as this reduces memory consumption in half and doubles performance.
+#### Printing 1 to 10,000 in parallel
+
```
~/D/parallel (master) $ seq 1 10000 | time -v target/release/x86_64-unknown-linux-musl/parallel echo > /dev/null
- Command being timed: "target/x86_64-unknown-linux-musl/release/parallel echo"
- User time (seconds): 0.55
- System time (seconds): 3.45
- Percent of CPU this job got: 65%
- Elapsed (wall clock) time (h:mm:ss or m:ss): 0:06.14
+Command being timed: "target/x86_64-unknown-linux-musl/release/parallel echo"
+ User time (seconds): 0.48
+ System time (seconds): 2.85
+ Percent of CPU this job got: 104%
+ Elapsed (wall clock) time (h:mm:ss or m:ss): 0:03.20
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1768
Average resident set size (kbytes): 0
- Major (requiring I/O) page faults: 1
- Minor (reclaiming a frame) page faults: 815825
- Voluntary context switches: 72504
- Involuntary context switches: 54392
+ Major (requiring I/O) page faults: 0
+ Minor (reclaiming a frame) page faults: 823754
+ Voluntary context switches: 82723
+ Involuntary context switches: 64834
Swaps: 0
- File system inputs: 8
- File system outputs: 352
+ File system inputs: 0
+ File system outputs: 320
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
@@ -99,25 +106,27 @@ It's highly recommend to compile Parallel with MUSL instead of glibc, as this re
Exit status: 0
```
+#### Cat the contents of every binary in /usr/bin
+
```
~/D/parallel (master) $ time -v target/release/x86_64-unknown-linux-musl/release/parallel cat ::: /usr/bin/* > /dev/null
- User time (seconds): 1.27
- System time (seconds): 5.07
- Percent of CPU this job got: 138%
- Elapsed (wall clock) time (h:mm:ss or m:ss): 0:04.58
+ User time (seconds): 0.96
+ System time (seconds): 4.61
+ Percent of CPU this job got: 192%
+ Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.89
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
- Maximum resident set size (kbytes): 1844
+ Maximum resident set size (kbytes): 1868
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
- Minor (reclaiming a frame) page faults: 350701
- Voluntary context switches: 67739
- Involuntary context switches: 44271
+ Minor (reclaiming a frame) page faults: 350216
+ Voluntary context switches: 68772
+ Involuntary context switches: 40085
Swaps: 0
- File system inputs: 0
- File system outputs: 480
+ File system inputs: 368
+ File system outputs: 416
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
@@ -137,8 +146,9 @@ parallel echo ::: 1 2 3 ::: A B C ::: D E F // Permutate the inputs.
parallel echo {} {1} {2} {3.} ::: 1 2 file.mkv // {N} tokens are replaced by the Nth input argument
parallel ::: "echo 1" "echo 2" "echo 3" // If no command is supplied, the input arguments become commands.
parallel 'cd {}; echo Directory: {}; echo - {}' // Commands may be chained in the platform\'s shell.
-seq 1 10 | parallel 'echo {}' // If no input arguments are supplied, stdin will be read.
-seq 1 10 | parallel --pipe cat // Piping arguments to the standard input of the given command.
+seq 1 10 | parallel 'echo {}' // If no input arguments are supplied, stdin will be read.
+seq 1 10 | parallel --pipe cat // Piping arguments to the standard input of the given command.
+#!/usr/bin/parallel --shebang echo // Ability to use within a shebang line.
```
## Manual
@@ -227,6 +237,7 @@ operates:
instead supply the arguments directly to the standard input of each child process.
- **-q**, **--quote**: Retains backslashes that are supplied as the command input.
- **-s**, **--silent**, **--quiet**: Disables printing the standard output of running processes.
+- **--shebang**: Grants ability to utilize the parallel command as an interpreter via calling it within a shebang line.
- **--shellquote**: Expands upon quote mode by escaping a wide variety of special characters.
- **--timeout**: If a command runs for longer than a specified number of seconds, it will be killed with a SIGKILL.
- **-v**, **--verbose**: Prints information about running processes.
diff --git a/TODO.md b/TODO.md
index 4724dde..a90dd12 100644
--- a/TODO.md
+++ b/TODO.md
@@ -10,8 +10,6 @@ The list is actively updated with each successful pull request.
- Compress the outputs of a file if the file exceeds a certain length
- Implement `memfree`
- Only execute a task if memory consumption is below a certain threshold.
-- Implement `skip-first-line` and `shebang`
- - Basically, enale the ability to use parallel in a shebang line
- Implement `workdir` and `tempdir`
- Allow the ability to change the default location of the temp and work directories
- Fix quoting issues
diff --git a/src/arguments/man.rs b/src/arguments/man.rs
index 936840d..930853b 100644
--- a/src/arguments/man.rs
+++ b/src/arguments/man.rs
@@ -113,6 +113,10 @@ OPTIONS
-s, --silent, --quiet:
Disables printing the standard output of running processes.
+ --shebang:
+ Grants ability to utilize the parallel command as an interpreter via
+ calling it within a shebang line.
+
--timeout:
If a command runs for longer than a specified number of seconds, it will be
killed with a SIGKILL.
@@ -142,6 +146,11 @@ EXAMPLES
# Reading inputs from files and command arguments
parallel 'echo {}' :::: list1 list2 ::: $(seq 1 10) :::: list3 list4
+ # Within a file, with each successive newline being an input argument
+ #!/usr/bin/parallel --shebang echo
+ 1
+ 2
+
HOW IT WORKS
The Parallel command consists of three phases: parsing, threading, and execution.
diff --git a/src/arguments/mod.rs b/src/arguments/mod.rs
index 11b5b18..7971be7 100644
--- a/src/arguments/mod.rs
+++ b/src/arguments/mod.rs
@@ -82,6 +82,8 @@ impl<'a> Args<'a> {
_ => (Mode::Arguments, 1)
};
+ let mut shebang = false;
+
if let Mode::Arguments = mode {
while let Some(argument) = arguments.get(index) {
index += 1;
@@ -144,6 +146,7 @@ impl<'a> Args<'a> {
"pipe" => self.flags |= PIPE_IS_ENABLED,
"quiet" | "silent" => self.flags |= QUIET_MODE,
"quote" => quote = Quoting::Basic,
+ // "shebang" => shebang = true,
"shellquote" => quote = Quoting::Shell,
"timeout" => {
let val = arguments.get(index).ok_or(ParseErr::DelayNoValue)?;
@@ -163,6 +166,11 @@ impl<'a> Args<'a> {
exit(0);
}
_ => {
+ if &argument[2..9] == "shebang" {
+ shebang = true;
+ comm.push_str(&argument[10..]);
+ break
+ }
return Err(ParseErr::InvalidArgument(index-1));
}
}
@@ -207,7 +215,12 @@ impl<'a> Args<'a> {
}
}
- parse_inputs(arguments, index, &mut current_inputs, &mut lists, &mut mode)?;
+ if shebang {
+ file_parse(&mut current_inputs, &arguments.last().unwrap())?;
+ } else {
+ parse_inputs(arguments, index, &mut current_inputs, &mut lists, &mut mode)?;
+ }
+
number_of_arguments = write_inputs_to_disk(lists, current_inputs, max_args, &mut disk_buffer)?;
}
@@ -491,8 +504,16 @@ fn parse_jobs(argument: &str, next_argument: Option<&String>, index: &mut usize)
fn file_parse<P: AsRef<Path>>(inputs: &mut Vec<String>, path: P) -> Result<(), ParseErr> {
let path = path.as_ref();
let file = fs::File::open(path).map_err(|err| ParseErr::File(FileErr::Open(path.to_owned(), err)))?;
- for line in BufReader::new(file).lines() {
- if let Ok(line) = line { inputs.push(line); }
+ let mut buffer = BufReader::new(file).lines();
+ if let Some(line) = buffer.next() {
+ if let Ok(line) = line {
+ if !line.is_empty() && !line.starts_with("#!") { inputs.push(line); }
+ }
+ }
+ for line in buffer {
+ if let Ok(line) = line {
+ if !line.is_empty() { inputs.push(line); }
+ }
}
Ok(())
}