FIPS module checksums: add scripts and Makefile rule

This adds the following scripts:

util/lang-compress.pl:

Compress source code, which language is determined by the first argument.
For the moment, we know 'perl' (perlasm source code), 'C' (C source code)
and 'S' (Assembler with C preprocessor directives).
This removes comments and empty lines, and compresses series of horizontal
spaces to one single space in the languages where that's appropriate.

util/fips-checksums.sh:

Takes source file names as arguments, pushes them through
util/lang-compress.pl and unifdef with FIPS_MODE defined, and calculates
the checksum on the result.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8871)

3 files changed, 274 insertions, 0 deletions

diff --git a/util/c-compress-test.pl b/util/c-compress-test.pl
new file mode 100755
index 0000000000..8ea3e045bc
--- /dev/null
+++ b/util/c-compress-test.pl
@@ -0,0 +1,54 @@
+#! /usr/bin/env perl
+#
+# TEST c-compress-pl with a number of examples and what should happen to them
+
+use strict;
+use warnings;
+
+use File::Basename;
+
+my @pairs =
+    (
+     [ <<'_____'
+/* A hell of a program */
+#def\
+ine foo/* bar */ 3
+#define bar /* haha "A /* comment */ that should    /* remain" */
+#define  haha /* hoho */ "A /* comment */ that should /* remain" */
+
+int main() {
+    int x;
+    /* one lonely comment */
+}
+_____
+       , <<'_____'
+#define foo 3
+#define bar that should
+#define haha "A /* comment */ that should /* remain" */
+int main() {
+int x;
+}
+_____
+     ]
+    );
+
+my $here = dirname $0;
+my $c_compress = "$here/lang-compress.pl";
+
+use FileHandle;
+use IPC::Open2;
+use Text::Diff;
+foreach (@pairs) {
+    my $source = $_->[0];
+    my $expected = $_->[1];
+    my $pid = open2(\*Reader, \*Writer, "perl $c_compress 'C'");
+    print Writer $source;
+    close Writer;
+
+    local $/ = undef;             # slurp
+    my $got = <Reader>;
+
+    if ($got ne $expected) {
+        print "MISMATCH:\n", diff \$expected, \$got;
+    }
+}
diff --git a/util/fips-checksums.sh b/util/fips-checksums.sh
new file mode 100755
index 0000000000..99f34fbc8f
--- /dev/null
+++ b/util/fips-checksums.sh
@@ -0,0 +1,31 @@
+#! /bin/sh
+
+HERE=`dirname $0`
+
+for f in "$@"; do
+    # It's worth nothing that 'openssl sha256 -r' assumes that all input
+    # is binary.  This isn't quite true, and we know better, so we convert
+    # the '*stdin' marker to the filename preceded by a space.  See the
+    # sha1sum manual for a specification of the format.
+    case "$f" in
+        *.c | *.h )
+            cat "$f" \
+                | $HERE/lang-compress.pl 'C' \
+                | unifdef -DFIPS_MODE=1 \
+                | openssl sha256 -r \
+                | sed -e "s| \\*stdin|  $f|"
+            ;;
+        *.pl ) 
+            cat "$f" \
+                | $HERE/lang-compress.pl 'perl' \
+                | openssl sha256 -r \
+                | sed -e "s| \\*stdin|  $f|"
+            ;;
+        *.S ) 
+            cat "$f" \
+                | $HERE/lang-compress.pl 'S' \
+                | openssl sha256 -r \
+                | sed -e "s| \\*stdin|  $f|"
+            ;;
+    esac
+done
diff --git a/util/lang-compress.pl b/util/lang-compress.pl
new file mode 100755
index 0000000000..6898877587
--- /dev/null
+++ b/util/lang-compress.pl
@@ -0,0 +1,189 @@
+#! /usr/bin/env perl
+#
+# C source compressor.  This:
+#
+# - merges continuation lines
+# - removes comments (not in strings)
+# - removes empty lines (not in strings)
+
+use strict;
+use warnings;
+
+my $debug = defined $ENV{DEBUG};
+my $lang = shift @ARGV;
+
+# Slurp the file
+$/ = undef;
+$_ = <>;
+
+if ($lang eq 'C') {
+    # Merge continuation lines
+    s{\\\n}{}g;
+
+    # Regexp for things that should be preserved
+    my $preserved =
+        qr{
+              (?:
+                  "                 # String start
+                  (?: \\. | [^\"])* # Any character, including escaped ones
+                  "                 # String end
+              )
+
+          |                         # OR
+
+              (?:
+                  '                 # Character start (multi-chars supported)
+                  (?: \\. | [^\'])+ # Any character, including escaped ones
+                  '                 # String end
+              )
+        }x;
+
+    # Remove comments while preserving strings
+    s{
+         (?|                        # All things preserved end up in $1
+
+             /\*                    # C comment start
+             .*?                    # Contents up until
+             \*/                    # C comment end
+
+         |                          # OR
+
+             (                      # Grouping for the replacement
+                 $preserved
+             )
+
+         )
+    }{
+        if ($debug) {
+            print STDERR "DEBUG: '$&' => '$1'\n" if defined $1;
+            print STDERR "DEBUG: '$&' removed\n" unless defined $1;
+        }
+        defined $1 ? $1 : ""
+    }gsxe;
+
+    # Remove empty lines
+    s{
+         (?|                        # All things preserved end up in $1
+
+             (^|\n)(?:\s*(?:\n|$))+ # Empty lines, preserve one newline
+
+         |                          # OR
+
+             (                      # Grouping for the replacement
+                 $preserved
+             )
+
+         )
+    }{$1}gsx;
+
+    # Remove extra spaces
+    s{
+         (?|                        # All things preserved end up in $1
+
+             (\n)\h+                # Spaces at start of lines removed
+
+         |
+
+             \h+(\n)                # Spaces at end of lines removed
+
+         |
+
+             \h+                    # Other horizontal spaces replaced with one
+
+         |                          # OR
+
+             (                      # Grouping for the replacement
+                 $preserved
+             )
+
+         )
+    }{
+        if ($debug) {
+            print STDERR "DEBUG: '$&' => '$1'\n" if defined $1;
+            print STDERR "DEBUG: '$&' => ' '\n" unless defined $1;
+        }
+        defined $1 ? $1 : " "
+    }gsxe;
+} elsif ($lang eq 'S') {
+    # Because we use C++ style comments in our .S files, all we can do
+    # is to drop them
+    s{
+         ^([^\n]*?)//[^\n]*?$   # Any line with a // comment
+    }{
+        if ($debug) {
+            print STDERR "DEBUG: '$&' => '$1'\n" if defined $1;
+            print STDERR "DEBUG: '$&' removed\n" unless defined $1;
+        }
+        defined $1 ? $1 : ""
+    }mgsxe;
+
+    # Drop all empty lines
+    s{
+         (^|\n)(?:\s*(?:\n|$))+ # Empty lines, preserve one newline
+    }{$1}gsx;
+} elsif ($lang eq 'perl') {
+    # Merge continuation lines
+    s{\\\n}{}g;
+
+    # Regexp for things that should be preserved
+    my $preserved =
+        qr{
+              (?:
+                  <<["']?(\w+)["']? # HERE document start
+                  .*?               # Its contents
+                  ^\g{-1}$
+              )
+          |
+              (?:
+                  "                 # Double quoted string start
+                  (?: \\. | [^\"])* # Any character, including escaped ones
+                  "                 # Double quoted string end
+              )
+
+          |                         # OR
+
+              (?:
+                  '                 # Single quoted string start
+                  [^\']*            # Any character
+                  '                 # Single quoted string end
+              )
+        }msx;
+
+    # Remove comments while preserving strings
+    s{
+         (?|                        # All things preserved end up in $1
+
+             \#.*?(\n|$)            # Perl comments
+
+         |                          # OR
+
+             (                      # Grouping for the replacement
+                 $preserved
+             )
+
+         )
+    }{
+        if ($debug) {
+            print STDERR "DEBUG: '$&' => '$1'\n" if defined $1;
+            print STDERR "DEBUG: '$&' removed\n" unless defined $1;
+        }
+        defined $1 ? $1 : ""
+    }gsxe;
+
+    # Remove empty lines
+    s{
+         (?|                        # All things preserved end up in $1
+
+             (^|\n)(?:\s*(?:\n|$))+ # Empty lines, preserve one newline
+
+         |                          # OR
+
+             (                      # Grouping for the replacement
+                 $preserved
+             )
+
+         )
+    }{$1}gsx;
+}
+
+print;