Ability to run integration tests without sh

Integration tests can now be run without using the 'sh' library by using
GITLINT_QA_USE_SH_LIB=0. Note that not all tests are currently passing
in this mode.

Eventually we'll entirely replace the 'sh' library with qa/shell.py, so
we can fully support integration tests on Windows.

For now, we will need to keep the 'sh' library around as certain tests
still rely on it for doing more complex interaction with std IO.

This commit also replaces the invocation of POSIX 'touch' and 'rm' commands with
python code equivalents.

Relates to #93

13 files changed, 210 insertions, 50 deletions

diff --git a/gitlint/tests/rules/test_configuration_rules.py b/gitlint/tests/rules/test_configuration_rules.py
index 11bdec0..73d42f3 100644
--- a/gitlint/tests/rules/test_configuration_rules.py
+++ b/gitlint/tests/rules/test_configuration_rules.py
@@ -16,7 +16,7 @@ class ConfigurationRuleTests(BaseTestCase):
         self.assert_logged([])  # nothing logged -> nothing ignored
 
         # Matching regex -> expect config to ignore all rules
-        rule = rules.IgnoreByTitle({"regex": "^Releäse(.*)"})
+        rule = rules.IgnoreByTitle({"regex": u"^Releäse(.*)"})
         expected_config = LintConfig()
         expected_config.ignore = "all"
         rule.apply(config, commit)
@@ -27,7 +27,7 @@ class ConfigurationRuleTests(BaseTestCase):
         self.assert_log_contains(expected_log_message)
 
         # Matching regex with specific ignore
-        rule = rules.IgnoreByTitle({"regex": "^Releäse(.*)",
+        rule = rules.IgnoreByTitle({"regex": u"^Releäse(.*)",
                                     "ignore": "T1,B2"})
         expected_config = LintConfig()
         expected_config.ignore = "T1,B2"
@@ -48,7 +48,7 @@ class ConfigurationRuleTests(BaseTestCase):
         self.assert_logged([])  # nothing logged -> nothing ignored
 
         # Matching regex -> expect config to ignore all rules
-        rule = rules.IgnoreByBody({"regex": "(.*)relëase(.*)"})
+        rule = rules.IgnoreByBody({"regex": u"(.*)relëase(.*)"})
         expected_config = LintConfig()
         expected_config.ignore = "all"
         rule.apply(config, commit)
@@ -60,7 +60,7 @@ class ConfigurationRuleTests(BaseTestCase):
         self.assert_log_contains(expected_log_message)
 
         # Matching regex with specific ignore
-        rule = rules.IgnoreByBody({"regex": "(.*)relëase(.*)",
+        rule = rules.IgnoreByBody({"regex": u"(.*)relëase(.*)",
                                    "ignore": "T1,B2"})
         expected_config = LintConfig()
         expected_config.ignore = "T1,B2"
diff --git a/qa/base.py b/qa/base.py
index f175c1a..eb59589 100644
--- a/qa/base.py
+++ b/qa/base.py
@@ -4,7 +4,7 @@
 
 import io
 import os
-import sys
+import shutil
 import tempfile
 from datetime import datetime
 from uuid import uuid4
@@ -16,26 +16,8 @@ except ImportError:
     # python 3.x
     from unittest import TestCase
 
-import sh
-from sh import git, rm, touch, RunningCommand  # pylint: disable=no-name-in-module
-
-DEFAULT_ENCODING = sh.DEFAULT_ENCODING
-
-
-def ustr(obj):
-    """ Python 2 and 3 utility method that converts an obj to unicode in python 2 and to a str object in python 3"""
-    if sys.version_info[0] == 2:
-        # If we are getting a string, then do an explicit decode
-        # else, just call the unicode method of the object
-        if type(obj) in [str, basestring]:  # pragma: no cover # noqa
-            return unicode(obj, DEFAULT_ENCODING)  # pragma: no cover # noqa
-        else:
-            return unicode(obj)  # pragma: no cover # noqa
-    else:
-        if type(obj) in [bytes]:
-            return obj.decode(DEFAULT_ENCODING)
-        else:
-            return str(obj)
+from qa.shell import git, RunningCommand
+from qa.utils import DEFAULT_ENCODING, ustr
 
 
 class BaseTestCase(TestCase):
@@ -48,6 +30,18 @@ class BaseTestCase(TestCase):
 
     GITLINT_USE_SH_LIB = os.environ.get("GITLINT_USE_SH_LIB", "[NOT SET]")
 
+    @classmethod
+    def setUpClass(cls):
+        """ Sets up the integration tests by creating a new temporary git repository """
+        cls.tmp_git_repos = []
+        cls.tmp_git_repo = cls.create_tmp_git_repo()
+
+    @classmethod
+    def tearDownClass(cls):
+        """ Cleans up the temporary git repositories """
+        for repo in cls.tmp_git_repos:
+            shutil.rmtree(repo)
+
     def setUp(self):
         self.tmpfiles = []
 
@@ -57,23 +51,11 @@ class BaseTestCase(TestCase):
 
     def assertEqualStdout(self, output, expected):  # pylint: disable=invalid-name
         self.assertIsInstance(output, RunningCommand)
-        output = output.stdout.decode(DEFAULT_ENCODING)
+        output = ustr(output.stdout)
         output = output.replace('\r', '')
         self.assertMultiLineEqual(output, expected)
 
     @classmethod
-    def setUpClass(cls):
-        """ Sets up the integration tests by creating a new temporary git repository """
-        cls.tmp_git_repos = []
-        cls.tmp_git_repo = cls.create_tmp_git_repo()
-
-    @classmethod
-    def tearDownClass(cls):
-        """ Cleans up the temporary git repositories """
-        for repo in cls.tmp_git_repos:
-            rm("-rf", repo)
-
-    @classmethod
     def generate_temp_path(cls):
         return os.path.realpath("/tmp/gitlint-test-{0}".format(datetime.now().strftime("%Y%m%d-%H%M%S-%f")))
 
@@ -113,8 +95,9 @@ class BaseTestCase(TestCase):
         if env:
             environment.update(env)
 
+        # Create file and add to git
         test_filename = u"test-fïle-" + str(uuid4())
-        touch(test_filename, _cwd=git_repo)
+        io.open(os.path.join(git_repo, test_filename), 'a', encoding=DEFAULT_ENCODING).close()
         git("add", test_filename, _cwd=git_repo)
         # https://amoffat.github.io/sh/#interactive-callbacks
         if not ok_code:
diff --git a/qa/shell.py b/qa/shell.py
new file mode 100644
index 0000000..1f7d8a9
--- /dev/null
+++ b/qa/shell.py
@@ -0,0 +1,90 @@
+
+# This code is mostly duplicated from the `gitlint.shell` module. We conciously duplicate this code as to not depend
+# on gitlint internals for our integration testing framework.
+
+import subprocess
+import sys
+from qa.utils import ustr, USE_SH_LIB
+
+if USE_SH_LIB:
+    from sh import git, echo, gitlint  # pylint: disable=unused-import,no-name-in-module
+
+    # import exceptions separately, this makes it a little easier to mock them out in the unit tests
+    from sh import CommandNotFound, ErrorReturnCode, RunningCommand
+else:
+
+    class CommandNotFound(Exception):
+        """ Exception indicating a command was not found during execution """
+        pass
+
+    class RunningCommand(object):
+        pass
+
+    class ShResult(RunningCommand):
+        """ Result wrapper class. We use this to more easily migrate from using https://amoffat.github.io/sh/ to using
+        the builtin subprocess module. """
+
+        def __init__(self, full_cmd, stdout, stderr='', exitcode=0):
+            self.full_cmd = full_cmd
+            # TODO(jorisroovers): The 'sh' library by default will merge stdout and stderr. We mimic this behavior
+            # for now until we fully remove the 'sh' library.
+            self.stdout = stdout + ustr(stderr)
+            self.stderr = stderr
+            self.exit_code = exitcode
+
+        def __str__(self):
+            return self.stdout
+
+    class ErrorReturnCode(ShResult, Exception):
+        """ ShResult subclass for unexpected results (acts as an exception). """
+        pass
+
+    def git(*command_parts, **kwargs):
+        return run_command("git", *command_parts, **kwargs)
+
+    def echo(*command_parts, **kwargs):
+        return run_command("echo", *command_parts, **kwargs)
+
+    def gitlint(*command_parts, **kwargs):
+        return run_command("gitlint", *command_parts, **kwargs)
+
+    def run_command(command, *args, **kwargs):
+        args = [command] + list(args)
+        result = _exec(*args, **kwargs)
+        # If we reach this point and the result has an exit_code that is larger than 0, this means that we didn't
+        # get an exception (which is the default sh behavior for non-zero exit codes) and so the user is expecting
+        # a non-zero exit code -> just return the entire result
+        if hasattr(result, 'exit_code') and result.exit_code > 0:
+            return result
+        return ustr(result)
+
+    def _exec(*args, **kwargs):
+        if sys.version_info[0] == 2:
+            no_command_error = OSError  # noqa pylint: disable=undefined-variable,invalid-name
+        else:
+            no_command_error = FileNotFoundError  # noqa pylint: disable=undefined-variable
+
+        pipe = subprocess.PIPE
+        popen_kwargs = {'stdout': pipe, 'stderr': pipe, 'shell': kwargs.get('_tty_out', False)}
+        if '_cwd' in kwargs:
+            popen_kwargs['cwd'] = kwargs['_cwd']
+
+        try:
+            p = subprocess.Popen(args, **popen_kwargs)
+            result = p.communicate()
+        except no_command_error:
+            raise CommandNotFound
+
+        exit_code = p.returncode
+        stdout = ustr(result[0])
+        stderr = result[1]  # 'sh' does not decode the stderr bytes to unicode
+        full_cmd = '' if args is None else ' '.join(args)
+
+        # If not _ok_code is specified, then only a 0 exit code is allowed
+        ok_exit_codes = kwargs.get('_ok_code', [0])
+
+        if exit_code in ok_exit_codes:
+            return ShResult(full_cmd, stdout, stderr, exit_code)
+
+        # Unexpected error code => raise ErrorReturnCode
+        raise ErrorReturnCode(full_cmd, stdout, stderr, p.returncode)
diff --git a/qa/test_commits.py b/qa/test_commits.py
index be6b35f..398ac49 100644
--- a/qa/test_commits.py
+++ b/qa/test_commits.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=too-many-function-args,unexpected-keyword-arg
-from sh import git, gitlint  # pylint: disable=no-name-in-module
+from qa.shell import git, gitlint
 from qa.base import BaseTestCase
 
 
diff --git a/qa/test_config.py b/qa/test_config.py
index 6e45720..270d270 100644
--- a/qa/test_config.py
+++ b/qa/test_config.py
@@ -5,7 +5,7 @@ import sys
 
 import arrow
 
-from sh import gitlint, git  # pylint: disable=no-name-in-module
+from qa.shell import git, gitlint
 from qa.base import BaseTestCase
 
 
@@ -28,6 +28,7 @@ class ConfigTests(BaseTestCase):
     def test_verbosity(self):
         self._create_simple_commit(u"WIP: Thïs is a title.\nContënt on the second line")
         output = gitlint("-v", _cwd=self.tmp_git_repo, _tty_in=True, _ok_code=[3])
+
         expected = u"1: T3\n1: T5\n2: B4\n"
         self.assertEqualStdout(output, expected)
 
diff --git a/qa/test_contrib.py b/qa/test_contrib.py
index 81b6c63..0bc4bfc 100644
--- a/qa/test_contrib.py
+++ b/qa/test_contrib.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=
-from sh import gitlint  # pylint: disable=no-name-in-module
+from qa.shell import gitlint
 from qa.base import BaseTestCase
 
 
diff --git a/qa/test_gitlint.py b/qa/test_gitlint.py
index a38e72c..37653c3 100644
--- a/qa/test_gitlint.py
+++ b/qa/test_gitlint.py
@@ -2,8 +2,9 @@
 # pylint: disable=too-many-function-args,unexpected-keyword-arg
 import io
 import os
-from sh import git, gitlint  # pylint: disable=no-name-in-module
-from qa.base import BaseTestCase, DEFAULT_ENCODING
+from qa.shell import git, gitlint
+from qa.base import BaseTestCase
+from qa.utils import DEFAULT_ENCODING
 
 
 class IntegrationTests(BaseTestCase):
diff --git a/qa/test_hooks.py b/qa/test_hooks.py
index 29080fc..3359ce9 100644
--- a/qa/test_hooks.py
+++ b/qa/test_hooks.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=too-many-function-args,unexpected-keyword-arg
 import os
-from sh import git, gitlint  # pylint: disable=no-name-in-module
+from qa.shell import git, gitlint
 from qa.base import BaseTestCase
 
 
diff --git a/qa/test_stdin.py b/qa/test_stdin.py
index 973741c..80f0e9d 100644
--- a/qa/test_stdin.py
+++ b/qa/test_stdin.py
@@ -2,8 +2,9 @@
 # pylint: disable=too-many-function-args,unexpected-keyword-arg
 import io
 import subprocess
-from sh import gitlint, echo  # pylint: disable=no-name-in-module
-from qa.base import BaseTestCase, ustr, DEFAULT_ENCODING
+from qa.shell import echo, gitlint
+from qa.base import BaseTestCase
+from qa.utils import ustr, DEFAULT_ENCODING
 
 
 class StdInTests(BaseTestCase):
diff --git a/qa/test_user_defined.py b/qa/test_user_defined.py
index 71b1721..2b8d9e9 100644
--- a/qa/test_user_defined.py
+++ b/qa/test_user_defined.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=too-many-function-args,unexpected-keyword-arg
-from sh import gitlint  # pylint: disable=no-name-in-module
+from qa.shell import gitlint
 from qa.base import BaseTestCase
 
 
diff --git a/qa/utils.py b/qa/utils.py
new file mode 100644
index 0000000..72d9650
--- /dev/null
+++ b/qa/utils.py
@@ -0,0 +1,84 @@
+# pylint: disable=bad-option-value,unidiomatic-typecheck,undefined-variable,no-else-return
+import platform
+import sys
+import os
+
+import locale
+
+########################################################################################################################
+# PLATFORM_IS_WINDOWS
+
+
+def platform_is_windows():
+    return "windows" in platform.system().lower()
+
+
+PLATFORM_IS_WINDOWS = platform_is_windows()
+
+########################################################################################################################
+# USE_SH_LIB
+# Determine whether to use the `sh` library
+# On windows we won't want to use the sh library since it's not supported - instead we'll use our own shell module.
+# However, we want to be able to overwrite this behavior for testing using the GITLINT_QA_USE_SH_LIB env var.
+
+
+def use_sh_library():
+    gitlint_use_sh_lib_env = os.environ.get('GITLINT_QA_USE_SH_LIB', None)
+    if gitlint_use_sh_lib_env:
+        return gitlint_use_sh_lib_env == "1"
+    return not PLATFORM_IS_WINDOWS
+
+
+USE_SH_LIB = use_sh_library()
+
+########################################################################################################################
+# DEFAULT_ENCODING
+
+
+def getpreferredencoding():
+    """ Modified version of local.getpreferredencoding() that takes into account LC_ALL, LC_CTYPE, LANG env vars
+        on windows and falls back to UTF-8. """
+    default_encoding = locale.getpreferredencoding() or "UTF-8"
+
+    # On Windows, we mimic git/linux by trying to read the LC_ALL, LC_CTYPE, LANG env vars manually
+    # (on Linux/MacOS the `getpreferredencoding()` call will take care of this).
+    # We fallback to UTF-8
+    if PLATFORM_IS_WINDOWS:
+        default_encoding = "UTF-8"
+        for env_var in ["LC_ALL", "LC_CTYPE", "LANG"]:
+            encoding = os.environ.get(env_var, False)
+            if encoding:
+                # Support dotted (C.UTF-8) and non-dotted (C or UTF-8) charsets:
+                # If encoding contains a dot: split and use second part, otherwise use everything
+                dot_index = encoding.find(".")
+                if dot_index != -1:
+                    default_encoding = encoding[dot_index + 1:]
+                else:
+                    default_encoding = encoding
+                break
+
+    return default_encoding
+
+
+DEFAULT_ENCODING = getpreferredencoding()
+
+########################################################################################################################
+# Unicode utility functions
+
+
+def ustr(obj):
+    """ Python 2 and 3 utility method that converts an obj to unicode in python 2 and to a str object in python 3"""
+    if sys.version_info[0] == 2:
+        # If we are getting a string, then do an explicit decode
+        # else, just call the unicode method of the object
+        if type(obj) in [str, basestring]:  # pragma: no cover # noqa
+            return unicode(obj, DEFAULT_ENCODING)  # pragma: no cover # noqa
+        else:
+            return unicode(obj)  # pragma: no cover # noqa
+    else:
+        if type(obj) in [bytes]:
+            return obj.decode(DEFAULT_ENCODING)
+        else:
+            return str(obj)
+
+########################################################################################################################
diff --git a/tools/create-test-repo.sh b/tools/create-test-repo.sh
index f9235d7..79934d6 100755
--- a/tools/create-test-repo.sh
+++ b/tools/create-test-repo.sh
@@ -18,7 +18,7 @@ cd $reponame
 git config user.name gïtlint-test-user
 git config user.email gitlint@test.com
 git config core.quotePath false
-git config core.precomposeunicde true
+git config core.precomposeUnicode true
 
 # Add a test commit
 echo "tëst 123" > test.txt
diff --git a/tools/win-create-test-repo.bat b/tools/win-create-test-repo.bat
index a8aa4df..4220ad1 100644
--- a/tools/win-create-test-repo.bat
+++ b/tools/win-create-test-repo.bat
@@ -21,7 +21,7 @@ cd %Reponame%
 git config user.name gïtlint-test-user
 git config user.email gitlint@test.com
 git config core.quotePath false
-git config core.precomposeunicde true
+git config core.precomposeUnicode true
 
 :: Add a test commit
 echo "tëst 123" > test.txt