add retry logic for running git commands to avoid index.lock problems

12 files changed, 93 insertions, 55 deletions

diff --git a/pkg/commands/branches.go b/pkg/commands/branches.go
index dd910af3b..cfc9803d6 100644
--- a/pkg/commands/branches.go
+++ b/pkg/commands/branches.go
@@ -11,19 +11,19 @@ import (
 
 // NewBranch create new branch
 func (c *GitCommand) NewBranch(name string, base string) error {
-	return c.OSCommand.RunCommand("git checkout -b %s %s", name, base)
+	return c.RunCommand("git checkout -b %s %s", name, base)
 }
 
 // CurrentBranchName get the current branch name and displayname.
 // the first returned string is the name and the second is the displayname
 // e.g. name is 123asdf and displayname is '(HEAD detached at 123asdf)'
 func (c *GitCommand) CurrentBranchName() (string, string, error) {
-	branchName, err := c.OSCommand.RunCommandWithOutput("git symbolic-ref --short HEAD")
+	branchName, err := c.RunCommandWithOutput("git symbolic-ref --short HEAD")
 	if err == nil && branchName != "HEAD\n" {
 		trimmedBranchName := strings.TrimSpace(branchName)
 		return trimmedBranchName, trimmedBranchName, nil
 	}
-	output, err := c.OSCommand.RunCommandWithOutput("git branch --contains")
+	output, err := c.RunCommandWithOutput("git branch --contains")
 	if err != nil {
 		return "", "", err
 	}
@@ -73,7 +73,7 @@ func (c *GitCommand) GetBranchGraph(branchName string) (string, error) {
 }
 
 func (c *GitCommand) GetUpstreamForBranch(branchName string) (string, error) {
-	output, err := c.OSCommand.RunCommandWithOutput("git rev-parse --abbrev-ref --symbolic-full-name %s@{u}", branchName)
+	output, err := c.RunCommandWithOutput("git rev-parse --abbrev-ref --symbolic-full-name %s@{u}", branchName)
 	return strings.TrimSpace(output), err
 }
 
@@ -86,11 +86,11 @@ func (c *GitCommand) GetBranchGraphCmdStr(branchName string) string {
 }
 
 func (c *GitCommand) SetUpstreamBranch(upstream string) error {
-	return c.OSCommand.RunCommand("git branch -u %s", upstream)
+	return c.RunCommand("git branch -u %s", upstream)
 }
 
 func (c *GitCommand) SetBranchUpstream(remoteName string, remoteBranchName string, branchName string) error {
-	return c.OSCommand.RunCommand("git branch --set-upstream-to=%s/%s %s", remoteName, remoteBranchName, branchName)
+	return c.RunCommand("git branch --set-upstream-to=%s/%s %s", remoteName, remoteBranchName, branchName)
 }
 
 func (c *GitCommand) GetCurrentBranchUpstreamDifferenceCount() (string, string) {
@@ -134,24 +134,24 @@ func (c *GitCommand) Merge(branchName string, opts MergeOpts) error {
 
 // AbortMerge abort merge
 func (c *GitCommand) AbortMerge() error {
-	return c.OSCommand.RunCommand("git merge --abort")
+	return c.RunCommand("git merge --abort")
 }
 
 func (c *GitCommand) IsHeadDetached() bool {
-	err := c.OSCommand.RunCommand("git symbolic-ref -q HEAD")
+	err := c.RunCommand("git symbolic-ref -q HEAD")
 	return err != nil
 }
 
 // ResetHardHead runs `git reset --hard`
 func (c *GitCommand) ResetHard(ref string) error {
-	return c.OSCommand.RunCommand("git reset --hard " + ref)
+	return c.RunCommand("git reset --hard " + ref)
 }
 
 // ResetSoft runs `git reset --soft HEAD`
 func (c *GitCommand) ResetSoft(ref string) error {
-	return c.OSCommand.RunCommand("git reset --soft " + ref)
+	return c.RunCommand("git reset --soft " + ref)
 }
 
 func (c *GitCommand) RenameBranch(oldName string, newName string) error {
-	return c.OSCommand.RunCommand("git branch --move %s %s", oldName, newName)
+	return c.RunCommand("git branch --move %s %s", oldName, newName)
 }
diff --git a/pkg/commands/commits.go b/pkg/commands/commits.go
index 27005ebec..4742ce448 100644
--- a/pkg/commands/commits.go
+++ b/pkg/commands/commits.go
@@ -11,7 +11,7 @@ import (
 
 // RenameCommit renames the topmost commit with the given name
 func (c *GitCommand) RenameCommit(name string) error {
-	return c.OSCommand.RunCommand("git commit --allow-empty --amend -m %s", c.OSCommand.Quote(name))
+	return c.RunCommand("git commit --allow-empty --amend -m %s", c.OSCommand.Quote(name))
 }
 
 // ResetToCommit reset to commit
@@ -74,7 +74,7 @@ func (c *GitCommand) ShowCmdStr(sha string, filterPath string) string {
 
 // Revert reverts the selected commit by sha
 func (c *GitCommand) Revert(sha string) error {
-	return c.OSCommand.RunCommand("git revert %s", sha)
+	return c.RunCommand("git revert %s", sha)
 }
 
 // CherryPickCommits begins an interactive rebase with the given shas being cherry picked onto HEAD
@@ -94,5 +94,5 @@ func (c *GitCommand) CherryPickCommits(commits []*models.Commit) error {
 
 // CreateFixupCommit creates a commit that fixes up a previous commit
 func (c *GitCommand) CreateFixupCommit(sha string) error {
-	return c.OSCommand.RunCommand("git commit --fixup=%s", sha)
+	return c.RunCommand("git commit --fixup=%s", sha)
 }
diff --git a/pkg/commands/config.go b/pkg/commands/config.go
index f22d72ffa..0bb310e6d 100644
--- a/pkg/commands/config.go
+++ b/pkg/commands/config.go
@@ -15,7 +15,7 @@ func (c *GitCommand) ConfiguredPager() string {
 	if os.Getenv("PAGER") != "" {
 		return os.Getenv("PAGER")
 	}
-	output, err := c.OSCommand.RunCommandWithOutput("git config --get-all core.pager")
+	output, err := c.RunCommandWithOutput("git config --get-all core.pager")
 	if err != nil {
 		return ""
 	}
diff --git a/pkg/commands/files.go b/pkg/commands/files.go
index 0f82d1186..fcf3ea51e 100644
--- a/pkg/commands/files.go
+++ b/pkg/commands/files.go
@@ -21,17 +21,17 @@ func (c *GitCommand) CatFile(fileName string) (string, error) {
 
 // StageFile stages a file
 func (c *GitCommand) StageFile(fileName string) error {
-	return c.OSCommand.RunCommand("git add -- %s", c.OSCommand.Quote(fileName))
+	return c.RunCommand("git add -- %s", c.OSCommand.Quote(fileName))
 }
 
 // StageAll stages all files
 func (c *GitCommand) StageAll() error {
-	return c.OSCommand.RunCommand("git add -A")
+	return c.RunCommand("git add -A")
 }
 
 // UnstageAll unstages all files
 func (c *GitCommand) UnstageAll() error {
-	return c.OSCommand.RunCommand("git reset")
+	return c.RunCommand("git reset")
 }
 
 // UnStageFile unstages a file
@@ -108,22 +108,22 @@ func (c *GitCommand) DiscardAllFileChanges(file *models.File) error {
 	quotedFileName := c.OSCommand.Quote(file.Name)
 
 	if file.ShortStatus == "AA" {
-		if err := c.OSCommand.RunCommand("git checkout --ours --  %s", quotedFileName); err != nil {
+		if err := c.RunCommand("git checkout --ours --  %s", quotedFileName); err != nil {
 			return err
 		}
-		if err := c.OSCommand.RunCommand("git add %s", quotedFileName); err != nil {
+		if err := c.RunCommand("git add %s", quotedFileName); err != nil {
 			return err
 		}
 		return nil
 	}
 
 	if file.ShortStatus == "DU" {
-		return c.OSCommand.RunCommand("git rm %s", quotedFileName)
+		return c.RunCommand("git rm %s", quotedFileName)
 	}
 
 	// if the file isn't tracked, we assume you want to delete it
 	if file.HasStagedChanges || file.HasMergeConflicts {
-		if err := c.OSCommand.RunCommand("git reset -- %s", quotedFileName); err != nil {
+		if err := c.RunCommand("git reset -- %s", quotedFileName); err != nil {
 			return err
 		}
 	}
@@ -149,7 +149,7 @@ func (c *GitCommand) DiscardUnstagedDirChanges(node *filetree.FileNode) error {
 	}
 
 	quotedPath := c.OSCommand.Quote(node.GetPath())
-	if err := c.OSCommand.RunCommand("git checkout -- %s", quotedPath); err != nil {
+	if err := c.RunCommand("git checkout -- %s", quotedPath); err != nil {
 		return err
 	}
 
@@ -174,7 +174,7 @@ func (c *GitCommand) RemoveUntrackedDirFiles(node *filetree.FileNode) error {
 // DiscardUnstagedFileChanges directly
 func (c *GitCommand) DiscardUnstagedFileChanges(file *models.File) error {
 	quotedFileName := c.OSCommand.Quote(file.Name)
-	return c.OSCommand.RunCommand("git checkout -- %s", quotedFileName)
+	return c.RunCommand("git checkout -- %s", quotedFileName)
 }
 
 // Ignore adds a file to the gitignore for the repo
@@ -219,7 +219,7 @@ func (c *GitCommand) ApplyPatch(patch string, flags ...string) error {
 		flagStr += " --" + flag
 	}
 
-	return c.OSCommand.RunCommand("git apply %s %s", flagStr, c.OSCommand.Quote(filepath))
+	return c.RunCommand("git apply %s %s", flagStr, c.OSCommand.Quote(filepath))
 }
 
 // ShowFileDiff get the diff of specified from and to. Typically this will be used for a single commit so it'll be 123abc^..123abc
@@ -245,7 +245,7 @@ func (c *GitCommand) ShowFileDiffCmdStr(from string, to string, reverse bool, fi
 
 // CheckoutFile checks out the file for the given commit
 func (c *GitCommand) CheckoutFile(commitSha, fileName string) error {
-	return c.OSCommand.RunCommand("git checkout %s %s", commitSha, fileName)
+	return c.RunCommand("git checkout %s %s", commitSha, fileName)
 }
 
 // DiscardOldFileChanges discards changes to a file from an old commit
@@ -255,7 +255,7 @@ func (c *GitCommand) DiscardOldFileChanges(commits []*models.Commit, commitIndex
 	}
 
 	// check if file exists in previous commit (this command returns an error if the file doesn't exist)
-	if err := c.OSCommand.RunCommand("git cat-file -e HEAD^:%s", fileName); err != nil {
+	if err := c.RunCommand("git cat-file -e HEAD^:%s", fileName); err != nil {
 		if err := c.OSCommand.Remove(fileName); err != nil {
 			return err
 		}
@@ -281,17 +281,17 @@ func (c *GitCommand) DiscardOldFileChanges(commits []*models.Commit, commitIndex
 
 // DiscardAnyUnstagedFileChanges discards any unstages file changes via `git checkout -- .`
 func (c *GitCommand) DiscardAnyUnstagedFileChanges() error {
-	return c.OSCommand.RunCommand("git checkout -- .")
+	return c.RunCommand("git checkout -- .")
 }
 
 // RemoveTrackedFiles will delete the given file(s) even if they are currently tracked
 func (c *GitCommand) RemoveTrackedFiles(name string) error {
-	return c.OSCommand.RunCommand("git rm -r --cached %s", name)
+	return c.RunCommand("git rm -r --cached %s", name)
 }
 
 // RemoveUntrackedFiles runs `git clean -fd`
 func (c *GitCommand) RemoveUntrackedFiles() error {
-	return c.OSCommand.RunCommand("git clean -fd")
+	return c.RunCommand("git clean -fd")
 }
 
 // ResetAndClean removes all unstaged changes and removes all untracked files
diff --git a/pkg/commands/git.go b/pkg/commands/git.go
index 0db74254e..ebf6268b4 100644
--- a/pkg/commands/git.go
+++ b/pkg/commands/git.go
@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/go-errors/errors"
 
@@ -197,3 +198,40 @@ func findDotGitDir(stat func(string) (os.FileInfo, error), readFile func(filenam
 func VerifyInGitRepo(osCommand *oscommands.OSCommand) error {
 	return osCommand.RunCommand("git rev-parse --git-dir")
 }
+
+func (c *GitCommand) RunCommand(formatString string, formatArgs ...interface{}) error {
+	_, err := c.RunCommandWithOutput(formatString, formatArgs...)
+	return err
+}
+
+func (c *GitCommand) RunCommandWithOutput(formatString string, formatArgs ...interface{}) (string, error) {
+	// TODO: have this retry logic in other places we run the command
+	waitTime := 50 * time.Millisecond
+	retryCount := 5
+	attempt := 0
+
+	for {
+		output, err := c.OSCommand.RunCommandWithOutput(formatString, formatArgs...)
+		if err != nil {
+			// if we have an error based on the index lock, we should wait a bit and then retry
+			if strings.Contains(output, ".git/index.lock") {
+				c.Log.Error(output)
+				c.Log.Info("index.lock prevented command from running. Retrying command after a small wait")
+				attempt++
+				time.Sleep(waitTime)
+				if attempt < retryCount {
+					continue
+				} else if attempt == retryCount {
+					// delete the lock file because some other process must have died leaving it there
+					// TODO: ensure this is the actual location of the lock
+					c.Log.Warn("WARNING: removing index.lock file after retrying command 5 times")
+					if err := os.Remove(".git/index.lock"); err != nil {
+						return "", err
+					}
+					continue
+				}
+			}
+		}
+		return output, err
+	}
+}
diff --git a/pkg/commands/loading_commit_files.go b/pkg/commands/loading_commit_files.go
index d0f8406b8..bddddc761 100644
--- a/pkg/commands/loading_commit_files.go
+++ b/pkg/commands/loading_commit_files.go
@@ -13,7 +13,7 @@ func (c *GitCommand) GetFilesInDiff(from string, to string, reverse bool) ([]*mo
 		reverseFlag = " -R "
 	}
 
-	filenames, err := c.OSCommand.RunCommandWithOutput("git diff --submodule --no-ext-diff --name-status -z --no-renames %s %s %s", reverseFlag, from, to)
+	filenames, err := c.RunCommandWithOutput("git diff --submodule --no-ext-diff --name-status -z --no-renames %s %s %s", reverseFlag, from, to)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/commands/loading_files.go b/pkg/commands/loading_files.go
index d3757b179..f85329f2f 100644
--- a/pkg/commands/loading_files.go
+++ b/pkg/commands/loading_files.go
@@ -83,7 +83,7 @@ func (c *GitCommand) GitStatus(opts GitStatusOptions) (string, error) {
 		noRenamesFlag = "--no-renames"
 	}
 
-	statusLines, err := c.OSCommand.RunCommandWithOutput("git status %s --porcelain -z %s", opts.UntrackedFilesArg, noRenamesFlag)
+	statusLines, err := c.RunCommandWithOutput("git status %s --porcelain -z %s", opts.UntrackedFilesArg, noRenamesFlag)
 	if err != nil {
 		return "", err
 	}
diff --git a/pkg/commands/loading_stash.go b/pkg/commands/loading_stash.go
index 8d0b4a817..e14b463dd 100644
--- a/pkg/commands/loading_stash.go
+++ b/pkg/commands/loading_stash.go
@@ -25,7 +25,7 @@ func (c *GitCommand) GetStashEntries(filterPath string) []*models.StashEntry {
 		return c.getUnfilteredStashEntries()
 	}
 
-	rawString, err := c.OSCommand.RunCommandWithOutput("git stash list --name-only")
+	rawString, err := c.RunCommandWithOutput("git stash list --name-only")
 	if err != nil {
 		return c.getUnfilteredStashEntries()
 	}
diff --git a/pkg/commands/remotes.go b/pkg/commands/remotes.go
index 779ce9e2b..535149c1a 100644
--- a/pkg/commands/remotes.go
+++ b/pkg/commands/remotes.go
@@ -7,19 +7,19 @@ import (
 )
 
 func (c *GitCommand) AddRemote(name string, url string) error {
-	return c.OSCommand.RunCommand("git remote add %s %s", name, url)
+	return c.RunCommand("git remote add %s %s", name, url)
 }
 
 func (c *GitCommand) RemoveRemote(name string) error {
-	return c.OSCommand.RunCommand("git remote remove %s", name)
+	return c.RunCommand("git remote remove %s", name)
 }
 
 func (c *GitCommand) RenameRemote(oldRemoteName string, newRemoteName string) error {
-	return c.OSCommand.RunCommand("git remote rename %s %s", oldRemoteName, newRemoteName)
+	return c.RunCommand("git remote rename %s %s", oldRemoteName, newRemoteName)
 }
 
 func (c *GitCommand) UpdateRemoteUrl(remoteName string, updatedUrl string) error {
-	return c.OSCommand.RunCommand("git remote set-url %s %s", remoteName, updatedUrl)
+	return c.RunCommand("git remote set-url %s %s", remoteName, updatedUrl)
 }
 
 func (c *GitCommand) DeleteRemoteBranch(remoteName string, branchName string, promptUserForCredential func(string) string) error {
diff --git a/pkg/commands/stash_entries.go b/pkg/commands/stash_entries.go
index 4fad9ffab..48944c626 100644
--- a/pkg/commands/stash_entries.go
+++ b/pkg/commands/stash_entries.go
@@ -4,13 +4,13 @@ import "fmt"
 
 // StashDo modify stash
 func (c *GitCommand) StashDo(index int, method string) error {
-	return c.OSCommand.RunCommand("git stash %s stash@{%d}", method, index)
+	return c.RunCommand("git stash %s stash@{%d}", method, index)
 }
 
 // StashSave save stash
 // TODO: before calling this, check if there is anything to save
 func (c *GitCommand) StashSave(message string) error {
-	return c.OSCommand.RunCommand("git stash save %s", c.OSCommand.Quote(message))
+	return c.RunCommand("git stash save %s", c.OSCommand.Quote(message))
 }
 
 // GetStashEntryDiff stash diff
@@ -22,7 +22,7 @@ func (c *GitCommand) ShowStashEntryCmdStr(index int) string {
 // shoutouts to Joe on https://stackoverflow.com/questions/14759748/stashing-only-staged-changes-in-git-is-it-possible
 func (c *GitCommand) StashSaveStagedChanges(message string) error {
 
-	if err := c.OSCommand.RunCommand("git stash --keep-index"); err != nil {
+	if err := c.RunCommand("git stash --keep-index"); err != nil {
 		return err
 	}
 
@@ -30,7 +30,7 @@ func (c *GitCommand) StashSaveStagedChanges(message string) error {
 		return err
 	}
 
-	if err := c.OSCommand.RunCommand("git stash apply stash@{1}"); err != nil {
+	if err := c.RunCommand("git stash apply stash@{1}"); err != nil {
 		return err
 	}
 
@@ -38,7 +38,7 @@ func (c *GitCommand) StashSaveStagedChanges(message string) error {
 		return err
 	}
 
-	if err := c.OSCommand.RunCommand("git stash drop stash@{1}"); err != nil {
+	if err := c.RunCommand("git stash drop stash@{1}"); err != nil {
 		return err
 	}
 
diff --git a/pkg/commands/submodules.go b/pkg/commands/submodules.go
index 101c6178c..856bc4cbe 100644
--- a/pkg/commands/submodules.go
+++ b/pkg/commands/submodules.go
@@ -69,28 +69,28 @@ func (c *GitCommand) SubmoduleStash(submodule *models.SubmoduleConfig) error {
 		return nil
 	}
 
-	return c.OSCommand.RunCommand("git -C %s stash --include-untracked", submodule.Path)
+	return c.RunCommand("git -C %s stash --include-untracked", submodule.Path)
 }
 
 func (c *GitCommand) SubmoduleReset(submodule *models.SubmoduleConfig) error {
-	return c.OSCommand.RunCommand("git submodule update --init --force %s", submodule.Path)
+	return c.RunCommand("git submodule update --init --force %s", submodule.Path)
 }
 
 func (c *GitCommand) SubmoduleUpdateAll() error {
 	// not doing an --init here because the user probably doesn't want that
-	return c.OSCommand.RunCommand("git submodule update --force")
+	return c.RunCommand("git submodule update --force")
 }
 
 func (c *GitCommand) SubmoduleDelete(submodule *models.SubmoduleConfig) error {
 	// based on https://gist.github.com/myusuf3/7f645819ded92bda6677
 
-	if err := c.OSCommand.RunCommand("git submodule deinit --force %s", submodule.Path); err != nil {
+	if err := c.RunCommand("git submodule deinit --force %s", submodule.Path); err != nil {
 		if strings.Contains(err.Error(), "did not match any file(s) known to git") {
-			if err := c.OSCommand.RunCommand("git config --file .gitmodules --remove-section submodule.%s", submodule.Name); err != nil {
+			if err := c.RunCommand("git config --file .gitmodules --remove-section submodule.%s", submodule.Name); err != nil {
 				return err
 			}
 
-			if err := c.OSCommand.RunCommand("git config --remove-section submodule.%s", submodule.Name); err != nil {
+			if err := c.RunCommand("git config --remove-section submodule.%s", submodule.Name); err != nil {
 				return err
 			}
 
@@ -100,7 +100,7 @@ func (c *GitCommand) SubmoduleDelete(submodule *models.SubmoduleConfig) error {
 		}
 	}
 
-	if err := c.OSCommand.RunCommand("git rm --force -r %s", submodule.Path); err != nil {
+	if err := c.RunCommand("git rm --force -r %s", submodule.Path); err != nil {
 		// if the directory isn't there then that's fine
 		c.Log.Error(err)
 	}
@@ -119,11 +119,11 @@ func (c *GitCommand) SubmoduleAdd(name string, path string, url string) error {
 
 func (c *GitCommand) SubmoduleUpdateUrl(name string, path string, newUrl string) error {
 	// the set-url command is only for later git versions so we're doing it manually here
-	if err := c.OSCommand.RunCommand("git config --file .gitmodules submodule.%s.url %s", name, newUrl); err != nil {
+	if err := c.RunCommand("git config --file .gitmodules submodule.%s.url %s", name, newUrl); err != nil {
 		return err
 	}
 
-	if err := c.OSCommand.RunCommand("git submodule sync %s", path); err != nil {
+	if err := c.RunCommand("git submodule sync %s", path); err != nil {
 		return err
 	}
 
@@ -131,11 +131,11 @@ func (c *GitCommand) SubmoduleUpdateUrl(name string, path string, newUrl string)
 }
 
 func (c *GitCommand) SubmoduleInit(path string) error {
-	return c.OSCommand.RunCommand("git submodule init %s", path)
+	return c.RunCommand("git submodule init %s", path)
 }
 
 func (c *GitCommand) SubmoduleUpdate(path string) error {
-	return c.OSCommand.RunCommand("git submodule update --init %s", path)
+	return c.RunCommand("git submodule update --init %s", path)
 }
 
 func (c *GitCommand) SubmoduleBulkInitCmdStr() string {
diff --git a/pkg/commands/tags.go b/pkg/commands/tags.go
index 121637311..18c09194c 100644
--- a/pkg/commands/tags.go
+++ b/pkg/commands/tags.go
@@ -3,11 +3,11 @@ package commands
 import "fmt"
 
 func (c *GitCommand) CreateLightweightTag(tagName string, commitSha string) error {
-	return c.OSCommand.RunCommand("git tag %s %s", tagName, commitSha)
+	return c.RunCommand("git tag %s %s", tagName, commitSha)
 }
 
 func (c *GitCommand) DeleteTag(tagName string) error {
-	return c.OSCommand.RunCommand("git tag -d %s", tagName)
+	return c.RunCommand("git tag -d %s", tagName)
 }
 
 func (c *GitCommand) PushTag(remoteName string, tagName string, promptUserForCredential func(string) string) error {