patch 8.1.0360: using an external diff program is slow and inflexiblev8.1.0360

Problem:    Using an external diff program is slow and inflexible.
Solution:   Include the xdiff library. (Christian Brabandt, closes #2732)
            Use it by default.

42 files changed, 5376 insertions, 209 deletions

diff --git a/Filelist b/Filelist
index f488839740..343a2bd5c4 100644
--- a/Filelist
+++ b/Filelist
@@ -273,6 +273,20 @@ SRC_ALL =	\
 		src/libvterm/t/92lp1640917.test \
 		src/libvterm/t/harness.c \
 		src/libvterm/t/run-test.pl \
+		src/xdiff/xdiff.h \
+		src/xdiff/xdiffi.c \
+		src/xdiff/xdiffi.h \
+		src/xdiff/xemit.c \
+		src/xdiff/xemit.h \
+		src/xdiff/xhistogram.c \
+		src/xdiff/xinclude.h \
+		src/xdiff/xmacros.h \
+		src/xdiff/xpatience.c \
+		src/xdiff/xprepare.c \
+		src/xdiff/xprepare.h \
+		src/xdiff/xtypes.h \
+		src/xdiff/xutils.c \
+		src/xdiff/xutils.h \
 
 
 # source files for Unix only
diff --git a/runtime/doc/diff.txt b/runtime/doc/diff.txt
index d56aea72ad..6670f45c64 100644
--- a/runtime/doc/diff.txt
+++ b/runtime/doc/diff.txt
@@ -39,7 +39,9 @@ The second and following arguments may also be a directory name.  Vim will
 then append the file name of the first argument to the directory name to find
 the file.
 
-This only works when a standard "diff" command is available.  See 'diffexpr'.
+By default an internal diff library will be used.  When 'diffopt' or
+'diffexpr' has been set an external "diff" command will be used.  This only
+works when such a diff program is available.
 
 Diffs are local to the current tab page |tab-page|.  You can't see diffs with
 a window in another tab page.  This does make it possible to have several
@@ -344,8 +346,9 @@ between file1 and file2: >
 
 The ">" is replaced with the value of 'shellredir'.
 
-The output of "diff" must be a normal "ed" style diff.  Do NOT use a context
-diff.  This example explains the format that Vim expects: >
+The output of "diff" must be a normal "ed" style diff or a unified diff.  Do
+NOT use a context diff.  This example explains the format that Vim expects for
+the "ed" style diff: >
 
 	1a2
 	> bbb
diff --git a/runtime/doc/options.txt b/runtime/doc/options.txt
index fca2e4662e..16aa422151 100644
--- a/runtime/doc/options.txt
+++ b/runtime/doc/options.txt
@@ -2609,8 +2609,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 			{not in Vi}
 			{not available when compiled without the |+diff|
 			feature}
-	Expression which is evaluated to obtain an ed-style diff file from two
-	versions of a file.  See |diff-diffexpr|.
+	Expression which is evaluated to obtain a diff file (either ed-style
+	or unified-style) from two versions of a file.  See |diff-diffexpr|.
 	This option cannot be set from a |modeline| or in the |sandbox|, for
 	security reasons.
 
@@ -2657,11 +2657,31 @@ A jump table for the options with a short description can be found at |Q_op|.
 		foldcolumn:{n}	Set the 'foldcolumn' option to {n} when
 				starting diff mode.  Without this 2 is used.
 
-	Examples: >
+		internal	Use the internal diff library.  This is
+				ignored when 'diffexpr' is set.  *E960*
+				When running out of memory when writing a
+				buffer this item will be ignored for diffs
+				involving that buffer.  Set the 'verbose'
+				option to see when this happens.
+
+		indent-heuristic
+                                 Use the indent heuristic for the internal
+                                 diff library.
+
+                algorithm:{text} Use the specified diff algorithm with the
+				internal diff engine. Currently supported 
+				algorithms are:
+				myers      the default algorithm
+				minimal    spend extra time to generate the
+					   smallest possible diff
+				patience   patience diff algorithm
+				histogram  histogram diff algorithm
 
-		:set diffopt=filler,context:4
+	Examples: >
+		:set diffopt=internal,filler,context:4
 		:set diffopt=
-		:set diffopt=filler,foldcolumn:3
+		:set diffopt=internal,filler,foldcolumn:3
+		:set diffopt-=internal  " do NOT use the internal diff parser
 <
 				     *'digraph'* *'dg'* *'nodigraph'* *'nodg'*
 'digraph' 'dg'		boolean	(default off)
diff --git a/src/Make_cyg_ming.mak b/src/Make_cyg_ming.mak
index 16cd71259c..2308135b52 100644
--- a/src/Make_cyg_ming.mak
+++ b/src/Make_cyg_ming.mak
@@ -166,6 +166,25 @@ else
 ifndef CROSS_COMPILE
 CROSS_COMPILE =
 endif
+
+# About the "sh.exe" condition, as explained by Ken Takata:
+#
+# If the makefile is executed with mingw32-make and sh.exe is not found in
+# $PATH, then $SHELL is set to "sh.exe" (without any path). In this case,
+# unix-like commands might not work and a dos-style path is needed.
+# 
+# If the makefile is executed with mingw32-make and sh.exe IS found in $PATH,
+# then $SHELL is set with the actual path of sh.exe (e.g.
+# "C:/msys64/usr/bin/sh.exe").  In this case, unix-like commands can be used.
+# 
+# If it is executed by the "make" command from cmd.exe, $SHELL is set to
+# "/bin/sh". If the "make" command is in the $PATH, other unix-like commands
+# might also work.
+# 
+# If it is executed by the "make" command from a unix-like shell,
+# $SHELL is set with the unix-style path (e.g. "/bin/bash").
+# In this case, unix-like commands can be used.
+#
 ifneq (sh.exe, $(SHELL))
 DEL = rm
 MKDIR = mkdir -p
@@ -810,6 +829,23 @@ OBJ += $(OUTDIR)/terminal.o \
 	$(OUTDIR)/term_vterm.o
 endif
 
+# Include xdiff
+OBJ +=  $(OUTDIR)/xdiffi.o \
+	$(OUTDIR)/xemit.o \
+	$(OUTDIR)/xprepare.o \
+	$(OUTDIR)/xutils.o \
+	$(OUTDIR)/xhistogram.o \
+	$(OUTDIR)/xpatience.o
+
+XDIFF_DEPS = \
+	xdiff/xdiff.h \
+	xdiff/xdiffi.h \
+	xdiff/xemit.h \
+	xdiff/xinclude.h \
+	xdiff/xmacros.h \
+	xdiff/xprepare.h \
+	xdiff/xtypes.h \
+	xdiff/xutils.h
 
 ifdef MZSCHEME
 MZSCHEME_SUFFIX = Z
@@ -1055,6 +1091,23 @@ $(OUTDIR)/term_unicode.o: libvterm/src/unicode.c $(TERM_DEPS)
 $(OUTDIR)/term_vterm.o: libvterm/src/vterm.c $(TERM_DEPS)
 	$(CCCTERM) libvterm/src/vterm.c -o $@
 
+$(OUTDIR)/xdiffi.o: xdiff/xdiffi.c $(XDIFF_DEPS)
+	$(CC) -c $(CFLAGS) xdiff/xdiffi.c -o $(OUTDIR)/xdiffi.o
+
+$(OUTDIR)/xemit.o: xdiff/xemit.c $(XDIFF_DEPS)
+	$(CC) -c $(CFLAGS) xdiff/xemit.c -o $(OUTDIR)/xemit.o
+
+$(OUTDIR)/xprepare.o: xdiff/xprepare.c $(XDIFF_DEPS)
+	$(CC) -c $(CFLAGS) xdiff/xprepare.c -o $(OUTDIR)/xprepare.o
+
+$(OUTDIR)/xutils.o: xdiff/xutils.c $(XDIFF_DEPS)
+	$(CC) -c $(CFLAGS) xdiff/xutils.c -o $(OUTDIR)/xutils.o
+
+$(OUTDIR)/xhistogram.o: xdiff/xhistogram.c $(XDIFF_DEPS)
+	$(CC) -c $(CFLAGS) xdiff/xhistogram.c -o $(OUTDIR)/xhistogram.o
+
+$(OUTDIR)/xpatience.o: xdiff/xpatience.c $(XDIFF_DEPS)
+	$(CC) -c $(CFLAGS) xdiff/xpatience.c -o $(OUTDIR)/xpatience.o
 
 pathdef.c: $(INCL)
 ifneq (sh.exe, $(SHELL))
diff --git a/src/Make_mvc.mak b/src/Make_mvc.mak
index f32903392e..1131f2b05f 100644
--- a/src/Make_mvc.mak
+++ b/src/Make_mvc.mak
@@ -813,6 +813,24 @@ CUI_OBJ = $(OUTDIR)\iscygpty.obj
 !endif
 SUBSYSTEM_TOOLS = console
 
+XDIFF_OBJ = $(OBJDIR)/xdiffi.obj \
+	$(OBJDIR)/xemit.obj \
+	$(OBJDIR)/xprepare.obj \
+	$(OBJDIR)/xutils.obj \
+	$(OBJDIR)/xhistogram.obj \
+	$(OBJDIR)/xpatience.obj
+
+XDIFF_DEPS = \
+	xdiff/xdiff.h \
+	xdiff/xdiffi.h \
+	xdiff/xemit.h \
+	xdiff/xinclude.h \
+	xdiff/xmacros.h \
+	xdiff/xprepare.h \
+	xdiff/xtypes.h \
+	xdiff/xutils.h
+
+
 !if "$(SUBSYSTEM_VER)" != ""
 SUBSYSTEM = $(SUBSYSTEM),$(SUBSYSTEM_VER)
 SUBSYSTEM_TOOLS = $(SUBSYSTEM_TOOLS),$(SUBSYSTEM_VER)
@@ -1204,12 +1222,12 @@ all:	$(VIM).exe \
 	tee/tee.exe \
 	GvimExt/gvimext.dll
 
-$(VIM).exe: $(OUTDIR) $(OBJ) $(GUI_OBJ) $(CUI_OBJ) $(OLE_OBJ) $(OLE_IDL) $(MZSCHEME_OBJ) \
+$(VIM).exe: $(OUTDIR) $(OBJ) $(XDIFF_OBJ) $(GUI_OBJ) $(CUI_OBJ) $(OLE_OBJ) $(OLE_IDL) $(MZSCHEME_OBJ) \
 		$(LUA_OBJ) $(PERL_OBJ) $(PYTHON_OBJ) $(PYTHON3_OBJ) $(RUBY_OBJ) $(TCL_OBJ) \
 		$(CSCOPE_OBJ) $(TERM_OBJ) $(NETBEANS_OBJ) $(CHANNEL_OBJ) $(XPM_OBJ) \
 		version.c version.h
 	$(CC) $(CFLAGS_OUTDIR) version.c
-	$(link) $(LINKARGS1) -out:$(VIM).exe $(OBJ) $(GUI_OBJ) $(CUI_OBJ) $(OLE_OBJ) \
+	$(link) $(LINKARGS1) -out:$(VIM).exe $(OBJ) $(XDIFF_OBJ) $(GUI_OBJ) $(CUI_OBJ) $(OLE_OBJ) \
 		$(LUA_OBJ) $(MZSCHEME_OBJ) $(PERL_OBJ) $(PYTHON_OBJ) $(PYTHON3_OBJ) $(RUBY_OBJ) \
 		$(TCL_OBJ) $(CSCOPE_OBJ) $(TERM_OBJ) $(NETBEANS_OBJ) $(CHANNEL_OBJ) \
 		$(XPM_OBJ) $(OUTDIR)\version.obj $(LINKARGS2)
@@ -1304,6 +1322,7 @@ $(NEW_TESTS):
 	$(MAKE) /NOLOGO -f Make_dos.mak nolog
 	$(MAKE) /NOLOGO -f Make_dos.mak $@.res
 	$(MAKE) /NOLOGO -f Make_dos.mak report
+	cat messages
 	cd ..
 
 ###########################################################################
@@ -1344,6 +1363,24 @@ $(OUTDIR)/dict.obj:	$(OUTDIR) dict.c  $(INCL)
 
 $(OUTDIR)/diff.obj:	$(OUTDIR) diff.c  $(INCL)
 
+$(OUTDIR)/xdiffi.obj:	$(OUTDIR) xdiff/xdiffi.c  $(XDIFF_DEPS)
+	$(CC) $(CFLAGS_OUTDIR) xdiff/xdiffi.c 
+
+$(OUTDIR)/xemit.obj:	$(OUTDIR) xdiff/xemit.c  $(XDIFF_DEPS)
+	$(CC) $(CFLAGS_OUTDIR) xdiff/xemit.c 
+
+$(OUTDIR)/xprepare.obj:	$(OUTDIR) xdiff/xprepare.c  $(XDIFF_DEPS)
+	$(CC) $(CFLAGS_OUTDIR) xdiff/xprepare.c 
+
+$(OUTDIR)/xutils.obj:	$(OUTDIR) xdiff/xutils.c  $(XDIFF_DEPS)
+	$(CC) $(CFLAGS_OUTDIR) xdiff/xutils.c 
+
+$(OUTDIR)/xhistogram.obj:	$(OUTDIR) xdiff/xhistogram.c  $(XDIFF_DEPS)
+	$(CC) $(CFLAGS_OUTDIR) xdiff/xhistogram.c 
+
+$(OUTDIR)/xpatience.obj:	$(OUTDIR) xdiff/xpatience.c  $(XDIFF_DEPS)
+	$(CC) $(CFLAGS_OUTDIR) xdiff/xpatience.c 
+
 $(OUTDIR)/digraph.obj:	$(OUTDIR) digraph.c  $(INCL)
 
 $(OUTDIR)/edit.obj:	$(OUTDIR) edit.c  $(INCL)
diff --git a/src/Makefile b/src/Makefile
index f2fafa4dca..d0cf956270 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -1400,6 +1400,32 @@ TERM_DEPS = \
 
 TERM_SRC = libvterm/src/*.c
 
+XDIFF_SRC = \
+	xdiff/xdiffi.c \
+	xdiff/xemit.c \
+	xdiff/xprepare.c \
+	xdiff/xutils.c \
+	xdiff/xhistogram.c \
+	xdiff/xpatience.c \
+
+XDIFF_OBJS = \
+	objects/xdiffi.o \
+	objects/xemit.o \
+	objects/xprepare.o \
+	objects/xutils.o \
+	objects/xhistogram.o \
+	objects/xpatience.o \
+
+XDIFF_INCL = \
+	xdiff/xdiff.h \
+	xdiff/xdiffi.h \
+	xdiff/xemit.h \
+	xdiff/xinclude.h \
+	xdiff/xmacros.h \
+	xdiff/xprepare.h \
+	xdiff/xtypes.h \
+	xdiff/xutils.h \
+
 ### Command to create dependencies based on #include "..."
 ### prototype headers are ignored due to -DPROTO, system
 ### headers #include <...> are ignored if we use the -MM option, as
@@ -1611,6 +1637,7 @@ BASIC_SRC = \
 SRC =	$(BASIC_SRC) \
 	$(GUI_SRC) \
 	$(TERM_SRC) \
+	$(XDIFF_SRC) \
 	$(HANGULIN_SRC) \
 	$(LUA_SRC) \
 	$(MZSCHEME_SRC) \
@@ -1728,6 +1755,7 @@ OBJ_COMMON = \
 	$(WORKSHOP_OBJ) \
 	$(NETBEANS_OBJ) \
 	$(CHANNEL_OBJ) \
+	$(XDIFF_OBJS) \
 	$(WSDEBUG_OBJ)
 
 # The files included by tests are not in OBJ_COMMON.
@@ -2731,7 +2759,7 @@ SHADOWDIR = shadow
 
 shadow:	runtime pixmaps
 	$(MKDIR_P) $(SHADOWDIR)
-	cd $(SHADOWDIR); ln -s ../*.[chm] ../*.in ../*.sh ../*.xs ../*.xbm ../gui_gtk_res.xml ../toolcheck ../proto ../libvterm ../vimtutor ../gvimtutor ../install-sh ../Make_all.mak .
+	cd $(SHADOWDIR); ln -s ../*.[chm] ../*.in ../*.sh ../*.xs ../*.xbm ../gui_gtk_res.xml ../toolcheck ../proto ../xdiff ../libvterm ../vimtutor ../gvimtutor ../install-sh ../Make_all.mak .
 	mkdir $(SHADOWDIR)/auto
 	cd $(SHADOWDIR)/auto; ln -s ../../auto/configure .
 	$(MKDIR_P) $(SHADOWDIR)/po
@@ -2915,7 +2943,7 @@ objects/crypt_zip.o: crypt_zip.c
 objects/dict.o: dict.c
 	$(CCC) -o $@ dict.c
 
-objects/diff.o: diff.c
+objects/diff.o: diff.c $(XDIFF_INCL)
 	$(CCC) -o $@ diff.c
 
 objects/digraph.o: digraph.c
@@ -3229,6 +3257,27 @@ objects/term_unicode.o: libvterm/src/unicode.c $(TERM_DEPS)
 objects/term_vterm.o: libvterm/src/vterm.c $(TERM_DEPS)
 	$(CCCTERM) -o $@ libvterm/src/vterm.c
 
+CCCDIFF = $(CCC_NF) $(ALL_CFLAGS)
+
+objects/xdiffi.o: xdiff/xdiffi.c $(XDIFF_INCL)
+	$(CCCDIFF) -o $@ xdiff/xdiffi.c
+
+objects/xprepare.o: xdiff/xprepare.c $(XDIFF_INCL)
+	$(CCCDIFF) -o $@ xdiff/xprepare.c
+
+objects/xutils.o: xdiff/xutils.c $(XDIFF_INCL)
+	$(CCCDIFF) -o $@ xdiff/xutils.c
+
+objects/xemit.o: xdiff/xemit.c $(XDIFF_INCL)
+	$(CCCDIFF) -o $@ xdiff/xemit.c
+
+objects/xhistogram.o: xdiff/xhistogram.c $(XDIFF_INCL)
+	$(CCCDIFF) -o $@ xdiff/xhistogram.c
+
+objects/xpatience.o: xdiff/xpatience.c $(XDIFF_INCL)
+	$(CCCDIFF) -o $@ xdiff/xpatience.c
+
+
 ###############################################################################
 ### MacOS X installation
 ###
diff --git a/src/diff.c b/src/diff.c
index c67654f621..4c0792baae 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -9,23 +9,32 @@
 
 /*
  * diff.c: code for diff'ing two, three or four buffers.
+ *
+ * There are three ways to diff:
+ * - Shell out to an external diff program, using files.
+ * - Use the compiled-in xdiff library.
+ * - Let 'diffexpr' do the work, using files.
  */
 
 #include "vim.h"
+#include "xdiff/xdiff.h"
 
 #if defined(FEAT_DIFF) || defined(PROTO)
 
 static int	diff_busy = FALSE;	/* ex_diffgetput() is busy */
 
 /* flags obtained from the 'diffopt' option */
-#define DIFF_FILLER	1	/* display filler lines */
-#define DIFF_ICASE	2	/* ignore case */
-#define DIFF_IWHITE	4	/* ignore change in white space */
-#define DIFF_HORIZONTAL	8	/* horizontal splits */
-#define DIFF_VERTICAL	16	/* vertical splits */
-#define DIFF_HIDDEN_OFF	32	/* diffoff when hidden */
+#define DIFF_FILLER	1	// display filler lines
+#define DIFF_ICASE	2	// ignore case
+#define DIFF_IWHITE	4	// ignore change in white space
+#define DIFF_HORIZONTAL	8	// horizontal splits
+#define DIFF_VERTICAL	16	// vertical splits
+#define DIFF_HIDDEN_OFF	32	// diffoff when hidden
+#define DIFF_INTERNAL	64	// use internal xdiff algorithm
 static int	diff_flags = DIFF_FILLER;
 
+static long diff_algorithm = 0;
+
 #define LBUFLEN 50		/* length of line in diff file */
 
 static int diff_a_works = MAYBE; /* TRUE when "diff -a" works, FALSE when it
@@ -36,22 +45,45 @@ static int diff_bin_works = MAYBE; /* TRUE when "diff --binary" works, FALSE
 				      checked yet */
 #endif
 
+// used for diff input
+typedef struct {
+    char_u	*din_fname;  // used for external diff
+    mmfile_t	din_mmfile;  // used for internal diff
+} diffin_T;
+
+// used for diff result
+typedef struct {
+    char_u	*dout_fname;  // used for external diff
+    garray_T	dout_ga;      // used for internal diff
+} diffout_T;
+
+// two diff inputs and one result
+typedef struct {
+    diffin_T	dio_orig;     // original file input
+    diffin_T	dio_new;      // new file input
+    diffout_T	dio_diff;     // diff result
+    int		dio_internal; // using internal diff
+} diffio_T;
+
 static int diff_buf_idx(buf_T *buf);
 static int diff_buf_idx_tp(buf_T *buf, tabpage_T *tp);
 static void diff_mark_adjust_tp(tabpage_T *tp, int idx, linenr_T line1, linenr_T line2, long amount, long amount_after);
 static void diff_check_unchanged(tabpage_T *tp, diff_T *dp);
 static int diff_check_sanity(tabpage_T *tp, diff_T *dp);
 static void diff_redraw(int dofold);
-static int diff_write(buf_T *buf, char_u *fname);
-static void diff_file(char_u *tmp_orig, char_u *tmp_new, char_u *tmp_diff);
+static int check_external_diff(diffio_T *diffio);
+static int diff_file(diffio_T *diffio);
 static int diff_equal_entry(diff_T *dp, int idx1, int idx2);
 static int diff_cmp(char_u *s1, char_u *s2);
 #ifdef FEAT_FOLDING
 static void diff_fold_update(diff_T *dp, int skip_idx);
 #endif
-static void diff_read(int idx_orig, int idx_new, char_u *fname);
+static void diff_read(int idx_orig, int idx_new, diffout_T *fname);
 static void diff_copy_entry(diff_T *dprev, diff_T *dp, int idx_orig, int idx_new);
 static diff_T *diff_alloc_new(tabpage_T *tp, diff_T *dprev, diff_T *dp);
+static int parse_diff_ed(char_u *line, linenr_T *lnum_orig, long *count_orig, linenr_T *lnum_new, long *count_new);
+static int parse_diff_unified(char_u *line, linenr_T *lnum_orig, long *count_orig, linenr_T *lnum_new, long *count_new);
+static int xdiff_out(void *priv, mmbuffer_t *mb, int nbuf);
 
 #ifndef USE_CR
 # define tag_fgets vim_fgets
@@ -631,81 +663,290 @@ diff_redraw(
 	}
 }
 
+    static void
+clear_diffin(diffin_T *din)
+{
+    if (din->din_fname == NULL)
+    {
+	vim_free(din->din_mmfile.ptr);
+	din->din_mmfile.ptr = NULL;
+    }
+    else
+	mch_remove(din->din_fname);
+}
+
+    static void
+clear_diffout(diffout_T *dout)
+{
+    if (dout->dout_fname == NULL)
+	ga_clear_strings(&dout->dout_ga);
+    else
+	mch_remove(dout->dout_fname);
+}
+
+/*
+ * Write buffer "buf" to a memory buffer.
+ * Return FAIL for failure.
+ */
+    static int
+diff_write_buffer(buf_T *buf, diffin_T *din)
+{
+    linenr_T	lnum;
+    char_u	*s;
+    long	len = 0;
+    char_u	*ptr;
+
+    // xdiff requires one big block of memory with all the text.
+    for (lnum = 1; lnum <= buf->b_ml.ml_line_count; ++lnum)
+	len += STRLEN(ml_get_buf(buf, lnum, FALSE)) + 1;
+    ptr = lalloc(len, TRUE);
+    if (ptr == NULL)
+    {
+	// Allocating memory failed.  This can happen, because we try to read
+	// the whole buffer text into memory.  Set the failed flag, the diff
+	// will be retried with external diff.  The flag is never reset.
+	buf->b_diff_failed = TRUE;
+	if (p_verbose > 0)
+	{
+	    verbose_enter();
+	    smsg((char_u *)
+		 _("Not enough memory to use internal diff for buffer \"%s\""),
+								 buf->b_fname);
+	    verbose_leave();
+	}
+	return FAIL;
+    }
+    din->din_mmfile.ptr = (char *)ptr;
+    din->din_mmfile.size = len;
+
+    len = 0;
+    for (lnum = 1; lnum <= buf->b_ml.ml_line_count; ++lnum)
+    {
+	for (s = ml_get_buf(buf, lnum, FALSE); *s != NUL; )
+	{
+	    if (diff_flags & DIFF_ICASE)
+	    {
+		int c;
+
+		// xdiff doesn't support ignoring case, fold-case the text.
+#ifdef FEAT_MBYTE
+		int	orig_len;
+		char_u	cbuf[MB_MAXBYTES + 1];
+
+		c = PTR2CHAR(s);
+		c = enc_utf8 ? utf_fold(c) : MB_TOLOWER(c);
+		orig_len = MB_PTR2LEN(s);
+		if (mb_char2bytes(c, cbuf) != orig_len)
+		    // TODO: handle byte length difference
+		    mch_memmove(ptr + len, s, orig_len);
+		else
+		    mch_memmove(ptr + len, cbuf, orig_len);
+
+		s += orig_len;
+		len += orig_len;
+#else
+		c = *s++;
+		ptr[len++] = TOLOWER_LOC(c);
+#endif
+	    }
+	    else
+		ptr[len++] = *s++;
+	}
+	ptr[len++] = NL;
+    }
+    return OK;
+}
+
 /*
- * Write buffer "buf" to file "name".
- * Always use 'fileformat' set to "unix".
- * Return FAIL for failure
+ * Write buffer "buf" to file or memory buffer.
+ * Return FAIL for failure.
  */
     static int
-diff_write(buf_T *buf, char_u *fname)
+diff_write(buf_T *buf, diffin_T *din)
 {
     int		r;
     char_u	*save_ff;
 
+    if (din->din_fname == NULL)
+	return diff_write_buffer(buf, din);
+
+    // Always use 'fileformat' set to "unix".
     save_ff = buf->b_p_ff;
     buf->b_p_ff = vim_strsave((char_u *)FF_UNIX);
-    r = buf_write(buf, fname, NULL, (linenr_T)1, buf->b_ml.ml_line_count,
-					     NULL, FALSE, FALSE, FALSE, TRUE);
+    r = buf_write(buf, din->din_fname, NULL,
+			(linenr_T)1, buf->b_ml.ml_line_count,
+			NULL, FALSE, FALSE, FALSE, TRUE);
     free_string_option(buf->b_p_ff);
     buf->b_p_ff = save_ff;
     return r;
 }
 
 /*
+ * Update the diffs for all buffers involved.
+ */
+    static void
+diff_try_update(
+	diffio_T    *dio,
+	int	    idx_orig,
+	exarg_T	    *eap)	// "eap" can be NULL
+{
+    buf_T	*buf;
+    int		idx_new;
+
+    if (dio->dio_internal)
+    {
+	ga_init2(&dio->dio_diff.dout_ga, sizeof(char *), 1000);
+    }
+    else
+    {
+	// We need three temp file names.
+	dio->dio_orig.din_fname = vim_tempname('o', TRUE);
+	dio->dio_new.din_fname = vim_tempname('n', TRUE);
+	dio->dio_diff.dout_fname = vim_tempname('d', TRUE);
+	if (dio->dio_orig.din_fname == NULL
+		|| dio->dio_new.din_fname == NULL
+		|| dio->dio_diff.dout_fname == NULL)
+	    goto theend;
+    }
+
+    // Check external diff is actually working.
+    if (!dio->dio_internal && check_external_diff(dio) == FAIL)
+	goto theend;
+
+    // :diffupdate!
+    if (eap != NULL && eap->forceit)
+	for (idx_new = idx_orig; idx_new < DB_COUNT; ++idx_new)
+	{
+	    buf = curtab->tp_diffbuf[idx_new];
+	    if (buf_valid(buf))
+		buf_check_timestamp(buf, FALSE);
+	}
+
+    // Write the first buffer to a tempfile or mmfile_t.
+    buf = curtab->tp_diffbuf[idx_orig];
+    if (diff_write(buf, &dio->dio_orig) == FAIL)
+	goto theend;
+
+    // Make a difference between the first buffer and every other.
+    for (idx_new = idx_orig + 1; idx_new < DB_COUNT; ++idx_new)
+    {
+	buf = curtab->tp_diffbuf[idx_new];
+	if (buf == NULL || buf->b_ml.ml_mfp == NULL)
+	    continue; // skip buffer that isn't loaded
+
+	// Write the other buffer and diff with the first one.
+	if (diff_write(buf, &dio->dio_new) == FAIL)
+	    continue;
+	if (diff_file(dio) == FAIL)
+	    continue;
+
+	// Read the diff output and add each entry to the diff list.
+	diff_read(idx_orig, idx_new, &dio->dio_diff);
+
+	clear_diffin(&dio->dio_new);
+	clear_diffout(&dio->dio_diff);
+    }
+    clear_diffin(&dio->dio_orig);
+
+theend:
+    vim_free(dio->dio_orig.din_fname);
+    vim_free(dio->dio_new.din_fname);
+    vim_free(dio->dio_diff.dout_fname);
+}
+
+/*
+ * Return TRUE if the options are set to use the internal diff library.
+ * Note that if the internal diff failed for one of the buffers, the external
+ * diff will be used anyway.
+ */
+    static int
+diff_internal(void)
+{
+    return (diff_flags & DIFF_INTERNAL) != 0 && *p_dex == NUL;
+}
+
+/*
+ * Return TRUE if the internal diff failed for one of the diff buffers.
+ */
+    static int
+diff_internal_failed(void)
+{
+    int idx;
+
+    // Only need to do something when there is another buffer.
+    for (idx = 0; idx < DB_COUNT; ++idx)
+	if (curtab->tp_diffbuf[idx] != NULL
+		&& curtab->tp_diffbuf[idx]->b_diff_failed)
+	    return TRUE;
+    return FALSE;
+}
+
+/*
  * Completely update the diffs for the buffers involved.
  * This uses the ordinary "diff" command.
  * The buffers are written to a file, also for unmodified buffers (the file
  * could have been produced by autocommands, e.g. the netrw plugin).
  */
     void
-ex_diffupdate(
-    exarg_T	*eap)	    /* can be NULL */
+ex_diffupdate(exarg_T *eap)	// "eap" can be NULL
 {
-    buf_T	*buf;
     int		idx_orig;
     int		idx_new;
-    char_u	*tmp_orig;
-    char_u	*tmp_new;
-    char_u	*tmp_diff;
-    FILE	*fd;
-    int		ok;
-    int		io_error = FALSE;
+    diffio_T	diffio;
 
-    /* Delete all diffblocks. */
+    // Delete all diffblocks.
     diff_clear(curtab);
     curtab->tp_diff_invalid = FALSE;
 
-    /* Use the first buffer as the original text. */
+    // Use the first buffer as the original text.
     for (idx_orig = 0; idx_orig < DB_COUNT; ++idx_orig)
 	if (curtab->tp_diffbuf[idx_orig] != NULL)
 	    break;
     if (idx_orig == DB_COUNT)
 	return;
 
-    /* Only need to do something when there is another buffer. */
+    // Only need to do something when there is another buffer.
     for (idx_new = idx_orig + 1; idx_new < DB_COUNT; ++idx_new)
 	if (curtab->tp_diffbuf[idx_new] != NULL)
 	    break;
     if (idx_new == DB_COUNT)
 	return;
 
-    /* We need three temp file names. */
-    tmp_orig = vim_tempname('o', TRUE);
-    tmp_new = vim_tempname('n', TRUE);
-    tmp_diff = vim_tempname('d', TRUE);
-    if (tmp_orig == NULL || tmp_new == NULL || tmp_diff == NULL)
-	goto theend;
+    // Only use the internal method if it did not fail for one of the buffers.
+    vim_memset(&diffio, 0, sizeof(diffio));
+    diffio.dio_internal = diff_internal() && !diff_internal_failed();
+
+    diff_try_update(&diffio, idx_orig, eap);
+    if (diffio.dio_internal && diff_internal_failed())
+    {
+	// Internal diff failed, use external diff instead.
+	vim_memset(&diffio, 0, sizeof(diffio));
+	diff_try_update(&diffio, idx_orig, eap);
+    }
+
+    // force updating cursor position on screen
+    curwin->w_valid_cursor.lnum = 0;
+
+    diff_redraw(TRUE);
+}
+
+/*
+ * Do a quick test if "diff" really works.  Otherwise it looks like there
+ * are no differences.  Can't use the return value, it's non-zero when
+ * there are differences.
+ */
+    static int
+check_external_diff(diffio_T *diffio)
+{
+    FILE	*fd;
+    int		ok;
+    int		io_error = FALSE;
 
-    /*
-     * Do a quick test if "diff" really works.  Otherwise it looks like there
-     * are no differences.  Can't use the return value, it's non-zero when
-     * there are differences.
-     * May try twice, first with "-a" and then without.
-     */