patch 8.0.0902: cannot specify directory or environment for a jobv8.0.0902

Problem:    Cannot specify directory or environment for a job.
Solution:   Add the "cwd" and "env" arguments to job options. (Yasuhiro
            Matsumoto, closes #1160)

9 files changed, 253 insertions, 44 deletions

diff --git a/runtime/doc/channel.txt b/runtime/doc/channel.txt
index f85c15022e..b9af2e6d82 100644
--- a/runtime/doc/channel.txt
+++ b/runtime/doc/channel.txt
@@ -427,8 +427,8 @@ When no message was available then the result is v:none for a JSON or JS mode
 channels, an empty string for a RAW or NL channel.  You can use |ch_canread()|
 to check if there is something to read.
 
-Note that when there is no callback message are dropped.  To avoid that add a
-close callback to the channel.
+Note that when there is no callback, messages are dropped.  To avoid that add
+a close callback to the channel.
 
 To read all output from a RAW channel that is available: >
 	let output = ch_readraw(channel)
@@ -475,11 +475,6 @@ it like this: >
 Without the handler you need to read the output with |ch_read()| or
 |ch_readraw()|. You can do this in the close callback, see |read-in-close-cb|.
 
-Note that if the job exits before you read the output, the output may be lost.
-This depends on the system (on Unix this happens because closing the write end
-of a pipe causes the read end to get EOF).  To avoid this make the job sleep
-for a short while before it exits.
-
 The handler defined for "out_cb" will not receive stderr.  If you want to
 handle that separately, add an "err_cb" handler: >
     let job = job_start(command, {"out_cb": "MyHandler",
@@ -494,6 +489,11 @@ started job gets the focus.  To avoid that, use the `foreground()` function.
 This might not always work when called early, put in the callback handler or
 use a timer to call it after the job has started.
 
+Depending on the system, starting a job can put Vim in the background, the
+started job gets the focus.  To avoid that, use the `foreground()` function.
+This might not always work when called early, put in the callback handler or
+use a timer to call it after the job has started.
+
 You can send a message to the command with ch_evalraw().  If the channel is in
 JSON or JS mode you can use ch_evalexpr().
 
@@ -696,6 +696,10 @@ See |job_setoptions()| and |ch_setoptions()|.
 "block_write": number	only for testing: pretend every other write to stdin
 			will block
 
+"env": dict		environment variables for the new process
+"cwd": "/path/to/dir"	current working directory for the new process;
+			if the directory does not exist an error is given
+
 
 Writing to a buffer ~
 							*out_io-buffer*
@@ -731,10 +735,6 @@ The "out_msg" option can be used to specify whether a new buffer will have the
 first line set to "Reading from channel output...".  The default is to add the
 message.  "err_msg" does the same for channel error.
 
-'modifiable' option off, or write to a buffer that has 'modifiable' off.  That
-means that lines will be appended to the buffer, but the user can't easily
-change the buffer.
-
 When an existing buffer is to be written where 'modifiable' is off and the
 "out_modifiable" or "err_modifiable" options is not zero, an error is given
 and the buffer will not be written to.
diff --git a/src/channel.c b/src/channel.c
index 7eb6ce73e3..3126cbd78f 100644
--- a/src/channel.c
+++ b/src/channel.c
@@ -4153,6 +4153,8 @@ free_job_options(jobopt_T *opt)
 	partial_unref(opt->jo_exit_partial);
     else if (opt->jo_exit_cb != NULL)
 	func_unref(opt->jo_exit_cb);
+    if (opt->jo_env != NULL)
+	dict_unref(opt->jo_env);
 }
 
 /*
@@ -4433,6 +4435,26 @@ get_job_options(typval_T *tv, jobopt_T *opt, int supported)
 		opt->jo_term_finish = *val;
 	    }
 #endif
+	    else if (STRCMP(hi->hi_key, "env") == 0)
+	    {
+		if (!(supported & JO2_ENV))
+		    break;
+		opt->jo_set |= JO2_ENV;
+		opt->jo_env = item->vval.v_dict;
+		++item->vval.v_dict->dv_refcount;
+	    }
+	    else if (STRCMP(hi->hi_key, "cwd") == 0)
+	    {
+		if (!(supported & JO2_CWD))
+		    break;
+		opt->jo_cwd = get_tv_string_buf_chk(item, opt->jo_cwd_buf);
+		if (opt->jo_cwd == NULL || !mch_isdir(opt->jo_cwd))
+		{
+		    EMSG2(_(e_invarg2), "cwd");
+		    return FAIL;
+		}
+		opt->jo_set |= JO2_CWD;
+	    }
 	    else if (STRCMP(hi->hi_key, "waittime") == 0)
 	    {
 		if (!(supported & JO_WAITTIME))
diff --git a/src/os_unix.c b/src/os_unix.c
index 141a3f0105..c56de66db2 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -5320,6 +5320,22 @@ mch_job_start(char **argv, job_T *job, jobopt_T *options)
 # endif
 	    set_default_child_environment();
 
+	if (options->jo_env != NULL)
+	{
+	    dict_T	*dict = options->jo_env;
+	    hashitem_T	*hi;
+	    int		todo = (int)dict->dv_hashtab.ht_used;
+
+	    for (hi = dict->dv_hashtab.ht_array; todo > 0; ++hi)
+		if (!HASHITEM_EMPTY(hi))
+		{
+		    typval_T *item = &dict_lookup(hi)->di_tv;
+
+		    vim_setenv((char_u*)hi->hi_key, get_tv_string(item));
+		    --todo;
+		}
+	}
+
 	if (use_null_for_in || use_null_for_out || use_null_for_err)
 	    null_fd = open("/dev/null", O_RDWR | O_EXTRA, 0);
 
@@ -5387,6 +5403,9 @@ mch_job_start(char **argv, job_T *job, jobopt_T *options)
 	if (null_fd >= 0)
 	    close(null_fd);
 
+	if (options->jo_cwd != NULL && mch_chdir((char *)options->jo_cwd) != 0)
+	    _exit(EXEC_FAILED);
+
 	/* See above for type of argv. */
 	execvp(argv[0], argv);
 
diff --git a/src/os_win32.c b/src/os_win32.c
index 9dc039b992..f98bd7e6ea 100644
--- a/src/os_win32.c
+++ b/src/os_win32.c
@@ -3981,31 +3981,46 @@ vim_create_process(
     BOOL		inherit_handles,
     DWORD		flags,
     STARTUPINFO		*si,
-    PROCESS_INFORMATION *pi)
+    PROCESS_INFORMATION *pi,
+    LPVOID		*env,
+    char		*cwd)
 {
 #ifdef FEAT_MBYTE
     if (enc_codepage >= 0 && (int)GetACP() != enc_codepage)
     {
-	WCHAR	*wcmd = enc_to_utf16((char_u *)cmd, NULL);
+	BOOL	ret;
+	WCHAR	*wcmd, *wcwd = NULL;
 
-	if (wcmd != NULL)
+	wcmd = enc_to_utf16((char_u *)cmd, NULL);
+	if (wcmd == NULL)
+	    goto fallback;
+	if (cwd != NULL)
 	{
-	    BOOL ret;
-	    ret = CreateProcessW(
-		NULL,			/* Executable name */
-		wcmd,			/* Command to execute */
-		NULL,			/* Process security attributes */
-		NULL,			/* Thread security attributes */
-		inherit_handles,	/* Inherit handles */
-		flags,			/* Creation flags */
-		NULL,			/* Environment */
-		NULL,			/* Current directory */
-		(LPSTARTUPINFOW)si,	/* Startup information */
-		pi);			/* Process information */
-	    vim_free(wcmd);
-	    return ret;
+	    wcwd = enc_to_utf16((char_u *)cwd, NULL);
+	    if (wcwd == NULL)
+	    {
+		vim_free(wcmd);
+		goto fallback;
+	    }
 	}
-    }
+
+	ret = CreateProcessW(
+	    NULL,			/* Executable name */
+	    wcmd,			/* Command to execute */
+	    NULL,			/* Process security attributes */
+	    NULL,			/* Thread security attributes */
+	    inherit_handles,	/* Inherit handles */
+	    flags,			/* Creation flags */
+	    env,			/* Environment */
+	    wcwd,			/* Current directory */
+	    (LPSTARTUPINFOW)si,	/* Startup information */
+	    pi);			/* Process information */
+	vim_free(wcmd);
+	if (wcwd != NULL)
+	    vim_free(wcwd);
+	return ret;
+    }
+fallback:
 #endif
     return CreateProcess(
 	NULL,			/* Executable name */
@@ -4014,8 +4029,8 @@ vim_create_process(
 	NULL,			/* Thread security attributes */
 	inherit_handles,	/* Inherit handles */
 	flags,			/* Creation flags */
-	NULL,			/* Environment */
-	NULL,			/* Current directory */
+	env,			/* Environment */
+	cwd,			/* Current directory */
 	si,			/* Startup information */
 	pi);			/* Process information */
 }
@@ -4079,7 +4094,8 @@ mch_system_classic(char *cmd, int options)
 
     /* Now, run the command */
     vim_create_process(cmd, FALSE,
-	    CREATE_DEFAULT_ERROR_MODE |	CREATE_NEW_CONSOLE, &si, &pi);
+	    CREATE_DEFAULT_ERROR_MODE |	CREATE_NEW_CONSOLE,
+	    &si, &pi, NULL, NULL);
 
     /* Wait for the command to terminate before continuing */
     {
@@ -4398,7 +4414,8 @@ mch_system_piped(char *cmd, int options)
      * About "Inherit handles" being TRUE: this command can be litigious,
      * handle inheritance was deactivated for pending temp file, but, if we
      * deactivate it, the pipes don't work for some reason. */
-     vim_create_process(p, TRUE, CREATE_DEFAULT_ERROR_MODE, &si, &pi);
+     vim_create_process(p, TRUE, CREATE_DEFAULT_ERROR_MODE,
+	     &si, &pi, NULL, NULL);
 
     if (p != cmd)
 	vim_free(p);
@@ -4835,7 +4852,8 @@ mch_call_shell(
 	     * inherit our handles which causes unpleasant dangling swap
 	     * files if we exit before the spawned process
 	     */
-	    if (vim_create_process((char *)newcmd, FALSE, flags, &si, &pi))
+	    if (vim_create_process((char *)newcmd, FALSE, flags,
+			&si, &pi, NULL, NULL))
 		x = 0;
 	    else if (vim_shell_execute((char *)newcmd, n_show_cmd)
 							       > (HINSTANCE)32)
@@ -4976,6 +4994,67 @@ job_io_file_open(
     return h;
 }
 
+/*
+ * Turn the dictionary "env" into a NUL separated list that can be used as the
+ * environment argument of vim_create_process().
+ */
+    static void
+make_job_env(garray_T *gap, dict_T *env)
+{
+    hashitem_T	*hi;
+    int		todo = (int)env->dv_hashtab.ht_used;
+    LPVOID	base = GetEnvironmentStringsW();
+
+    /* for last \0 */
+    if (ga_grow(gap, 1) == FAIL)
+	return;
+
+    if (base)
+    {
+	WCHAR	*p = (WCHAR*) base;
+
+	/* for last \0 */
+	if (ga_grow(gap, 1) == FAIL)
+	    return;
+
+	while (*p != 0 || *(p + 1) != 0)
+	{
+	    if (ga_grow(gap, 1) == OK)
+		*((WCHAR*)gap->ga_data + gap->ga_len++) = *p;
+	    p++;
+	}
+	FreeEnvironmentStrings(base);
+	*((WCHAR*)gap->ga_data + gap->ga_len++) = L'\0';
+    }
+
+    for (hi = env->dv_hashtab.ht_array; todo > 0; ++hi)
+    {
+	if (!HASHITEM_EMPTY(hi))
+	{
+	    typval_T *item = &dict_lookup(hi)->di_tv;
+	    WCHAR   *wkey = enc_to_utf16((char_u *)hi->hi_key, NULL);
+	    WCHAR   *wval = enc_to_utf16(get_tv_string(item), NULL);
+	    --todo;
+	    if (wkey != NULL && wval != NULL)
+	    {
+		int n, lkey = wcslen(wkey), lval = wcslen(wval);
+		if (ga_grow(gap, lkey + lval + 2) != OK)
+		    continue;
+		for (n = 0; n < lkey; n++)
+		    *((WCHAR*)gap->ga_data + gap->ga_len++) = wkey[n];
+		*((WCHAR*)gap->ga_data + gap->ga_len++) = L'=';
+		for (n = 0; n < lval; n++)
+		    *((WCHAR*)gap->ga_data + gap->ga_len++) = wval[n];
+		*((WCHAR*)gap->ga_data + gap->ga_len++) = L'\0';
+	    }
+	    if (wkey != NULL) vim_free(wkey);
+	    if (wval != NULL) vim_free(wval);
+	}
+    }
+
+    *((WCHAR*)gap->ga_data + gap->ga_len++) = L'\0';
+}
+
     void
 mch_job_start(char *cmd, job_T *job, jobopt_T *options)
 {
@@ -4987,6 +5066,7 @@ mch_job_start(char *cmd, job_T *job, jobopt_T *options)
     HANDLE		ifd[2];
     HANDLE		ofd[2];
     HANDLE		efd[2];
+    garray_T		ga;
 
     int		use_null_for_in = options->jo_io[PART_IN] == JIO_NULL;
     int		use_null_for_out = options->jo_io[PART_OUT] == JIO_NULL;
@@ -5005,6 +5085,7 @@ mch_job_start(char *cmd, job_T *job, jobopt_T *options)
     ofd[1] = INVALID_HANDLE_VALUE;
     efd[0] = INVALID_HANDLE_VALUE;
     efd[1] = INVALID_HANDLE_VALUE;
+    ga_init2(&ga, (int)sizeof(wchar_t), 500);
 
     jo = CreateJobObject(NULL, NULL);
     if (jo == NULL)
@@ -5013,6 +5094,9 @@ mch_job_start(char *cmd, job_T *job, jobopt_T *options)
 	goto failed;
     }
 
+    if (options->jo_env != NULL)
+	make_job_env(&ga, options->jo_env);
+
     ZeroMemory(&pi, sizeof(pi));
     ZeroMemory(&si, sizeof(si));
     si.cb = sizeof(si);
@@ -5100,14 +5184,19 @@ mch_job_start(char *cmd, job_T *job, jobopt_T *options)
 	    CREATE_SUSPENDED |
 	    CREATE_DEFAULT_ERROR_MODE |
 	    CREATE_NEW_PROCESS_GROUP |
+	    CREATE_UNICODE_ENVIRONMENT |
 	    CREATE_NEW_CONSOLE,
-	    &si, &pi))
+	    &si, &pi,
+	    ga.ga_data,
+	    (char *)options->jo_cwd))
     {
 	CloseHandle(jo);
 	job->jv_status = JOB_FAILED;
 	goto failed;
     }
 
+    ga_clear(&ga);
+
     if (!AssignProcessToJobObject(jo, pi.hProcess))
     {
 	/* if failing, switch the way to terminate
@@ -5148,6 +5237,7 @@ failed:
     CloseHandle(ofd[1]);
     CloseHandle(efd[1]);
     channel_unref(channel);
+    ga_clear(&ga);
 }
 
     char *
diff --git a/src/structs.h b/src/structs.h
index 618cabfe75..16f2cc1d07 100644
--- a/src/structs.h
+++ b/src/structs.h
@@ -1686,7 +1686,9 @@ struct channel_S {
 #define JO2_ERR_MSG	    0x0002	/* "err_msg" (JO_OUT_ << 1) */
 #define JO2_TERM_NAME	    0x0004	/* "term_name" */
 #define JO2_TERM_FINISH	    0x0008	/* "term_finish" */
-#define JO2_ALL		    0x000F
+#define JO2_ENV		    0x0010	/* "env" */
+#define JO2_CWD		    0x0020	/* "cwd" */
+#define JO2_ALL		    0x003F
 
 #define JO_MODE_ALL	(JO_MODE + JO_IN_MODE + JO_OUT_MODE + JO_ERR_MODE)
 #define JO_CB_ALL \
@@ -1738,6 +1740,9 @@ typedef struct
     int		jo_id;
     char_u	jo_soe_buf[NUMBUFLEN];
     char_u	*jo_stoponexit;
+    dict_T	*jo_env;	/* environment variables */
+    char_u	jo_cwd_buf[NUMBUFLEN];
+    char_u	*jo_cwd;
 
 #ifdef FEAT_TERMINAL
     /* when non-zero run the job in a terminal window of this size */
diff --git a/src/terminal.c b/src/terminal.c
index b22cc1a39a..c215b3fa8c 100644
--- a/src/terminal.c
+++ b/src/terminal.c
@@ -2362,7 +2362,8 @@ f_term_start(typval_T *argvars, typval_T *rettv)
 	    && get_job_options(&argvars[1], &opt,
 		JO_TIMEOUT_ALL + JO_STOPONEXIT
 		+ JO_EXIT_CB + JO_CLOSE_CALLBACK
-		+ JO2_TERM_NAME + JO2_TERM_FINISH) == FAIL)
+		+ JO2_TERM_NAME + JO2_TERM_FINISH
+		+ JO2_CWD + JO2_ENV) == FAIL)
 	return;
 
     term_start(cmd, &opt);
diff --git a/src/testdir/test_channel.vim b/src/testdir/test_channel.vim
index c988968bb2..42f0810ee3 100644
--- a/src/testdir/test_channel.vim
+++ b/src/testdir/test_channel.vim
@@ -1664,6 +1664,45 @@ func Test_read_from_terminated_job()
   call assert_equal(1, g:linecount)
 endfunc
 
+func Test_env()
+  if !has('job')
+    return
+  endif
+
+  let s:envstr = ''
+  if has('win32')
+    call job_start(['cmd', '/c', 'echo %FOO%'], {'callback': {ch,msg->execute(":let s:envstr .= msg")}, 'env':{'FOO': 'bar'}})
+  else
+    call job_start([&shell, &shellcmdflag, 'echo $FOO'], {'callback': {ch,msg->execute(":let s:envstr .= msg")}, 'env':{'FOO': 'bar'}})
+  endif
+  call WaitFor('"" != s:envstr')
+  call assert_equal("bar", s:envstr)
+  unlet s:envstr
+endfunc
+
+func Test_cwd()
+  if !has('job')
+    return
+  endif
+
+  let s:envstr = ''
+  if has('win32')
+    let expect = $TEMP
+    call job_start(['cmd', '/c', 'echo %CD%'], {'callback': {ch,msg->execute(":let s:envstr .= msg")}, 'cwd': expect})
+  else
+    let expect = $HOME
+    call job_start(['pwd'], {'callback': {ch,msg->execute(":let s:envstr .= msg")}, 'cwd': expect})
+  endif
+  call WaitFor('"" != s:envstr')
+  let expect = substitute(expect, '[/\\]$', '', '')
+  let s:envstr = substitute(s:envstr, '[/\\]$', '', '')
+  if $CI != '' && stridx(s:envstr, '/private/') == 0
+    let s:envstr = s:envstr[8:]
+  endif
+  call assert_equal(expect, s:envstr)
+  unlet s:envstr
+endfunc
+
 function Ch_test_close_lambda(port)
   let handle = ch_open('localhost:' . a:port, s:chopt)
   if ch_status(handle) == "fail"
diff --git a/src/testdir/test_terminal.vim b/src/testdir/test_terminal.vim
index 89a784b340..9cc3a55cfd 100644
--- a/src/testdir/test_terminal.vim
+++ b/src/testdir/test_terminal.vim
@@ -8,8 +8,8 @@ source shared.vim
 
 " Open a terminal with a shell, assign the job to g:job and return the buffer
 " number.
-func Run_shell_in_terminal()
-  let buf = term_start(&shell)
+func Run_shell_in_terminal(options)
+  let buf = term_start(&shell, a:options)
 
   let termlist = term_list()
   call assert_equal(1, len(termlist))
@@ -32,7 +32,7 @@ func Stop_shell_in_terminal(buf)
 endfunc
 
 func Test_terminal_basic()
-  let buf = Run_shell_in_terminal()
+  let buf = Run_shell_in_terminal({})
   if has("unix")
     call assert_match("^/dev/", job_info(g:job).tty)
     call assert_match("^/dev/", term_gettty(''))
@@ -51,7 +51,7 @@ func Test_terminal_basic()
 endfunc
 
 func Test_terminal_make_change()
-  let buf = Run_shell_in_terminal()
+  let buf = Run_shell_in_terminal({})
   call Stop_shell_in_terminal(buf)
   call term_wait(buf)
 
@@ -65,7 +65,7 @@ func Test_terminal_make_change()
 endfunc
 
 func Test_terminal_wipe_buffer()
-  let buf = Run_shell_in_terminal()
+  let buf = Run_shell_in_terminal({})
   call assert_fails(buf . 'bwipe', 'E517')
   exe buf . 'bwipe!'
   call WaitFor('job_status(g:job) == "dead"')
@@ -76,7 +76,7 @@ func Test_terminal_wipe_buffer()
 endfunc
 
 func Test_terminal_hide_buffer()
-  let buf = Run_shell_in_terminal()
+  let buf = Run_shell_in_terminal({})
   quit
   for nr in range(1, winnr('$'))
     call assert_notequal(winbufnr(nr), buf)
@@ -266,9 +266,11 @@ func Test_terminal_size()
 endfunc
 
 func Test_finish_close()
+  return
+  " TODO: use something that takes much less than a whole second
+  echo 'This will take five seconds...'
   call assert_equal(1, winnr('$'))
 
-  " TODO: use something that takes much less than a whole second
   if has('win32')
     let cmd = $windir . '\system32\timeout.exe 1'
   else
@@ -304,3 +306,32 @@ func Test_finish_close()
 
   bwipe
 endfunc
+
+func Test_terminal_cwd()
+  if !has('unix')
+    return
+  endif
+  call mkdir('Xdir')
+  let buf = term_start('pwd', {'cwd': 'Xdir'})
+  sleep 100m
+  call term_wait(buf)
+  call assert_equal(getcwd() . '/Xdir', getline(1))
+
+  exe buf . 'bwipe'
+  call delete('Xdir', 'rf')
+endfunc
+
+func Test_terminal_env()
+  if !has('unix')
+    return
+  endif
+  let buf = Run_shell_in_terminal({'env': {'TESTENV': 'correct'}})
+  call term_wait(buf)
+  call term_sendkeys(buf, "echo $TESTENV\r")
+  call term_wait(buf)
+  call Stop_shell_in_terminal(buf)
+  call term_wait(buf)
+  call assert_equal('correct', getline(2))
+
+  exe buf . 'bwipe'
+endfunc
diff --git a/src/version.c b/src/version.c
index 469e01a5ce..ce00ff2632 100644
--- a/src/version.c
+++ b/src/version.c
@@ -770,6 +770,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    902,
+/**/
     901,
 /**/
     900,