patch 8.1.0735: cannot handle binary datav8.1.0735

Problem:    Cannot handle binary data.
Solution:   Add the Blob type. (Yasuhiro Matsumoto, closes #3638)

3 files changed, 95 insertions, 31 deletions

diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index 86026239a9..b9c9ca42ad 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -72,6 +72,10 @@ Job		Used for a job, see |job_start()|. *Job* *Jobs*
 
 Channel		Used for a channel, see |ch_open()|. *Channel* *Channels*
 
+Blob		Binary Large Object. Stores any sequence of bytes. *Blob*
+		Example: 0zFF00ED015DAF
+		0z is an empty Blob.
+
 The Number and String types are converted automatically, depending on how they
 are used.
 
@@ -124,7 +128,8 @@ Note that " " and "0" are also non-empty strings, thus considered to be TRUE.
 A List, Dictionary or Float is not a Number or String, thus evaluate to FALSE.
 
 		*E745* *E728* *E703* *E729* *E730* *E731* *E908* *E910* *E913*
-List, Dictionary, Funcref, Job and Channel types are not automatically
+		*E974* *E975* *E976*
+List, Dictionary, Funcref, Job, Channel and Blob types are not automatically
 converted.
 
 							*E805* *E806* *E808*
@@ -1017,6 +1022,12 @@ just above. Also see |sublist| below.  Examples: >
 	:let l = mylist[4:4]		" List with one item
 	:let l = mylist[:]		" shallow copy of a List
 
+If expr8 is a |Blob| this results in a new |Blob| with the bytes in the
+indexes expr1a and expr1b, inclusive.  Examples: >
+	:let b = 0zDEADBEEF
+	:let bs = b[1:2]		" 0zADBE
+	:let bs = b[]			" copy ov 0zDEADBEEF
+
 Using expr8[expr1] or expr8[expr1a : expr1b] on a |Funcref| results in an
 error.
 
@@ -1156,6 +1167,14 @@ of 'encoding'.
 Note that "\000" and "\x00" force the end of the string.
 
 
+blob-literal				*blob-literal* *E973* *E977* *E978*
+------------
+
+Hexadecimal starting with 0z or 0Z, with an arbitrary number of bytes.
+The sequence must be an even number of hex characters.  Example: >
+	:let b = 0zFF00ED015DAF
+
+
 literal-string						*literal-string* *E115*
 ---------------
 'string'		string constant			*expr-'*
@@ -1911,6 +1930,8 @@ v:t_none	Value of None type.  Read-only.  See: |type()|
 v:t_number	Value of Number type.  Read-only.  See: |type()|
 					*v:t_string* *t_string-variable*
 v:t_string	Value of String type.  Read-only.  See: |type()|
+					*v:t_blob* *t_blob-variable*
+v:t_blob	Value of Blob type.  Read-only.  See: |type()|
 
 				*v:termresponse* *termresponse-variable*
 v:termresponse	The escape sequence returned by the terminal for the |t_RV|
@@ -2094,12 +2115,14 @@ ch_logfile({fname} [, {mode}])	none	start logging channel activity
 ch_open({address} [, {options}])
 				Channel	open a channel to {address}
 ch_read({handle} [, {options}]) String	read from {handle}
+ch_readblob({handle} [, {options}])
+				Blob	read Blob from {handle}
 ch_readraw({handle} [, {options}])
 				String	read raw from {handle}
 ch_sendexpr({handle}, {expr} [, {options}])
 				any	send {expr} over JSON {handle}
-ch_sendraw({handle}, {string} [, {options}])
-				any	send {string} over raw {handle}
+ch_sendraw({handle}, {expr} [, {options}])
+				any	send {expr} over raw {handle}
 ch_setoptions({handle}, {options})
 				none	set options for {handle}
 ch_status({handle} [, {options}])
@@ -2239,8 +2262,8 @@ hlID({name})			Number	syntax ID of highlight group {name}
 hostname()			String	name of the machine Vim is running on
 iconv({expr}, {from}, {to})	String	convert encoding of {expr}
 indent({lnum})			Number	indent of line {lnum}
-index({list}, {expr} [, {start} [, {ic}]])
-				Number	index in {list} where {expr} appears
+index({object}, {expr} [, {start} [, {ic}]])
+				Number	index in {object} where {expr} appears
 input({prompt} [, {text} [, {completion}]])
 				String	get input from the user
 inputdialog({prompt} [, {text} [, {completion}]])
@@ -2249,7 +2272,7 @@ inputlist({textlist})		Number	let the user pick from a choice list
 inputrestore()			Number	restore typeahead
 inputsave()			Number	save and clear typeahead
 inputsecret({prompt} [, {text}]) String	like input() but hiding the text
-insert({list}, {item} [, {idx}]) List	insert {item} in {list} [before {idx}]
+insert({object}, {item} [, {idx}]) List	insert {item} in {object} [before {idx}]
 invert({expr})			Number	bitwise invert
 isdirectory({directory})	Number	|TRUE| if {directory} is a directory
 islocked({expr})		Number	|TRUE| if {expr} is locked
@@ -2339,7 +2362,7 @@ py3eval({expr})			any	evaluate |python3| expression
 pyxeval({expr})			any	evaluate |python_x| expression
 range({expr} [, {max} [, {stride}]])
 				List	items from {expr} to {max}
-readfile({fname} [, {binary} [, {max}]])
+readfile({fname} [, {type} [, {max}]])
 				List	get list of lines from file {fname}
 reg_executing()			String	get the executing register name
 reg_recording()			String	get the recording register name
@@ -2554,8 +2577,8 @@ winrestview({dict})		none	restore view of current window
 winsaveview()			Dict	save view of current window
 winwidth({nr})			Number	width of window {nr}
 wordcount()			Dict	get byte/char/word statistics
-writefile({list}, {fname} [, {flags}])
-				Number	write list of lines to file {fname}
+writefile({object}, {fname} [, {flags}])
+				Number	write |Blob| or |List| of lines to file
 xor({expr}, {expr})		Number	bitwise XOR
 
 
@@ -3199,6 +3222,11 @@ ch_read({handle} [, {options}])					*ch_read()*
 		See |channel-more|.
 		{only available when compiled with the |+channel| feature}
 
+ch_readblob({handle} [, {options}])			*ch_readblob()*
+		Like ch_read() but reads binary data and returns a Blob.
+		See |channel-more|.
+		{only available when compiled with the |+channel| feature}
+
 ch_readraw({handle} [, {options}])			*ch_readraw()*
 		Like ch_read() but for a JS and JSON channel does not decode
 		the message.  For a NL channel it does not block waiting for
@@ -3215,8 +3243,8 @@ ch_sendexpr({handle}, {expr} [, {options}])			*ch_sendexpr()*
 
 		{only available when compiled with the |+channel| feature}
 
-ch_sendraw({handle}, {string} [, {options}])		*ch_sendraw()*
-		Send {string} over {handle}.
+ch_sendraw({handle}, {expr} [, {options}])		*ch_sendraw()*
+		Send string or Blob {expr} over {handle}.
 		Works like |ch_sendexpr()|, but does not encode the request or
 		decode the response.  The caller is responsible for the
 		correct contents.  Also does not add a newline for a channel
@@ -5375,17 +5403,21 @@ indent({lnum})	The result is a Number, which is indent of line {lnum} in the
 		When {lnum} is invalid -1 is returned.
 
 
-index({list}, {expr} [, {start} [, {ic}]])			*index()*
-		Return the lowest index in |List| {list} where the item has a
-		value equal to {expr}.  There is no automatic conversion, so
-		the String "4" is different from the Number 4.  And the number
-		4 is different from the Float 4.0.  The value of 'ignorecase'
-		is not used here, case always matters.
+index({object}, {expr} [, {start} [, {ic}]])			*index()*
+		If {object} is a |List| return the lowest index where the item
+		has a value equal to {expr}.  There is no automatic
+		conversion, so the String "4" is different from the Number 4.
+		And the number 4 is different from the Float 4.0.  The value
+		of 'ignorecase' is not used here, case always matters.
+
+		If {object} is |Blob| return the lowest index where the byte
+		value is equal to {expr}.
+
 		If {start} is given then start looking at the item with index
 		{start} (may be negative for an item relative to the end).
 		When {ic} is given and it is |TRUE|, ignore case.  Otherwise
 		case must match.
-		-1 is returned when {expr} is not found in {list}.
+		-1 is returned when {expr} is not found in {object}.
 		Example: >
 			:let idx = index(words, "the")
 			:if index(numbers, 123) >= 0
@@ -5491,13 +5523,16 @@ inputsecret({prompt} [, {text}])			*inputsecret()*
 		typed on the command-line in response to the issued prompt.
 		NOTE: Command-line completion is not supported.
 
-insert({list}, {item} [, {idx}])			*insert()*
-		Insert {item} at the start of |List| {list}.
+insert({object}, {item} [, {idx}])			*insert()*
+		When {object} is a |List| or a |Blob| insert {item} at the start
+		of it.
+
 		If {idx} is specified insert {item} before the item with index
 		{idx}.  If {idx} is zero it goes before the first item, just
 		like omitting {idx}.  A negative {idx} is also possible, see
 		|list-index|.  -1 inserts just before the last item.
-		Returns the resulting |List|.  Examples: >
+
+		Returns the resulting |List| or |Blob|.  Examples: >
 			:let mylist = insert([2, 3, 5], 1)
 			:call insert(mylist, 4, -1)
 			:call insert(mylist, 6, len(mylist))
@@ -5763,6 +5798,7 @@ json_encode({expr})					*json_encode()*
 					used recursively: []
 		   Dict			as an object (possibly null); when
 					used recursively: {}
+		   Blob			as an array of the individual bytes
 		   v:false		"false"
 		   v:true		"true"
 		   v:none		"null"
@@ -6947,16 +6983,18 @@ range({expr} [, {max} [, {stride}]])				*range()*
 			range(2, 0)		" error!
 <
 							*readfile()*
-readfile({fname} [, {binary} [, {max}]])
+readfile({fname} [, {type} [, {max}]])
 		Read file {fname} and return a |List|, each line of the file
 		as an item.  Lines are broken at NL characters.  Macintosh
 		files separated with CR will result in a single long line
 		(unless a NL appears somewhere).
 		All NUL characters are replaced with a NL character.
-		When {binary} contains "b" binary mode is used:
+		When {type} contains "b" binary mode is used:
 		- When the last line ends in a NL an extra empty list item is
 		  added.
 		- No CR characters are removed.
+		When {type} contains "B" a |Blob| is returned with the binary
+		data of the file unmodified.
 		Otherwise:
 		- CR characters that appear before a NL are removed.
 		- Whether the last line ends in a NL or not does not matter.
@@ -7132,6 +7170,16 @@ remove({list}, {idx} [, {end}])				*remove()*
 		Example: >
 			:echo "last item: " . remove(mylist, -1)
 			:call remove(mylist, 0, 9)
+remove({blob}, {idx} [, {end}])
+		Without {end}: Remove the byte at {idx} from |Blob| {blob} and
+		return the byte.
+		With {end}: Remove bytes from {idx} to {end} (inclusive) and
+		return a |Blob| with these bytes.  When {idx} points to the same
+		byte as {end} a |Blob| with one byte is returned.  When {end}
+		points to a byte before {idx} this is an error.
+		Example: >
+			:echo "last byte: " . remove(myblob, -1)
+			:call remove(mylist, 0, 9)
 remove({dict}, {key})
 		Remove the entry from {dict} with key {key}.  Example: >
 			:echo "removed " . remove(dict, "one")
@@ -7172,9 +7220,11 @@ resolve({filename})					*resolve()* *E655*
 		path name) and also keeps a trailing path separator.
 
 							*reverse()*
-reverse({list})	Reverse the order of items in {list} in-place.  Returns
-		{list}.
-		If you want a list to remain unmodified make a copy first: >
+reverse({object})
+		Reverse the order of items in {object} in-place.
+		{object} can be a |List| or a |Blob|.
+		Returns {object}.
+		If you want an object to remain unmodified make a copy first: >
 			:let revlist = reverse(copy(mylist))
 
 round({expr})							*round()*
@@ -9518,6 +9568,7 @@ type({expr})	The result is a Number representing the type of {expr}.
 			None	    7  |v:t_none| (v:null and v:none)
 			Job	    8  |v:t_job|
 			Channel	    9  |v:t_channel|
+			Blob	   10  |v:t_blob|
 		For backward compatibility, this method can be used: >
 			:if type(myvar) == type(0)
 			:if type(myvar) == type("")
@@ -9865,14 +9916,17 @@ wordcount()						*wordcount()*
 
 
 							*writefile()*
-writefile({list}, {fname} [, {flags}])
-		Write |List| {list} to file {fname}.  Each list item is
-		separated with a NL.  Each list item must be a String or
-		Number.
+writefile({object}, {fname} [, {flags}])
+		When {object} is a |List| write it to file {fname}.  Each list
+		item is separated with a NL.  Each list item must be a String
+		or Number.
 		When {flags} contains "b" then binary mode is used: There will
 		not be a NL after the last list item.  An empty item at the
 		end does cause the last line in the file to end in a NL.
 
+		When {object} is a |Blob| write the bytes to file {fname}
+		unmodified.
+
 		When {flags} contains "a" then append mode is used, lines are
 		appended to the file: >
 			:call writefile(["foo"], "event.log", "a")
@@ -10575,7 +10629,10 @@ This does NOT work: >
 			This cannot be used to set a byte in a String.  You
 			can do that like this: >
 				:let var = var[0:2] . 'X' . var[4:]
-<
+<			When {var-name} is a |Blob| then {idx} can be the
+			length of the blob, in which case one byte is
+			appended.
+
 							*E711* *E719*
 :let {var-name}[{idx1}:{idx2}] = {expr1}		*E708* *E709* *E710*
 			Set a sequence of items in a |List| to the result of
diff --git a/runtime/doc/if_perl.txt b/runtime/doc/if_perl.txt
index 26ea29e0db..cce3cdf5d9 100644
--- a/runtime/doc/if_perl.txt
+++ b/runtime/doc/if_perl.txt
@@ -191,6 +191,9 @@ VIM::Eval({expr})	Evaluates {expr} and returns (success, value) in list
 			A |List| is turned into a string by joining the items
 			and inserting line breaks.
 
+							*perl-Blob*
+VIM::Blob({expr})	Return Blob literal string 0zXXXX from scalar value.
+
 							*perl-SetHeight*
 Window->SetHeight({height})
 			Sets the Window height to {height}, within screen
diff --git a/runtime/doc/if_ruby.txt b/runtime/doc/if_ruby.txt
index b1404769f8..c9fc797eb1 100644
--- a/runtime/doc/if_ruby.txt
+++ b/runtime/doc/if_ruby.txt
@@ -110,6 +110,10 @@ Module Functions:
 Vim::message({msg})
 	Displays the message {msg}.
 
+							*ruby-blob*
+Vim::blob({arg})
+	Return Blob literal string from {arg}.
+
 							*ruby-set_option*
 Vim::set_option({arg})
 	Sets a vim option.  {arg} can be any argument that the ":set" command