patch 8.2.4648: handling LSP messages is a bit slowv8.2.4648

Problem:    Handling LSP messages is a bit slow.
Solution:   Included support for LSP messages. (Yegappan Lakshmanan,
            closes #10025)

2 files changed, 508 insertions, 0 deletions

diff --git a/src/testdir/test_channel.vim b/src/testdir/test_channel.vim
index d9ab521f88..122bc570aa 100644
--- a/src/testdir/test_channel.vim
+++ b/src/testdir/test_channel.vim
@@ -2378,5 +2378,214 @@ func Test_job_start_with_invalid_argument()
   call assert_fails('call job_start([0zff])', 'E976:')
 endfunc
 
+" Test for the 'lsp' channel mode
+func LspCb(chan, msg)
+  call add(g:lspNotif, a:msg)
+endfunc
+
+func LspOtCb(chan, msg)
+  call add(g:lspOtMsgs, a:msg)
+endfunc
+
+func LspTests(port)
+  " call ch_logfile('Xlsprpc.log', 'w')
+  let ch = ch_open(s:localhost .. a:port, #{mode: 'lsp', callback: 'LspCb'})
+  if ch_status(ch) == "fail"
+    call assert_report("Can't open the lsp channel")
+    return
+  endif
+
+  " check for channel information
+  let info = ch_info(ch)
+  call assert_equal('LSP', info.sock_mode)
+
+  " Evaluate an expression
+  let resp = ch_evalexpr(ch, #{method: 'simple-rpc', params: [10, 20]})
+  call assert_false(empty(resp))
+  call assert_equal(#{id: 1, jsonrpc: '2.0', result: 'simple-rpc'}, resp)
+
+  " Evaluate an expression. While waiting for the response, a notification
+  " message is delivered.
+  let g:lspNotif = []
+  let resp = ch_evalexpr(ch, #{method: 'rpc-with-notif', params: {'v': 10}})
+  call assert_false(empty(resp))
+  call assert_equal(#{id: 2, jsonrpc: '2.0', result: 'rpc-with-notif-resp'},
+        \ resp)
+  call assert_equal([#{jsonrpc: '2.0', result: 'rpc-with-notif-notif'}],
+        \ g:lspNotif)
+
+  " Wrong payload notification test
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'wrong-payload', params: {}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result: 'wrong-payload'}], g:lspNotif)
+
+  " Test for receiving a response with incorrect 'id' and additional
+  " notification messages while evaluating an expression.
+  let g:lspNotif = []
+  let resp = ch_evalexpr(ch, #{method: 'rpc-resp-incorrect-id',
+        \ params: {'a': [1, 2]}})
+  call assert_false(empty(resp))
+  call assert_equal(#{id: 4, jsonrpc: '2.0',
+        \ result: 'rpc-resp-incorrect-id-4'}, resp)
+  call assert_equal([#{jsonrpc: '2.0', result: 'rpc-resp-incorrect-id-1'},
+        \ #{jsonrpc: '2.0', result: 'rpc-resp-incorrect-id-2'},
+        \ #{jsonrpc: '2.0', id: 1, result: 'rpc-resp-incorrect-id-3'}],
+        \ g:lspNotif)
+
+  " simple notification test
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'simple-notif', params: [#{a: 10, b: []}]})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result: 'simple-notif'}], g:lspNotif)
+
+  " multiple notifications test
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'multi-notif', params: [#{a: {}, b: {}}]})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result: 'multi-notif1'},
+        \ #{jsonrpc: '2.0', result: 'multi-notif2'}], g:lspNotif)
+
+  " Test for sending a message with an identifier.
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'msg-with-id', id: 93, params: #{s: 'str'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', id: 93, result: 'msg-with-id'}],
+        \ g:lspNotif)
+
+  " Test for setting the 'id' value in a request message
+  let resp = ch_evalexpr(ch, #{method: 'ping', id: 1, params: {}})
+  call assert_equal(#{id: 8, jsonrpc: '2.0', result: 'alive'}, resp)
+
+  " Test for using a one time callback function to process a response
+  let g:lspOtMsgs = []
+  call ch_sendexpr(ch, #{method: 'msg-specifc-cb', params: {}},
+        \ #{callback: 'LspOtCb'})
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{id: 9, jsonrpc: '2.0', result: 'msg-specifc-cb'}],
+        \ g:lspOtMsgs)
+
+  " Test for generating a request message from the other end (server)
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'server-req', params: #{}})
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([{'id': 201, 'jsonrpc': '2.0',
+        \ 'result': {'method': 'checkhealth', 'params': {'a': 20}}}],
+        \ g:lspNotif)
+
+  " Test for sending a message without an id
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'echo', params: #{s: 'msg-without-id'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result:
+        \ #{method: 'echo', jsonrpc: '2.0', params: #{s: 'msg-without-id'}}}],
+        \ g:lspNotif)
+
+  " Test for sending a notification message with an id
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'echo', id: 110, params: #{s: 'msg-with-id'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result:
+        \ #{method: 'echo', jsonrpc: '2.0', id: 110,
+        \ params: #{s: 'msg-with-id'}}}], g:lspNotif)
+
+  " Test for processing the extra fields in the HTTP header
+  let resp = ch_evalexpr(ch, #{method: 'extra-hdr-fields', params: {}})
+  call assert_equal({'id': 14, 'jsonrpc': '2.0', 'result': 'extra-hdr-fields'},
+        \ resp)
+
+  " Test for processing a HTTP header without the Content-Length field
+  let resp = ch_evalexpr(ch, #{method: 'hdr-without-len', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 16, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for processing a HTTP header with wrong length
+  let resp = ch_evalexpr(ch, #{method: 'hdr-with-wrong-len', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 18, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for processing a HTTP header with negative length
+  let resp = ch_evalexpr(ch, #{method: 'hdr-with-negative-len', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 20, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for an empty header
+  let resp = ch_evalexpr(ch, #{method: 'empty-header', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 22, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for an empty payload
+  let resp = ch_evalexpr(ch, #{method: 'empty-payload', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 24, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for invoking an unsupported method
+  let resp = ch_evalexpr(ch, #{method: 'xyz', params: {}}, #{timeout: 200})
+  call assert_equal('', resp)
+
+  " Test for sending a message without a callback function. Notification
+  " message should be dropped but RPC response should not be dropped.
+  call ch_setoptions(ch, #{callback: ''})
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'echo', params: #{s: 'no-callback'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([], g:lspNotif)
+  " Restore the callback function
+  call ch_setoptions(ch, #{callback: 'LspCb'})
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'echo', params: #{s: 'no-callback'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result:
+        \ #{method: 'echo', jsonrpc: '2.0', params: #{s: 'no-callback'}}}],
+        \ g:lspNotif)
+
+  " " Test for sending a raw message
+  " let g:lspNotif = []
+  " let s = "Content-Length: 62\r\n"
+  " let s ..= "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+  " let s ..= "\r\n"
+  " let s ..= '{"method":"echo","jsonrpc":"2.0","params":{"m":"raw-message"}}'
+  " call ch_sendraw(ch, s)
+  " call ch_evalexpr(ch, #{method: 'ping'})
+  " call assert_equal([{'jsonrpc': '2.0',
+  "       \ 'result': {'method': 'echo', 'jsonrpc': '2.0',
+  "       \ 'params': {'m': 'raw-message'}}}], g:lspNotif)
+
+  " Invalid arguments to ch_evalexpr() and ch_sendexpr()
+  call assert_fails('call ch_sendexpr(ch, #{method: "cookie", id: "cookie"})',
+        \ 'E475:')
+  call assert_fails('call ch_evalexpr(ch, #{method: "ping", id: [{}]})', 'E475:')
+  call assert_fails('call ch_evalexpr(ch, [1, 2, 3])', 'E1206:')
+  call assert_fails('call ch_sendexpr(ch, "abc")', 'E1206:')
+  call assert_fails('call ch_evalexpr(ch, #{method: "ping"}, #{callback: "LspOtCb"})', 'E917:')
+  " call ch_logfile('', 'w')
+endfunc
+
+func Test_channel_lsp_mode()
+  call RunServer('test_channel_lsp.py', 'LspTests', [])
+endfunc
 
 " vim: shiftwidth=2 sts=2 expandtab
diff --git a/src/testdir/test_channel_lsp.py b/src/testdir/test_channel_lsp.py
new file mode 100644
index 0000000000..530258d844
--- /dev/null
+++ b/src/testdir/test_channel_lsp.py
@@ -0,0 +1,299 @@
+#!/usr/bin/env python
+#
+# Server that will accept connections from a Vim channel.
+# Used by test_channel.vim to test LSP functionality.
+#
+# This requires Python 2.6 or later.
+
+from __future__ import print_function
+import json
+import socket
+import sys
+import time
+import threading
+
+try:
+    # Python 3
+    import socketserver
+except ImportError:
+    # Python 2
+    import SocketServer as socketserver
+
+class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
+
+    def setup(self):
+        self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+    def send_lsp_msg(self, msgid, resp_dict):
+        v = {'jsonrpc': '2.0', 'result': resp_dict}
+        if msgid != -1:
+            v['id'] = msgid
+        s = json.dumps(v)
+        resp = "Content-Length: " + str(len(s)) + "\r\n"
+        resp += "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "\r\n"
+        resp += s
+        if self.debug:
+            with open("Xlspdebug.log", "a") as myfile:
+                myfile.write("\n=> send\n" + resp)
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_wrong_payload(self):
+        v = 'wrong-payload'
+        s = json.dumps(v)
+        resp = "Content-Length: " + str(len(s)) + "\r\n"
+        resp += "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_empty_header(self, msgid, resp_dict):
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_empty_payload(self):
+        resp = "Content-Length: 0\r\n"
+        resp += "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "\r\n"
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_extra_hdr_fields(self, msgid, resp_dict):
+        # test for sending extra fields in the http header
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "Host: abc.vim.org\r\n"
+        resp += "User-Agent: Python\r\n"
+        resp += "Accept-Language: en-US,en\r\n"
+        resp += "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "Content-Length: " + str(len(s)) + "\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_hdr_without_len(self, msgid, resp_dict):
+        # test for sending the http header without length
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_hdr_with_wrong_len(self, msgid, resp_dict):
+        # test for sending the http header with wrong length
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "Content-Length: 1000\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_hdr_with_negative_len(self, msgid, resp_dict):
+        # test for sending the http header with negative length
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "Content-Length: -1\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def do_ping(self, payload):
+        time.sleep(0.2)
+        self.send_lsp_msg(payload['id'], 'alive')
+
+    def do_echo(self, payload):
+        self.send_lsp_msg(-1, payload)
+
+    def do_simple_rpc(self, payload):
+        # test for a simple RPC request
+        self.send_lsp_msg(payload['id'], 'simple-rpc')
+
+    def do_rpc_with_notif(self, payload):
+        # test for sending a notification before replying to a request message
+        self.send_lsp_msg(-1, 'rpc-with-notif-notif')
+        # sleep for some time to make sure the notification is delivered
+        time.sleep(0.2)
+        self.send_lsp_msg(payload['id'], 'rpc-with-notif-resp')
+
+    def do_wrong_payload(self, payload):
+        # test for sending a non dict payload
+        self.send_wrong_payload()
+        time.sleep(0.2)
+        self.send_lsp_msg(-1, 'wrong-payload')
+
+    def do_rpc_resp_incorrect_id(self, payload):
+        self.send_lsp_msg(-1, 'rpc-resp-incorrect-id-1')
+        self.send_lsp_msg(-1, 'rpc-resp-incorrect-id-2')
+        self.send_lsp_msg(1, 'rpc-resp-incorrect-id-3')
+        time.sleep(0.2)
+        self.send_lsp_msg(payload['id'], 'rpc-resp-incorrect-id-4')
+
+    def do_simple_notif(self, payload):
+        # notification message test
+        self.send_lsp_msg(-1, 'simple-notif')
+
+    def do_multi_notif(self, payload):
+        # send multiple notifications
+        self.send_lsp_msg(-1, 'multi-notif1')
+        self.send_lsp_msg(-1, 'multi-notif2')
+
+    def do_msg_with_id(self, payload):
+        self.send_lsp_msg(payload['id'], 'msg-with-id')
+
+    def do_msg_specific_cb(self, payload):
+        self.send_lsp_msg(payload['id'], 'msg-specifc-cb')
+
+    def do_server_req(self, payload):
+        self.send_lsp_msg(201, {'method': 'checkhealth', 'params': {'a': 20}})
+
+    def do_extra_hdr_fields(self, payload):
+        self.send_extra_hdr_fields(payload['id'], 'extra-hdr-fields')
+
+    def do_hdr_without_len(self, payload):
+        self.send_hdr_without_len(payload['id'], 'hdr-without-len')
+
+    def do_hdr_with_wrong_len(self, payload):
+        self.send_hdr_with_wrong_len(payload['id'], 'hdr-with-wrong-len')
+
+    def do_hdr_with_negative_len(self, payload):
+        self.send_hdr_with_negative_len(payload['id'], 'hdr-with-negative-len')
+
+    def do_empty_header(self, payload):
+        self.send_empty_header(payload['id'], 'empty-header')
+
+    def do_empty_payload(self, payload):
+        self.send_empty_payload()
+
+    def process_msg(self, msg):
+        try:
+            decoded = json.loads(msg)
+            print("Decoded:")
+            print(str(decoded))
+            if 'method' in decoded:
+                test_map = {
+                        'ping': self.do_ping,
+                        'echo': self.do_echo,
+                        'simple-rpc': self.do_simple_rpc,
+                        'rpc-with-notif': self.do_rpc_with_notif,
+                        'wrong-payload': self.do_wrong_payload,
+                        'rpc-resp-incorrect-id': self.do_rpc_resp_incorrect_id,
+                        'simple-notif': self.do_simple_notif,
+                        'multi-notif': self.do_multi_notif,
+                        'msg-with-id': self.do_msg_with_id,
+                        'msg-specifc-cb': self.do_msg_specific_cb,
+                        'server-req': self.do_server_req,
+                        'extra-hdr-fields': self.do_extra_hdr_fields,
+                        'hdr-without-len': self.do_hdr_without_len,
+                        'hdr-with-wrong-len': self.do_hdr_with_wrong_len,
+                        'hdr-with-negative-len': self.do_hdr_with_negative_len,
+                        'empty-header': self.do_empty_header,
+                        'empty-payload': self.do_empty_payload
+                        }
+                if decoded['method'] in test_map:
+                    test_map[decoded['method']](decoded)
+                else:
+                    print("Error: Unsupported method: " + decoded['method'])
+            else:
+                print("Error: 'method' field is not found")
+
+        except ValueError:
+            print("json decoding failed")
+
+    def process_msgs(self, msgbuf):
+        while True:
+            sidx = msgbuf.find('Content-Length: ')
+            if sidx == -1:
+                return msgbuf
+            sidx += 16
+            eidx = msgbuf.find('\r\n')
+            if eidx == -1:
+                return msgbuf
+            msglen = int(msgbuf[sidx:eidx])
+
+            hdrend = msgbuf.find('\r\n\r\n')
+            if hdrend == -1:
+                return msgbuf
+
+            # Remove the header
+            msgbuf = msgbuf[hdrend + 4:]
+            payload = msgbuf[:msglen]
+
+            self.process_msg(payload)
+
+            # Remove the processed message
+            msgbuf = msgbuf[msglen:]
+
+    def handle(self):
+        print("=== socket opened ===")
+        self.debug = False
+        msgbuf = ''
+        while True:
+            try:
+                received = self.request.recv(4096).decode('utf-8')
+            except socket.error:
+                print("=== socket error ===")
+                break
+            except IOError:
+                print("=== socket closed ===")
+                break
+            if received == '':
+                print("=== socket closed ===")
+                break
+            print("\nReceived:\n{0}".format(received))
+
+            # Write the received lines into the file for debugging
+            if self.debug:
+                with open("Xlspdebug.log", "a") as myfile:
+                    myfile.write("\n<= recv\n" + received)
+
+            # Can receive more than one line in a response or a partial line.
+            # Accumulate all the received characters and process one line at
+            # a time.
+            msgbuf += received
+            msgbuf = self.process_msgs(msgbuf)
+
+class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+    pass
+
+def writePortInFile(port):
+    # Write the port number in Xportnr, so that the test knows it.
+    f = open("Xportnr", "w")
+    f.write("{0}".format(port))
+    f.close()
+
+def main(host, port, server_class=ThreadedTCPServer):
+    # Wait half a second before opening the port to test waittime in ch_open().
+    # We do want to get the port number, get that first.  We cannot open the
+    # socket, guess a port is free.
+    if len(sys.argv) >= 2 and sys.argv[1] == 'delay':
+        port = 13684
+        writePortInFile(port)
+
+        print("Wait for it...")
+        time.sleep(0.5)
+
+    server = server_class((host, port), ThreadedTCPRequestHandler)
+    ip, port = server.server_address[0:2]
+
+    # Start a thread with the server.  That thread will then start a new thread
+    # for each connection.
+    server_thread = threading.Thread(target=server.serve_forever)
+    server_thread.start()
+
+    writePortInFile(port)
+
+    print("Listening on port {0}".format(port))
+
+    # Main thread terminates, but the server continues running
+    # until server.shutdown() is called.
+    try:
+        while server_thread.is_alive():
+            server_thread.join(1)
+    except (KeyboardInterrupt, SystemExit):
+        server.shutdown()
+
+if __name__ == "__main__":
+    main("localhost", 0)