Improvements to CSV highlighting

8 files changed, 203 insertions, 50 deletions

diff --git a/.gitmodules b/.gitmodules
index 4b6bc34e..12beb70c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -61,9 +61,6 @@
 [submodule "assets/syntaxes/Puppet"]
 	path = assets/syntaxes/02_Extra/Puppet
 	url = https://github.com/russCloak/SublimePuppet
-[submodule "assets/syntaxes/CSV"]
-	path = assets/syntaxes/02_Extra/CSV
-	url = https://github.com/wadetb/Sublime-Text-Advanced-CSV
 [submodule "assets/themes/onehalf"]
 	path = assets/themes/onehalf
 	url = https://github.com/sonph/onehalf
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e438d94f..d3341148 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,8 @@
 
 ## Syntaxes
 
+- Replaced "Advanced CSV" with a custom CSV syntax definition written especially for `bat`; see #1574 (@keith-hall)
+
 ## New themes
 
 ## `bat` as a library
diff --git a/assets/patches/Monokai-Extended.tmTheme.patch b/assets/patches/Monokai-Extended.tmTheme.patch
index 24573a7b..16624d5f 100644
--- a/assets/patches/Monokai-Extended.tmTheme.patch
+++ b/assets/patches/Monokai-Extended.tmTheme.patch
@@ -7,7 +7,7 @@ index 9c2aa3e..180cbbf 100644
        <string>JSON String</string>
        <key>scope</key>
 -      <string>meta.structure.dictionary.json string.quoted.double.json</string>
-+      <string>meta.mapping.key.json string.quoted.double.json</string>
++      <string>meta.mapping.key.json string.quoted.double.json, punctuation.separator.sequence.csv</string>
        <key>settings</key>
        <dict>
          <key>foreground</key>
diff --git a/assets/syntaxes/02_Extra/CSV b/assets/syntaxes/02_Extra/CSV
deleted file mode 160000
-Subproject 4786d037a761bc45e516f6b0624a839919ec6d0
diff --git a/assets/syntaxes/02_Extra/CSV.sublime-syntax b/assets/syntaxes/02_Extra/CSV.sublime-syntax
index 1bbc5b97..10e5c7cc 100644
--- a/assets/syntaxes/02_Extra/CSV.sublime-syntax
+++ b/assets/syntaxes/02_Extra/CSV.sublime-syntax
@@ -1,46 +1,84 @@
 %YAML 1.2
 ---
-# http://www.sublimetext.com/docs/3/syntax.html
-name: Advanced CSV
+# See http://www.sublimetext.com/docs/3/syntax.html
+name: Comma Separated Values
 file_extensions:
   - csv
   - tsv
-scope: text.advanced_csv
+scope: text.csv
+variables:
+  field_separator: (?:[,;\t])
+  record_separator: (?:$\n?)
 contexts:
-  main:
-    - match: (\")
-      captures:
-        1: string.quoted.double.advanced_csv
-      push:
-        - meta_scope: meta.quoted.advanced_csv
-        - match: (\")
-          captures:
-            1: string.quoted.double.advanced_csv
-          pop: true
-        - include: main
-    - match: '(\[([+-]?\d*)(\:)?([+-]?\d*)(\,)?([+-]?\d*)(\:)?([+-]?\d*)\])?\s*([<>v^])?\s*(=)'
-      captures:
-        1: keyword.operator.advanced_csv
-        2: constant.numeric.formula.advanced_csv
-        4: constant.numeric.formula.advanced_csv
-        6: constant.numeric.formula.advanced_csv
-        8: constant.numeric.formula.advanced_csv
-        9: keyword.operator.advanced_csv
-        10: keyword.operator.advanced_csv
+  prototype:
+    - match: (?={{record_separator}})
+      pop: true
+  fields:
+    - match: ''
       push:
-        - meta_scope: meta.range.advanced_csv
-        - match: (?=(\")|$)
-          pop: true
-        - include: scope:source.python
-    - match: '(?<=^|,|\s|\")([0-9.eE+-]+)(?=$|,|\s|\")'
-      scope: meta.number.advanced_csv
-      captures:
-        1: constant.numeric.advanced_csv
-    - match: '(?<=^|,|\s|\")([^, \t\"]+)(?=$|,|\s|\")'
-      scope: meta.nonnumber.advanced_csv
-      captures:
-        1: storage.type.advanced_csv
-    - match: (\,)
-      scope: meta.delimiter.advanced_csv
-      captures:
-        1: keyword.operator.advanced_csv
+        - field_or_record_separator
+        - field5
+        - field_or_record_separator
+        - field4
+        - field_or_record_separator
+        - field3
+        - field_or_record_separator
+        - field2
+        - field_or_record_separator
+        - field1
+  main:
+    - meta_include_prototype: false
+    - match: '^'
+      set: fields
+
+  field_or_record_separator:
+    - meta_include_prototype: false
+    - match: '{{record_separator}}'
+      scope: punctuation.terminator.record.csv
+      pop: true
+    - match: '{{field_separator}}'
+      scope: punctuation.separator.sequence.csv
+      pop: true
+
+  field_contents:
+    - match: '"'
+      scope: punctuation.definition.string.begin.csv
+      push: double_quoted_string
+
+    - match: (?={{field_separator}}|{{record_separator}})
+      pop: true
+
+  double_quoted_string:
+    - meta_include_prototype: false
+    - meta_scope: string.quoted.double.csv
+    - match: '""'
+      scope: constant.character.escape.csv
+    - match: '"'
+      scope: punctuation.definition.string.end.csv
+      pop: true
+
+  field1:
+    - match: ''
+      set:
+        - meta_content_scope: meta.field-1.csv support.type
+        - include: field_contents
+  field2:
+    - match: ''
+      set:
+        - meta_content_scope: meta.field-2.csv support.function
+        - include: field_contents
+  field3:
+    - match: ''
+      set:
+        - meta_content_scope: meta.field-3.csv variable.other
+        - include: field_contents
+  field4:
+    - match: ''
+      set:
+        - meta_content_scope: meta.field-4.csv constant.numeric
+        - include: field_contents
+  field5:
+    - match: ''
+      set:
+        - meta_content_scope: meta.field-5.csv keyword.operator
+        - include: field_contents
diff --git a/assets/syntaxes/02_Extra/syntax_test_csv.csv b/assets/syntaxes/02_Extra/syntax_test_csv.csv
new file mode 100644
index 00000000..1d7ae9f9
--- /dev/null
+++ b/assets/syntaxes/02_Extra/syntax_test_csv.csv
@@ -0,0 +1,58 @@
+# SYNTAX TEST "CSV.sublime-syntax"
+Forename,Surname,House No.,Street,City,Postcode
+#^^^^^^^ meta.field-1
+#       ^ punctuation.separator.sequence
+#        ^^^^^^^ meta.field-2
+#               ^ punctuation.separator.sequence
+#                ^^^^^^^^^ meta.field-3
+#                         ^ punctuation.separator.sequence
+#                          ^^^^^^ meta.field-4
+#                                ^ punctuation.separator.sequence
+#                                 ^^^^ meta.field-5
+#                                     ^ punctuation.separator.sequence
+#                                      ^^^^^^^^ meta.field-1
+John,Doe,87,Citiville,TU43 6GH
+#^^^ meta.field-1
+#   ^ punctuation.separator.sequence
+#    ^^^ meta.field-2
+#       ^ punctuation.separator.sequence
+#        ^^ meta.field-3
+#          ^ punctuation.separator.sequence
+#           ^^^^^^^^^ meta.field-4
+#                    ^ punctuation.separator.sequence
+#                     ^^^^^^^^ meta.field-5
+Jane,Doe,22,"""Fun"" Street","The City, County",FA90 1XA
+#^^^ meta.field-1
+#   ^ punctuation.separator.sequence
+#    ^^^ meta.field-2
+#       ^ punctuation.separator.sequence
+#        ^^ meta.field-3
+#          ^ punctuation.separator.sequence
+#           ^^^^^^^^^^^^^^^^ meta.field-4 string.quoted.double
+#            ^^ constant.character.escape
+#                 ^^ constant.character.escape
+#                           ^ punctuation.separator.sequence
+#                            ^ meta.field-5 string.quoted.double punctuation.definition.string.begin
+#                             ^^^^^^^^^^^^^^^^^ meta.field-5 string.quoted.double
+#                                             ^ meta.field-5 string.quoted.double punctuation.definition.string.end
+#                                              ^ punctuation.separator.sequence
+#                                               ^^^^^^^^ meta.field-1
+Fred,Bloggs,"19","Street
+#^^^ meta.field-1
+#   ^ punctuation.separator.sequence
+#    ^^^^^^ meta.field-2
+#          ^ punctuation.separator.sequence
+#            ^^^ meta.field-3 string.quoted.double
+#               ^ punctuation.separator.sequence
+#                ^ meta.field-4 punctuation.definition.string.begin
+#                ^^^^^^^^ meta.field-4 string.quoted.double
+Spanning Multiple ""
+#^^^^^^^^^^^^^^^^^^^^ meta.field-4 string.quoted.double
+#                 ^^ constant.character.escape
+Lines",Citee,P0stc0d3
+#^^^^^ meta.field-4 string.quoted.double
+#    ^ meta.field-4 punctuation.definition.string.end
+#     ^ punctuation.separator.sequence
+#      ^^^^^ meta.field-5
+#           ^ punctuation.separator.sequence
+#            ^^^^^^^^ meta.field-1
diff --git a/assets/syntaxes/02_Extra/syntax_test_tsv.tsv b/assets/syntaxes/02_Extra/syntax_test_tsv.tsv
new file mode 100644
index 00000000..6d005214
--- /dev/null
+++ b/assets/syntaxes/02_Extra/syntax_test_tsv.tsv
@@ -0,0 +1,58 @@
+# SYNTAX TEST "CSV.sublime-syntax"
+Forename	Surname	House No.	Street	City	Postcode
+#^^^^^^^ meta.field-1
+#       ^ punctuation.separator.sequence
+#        ^^^^^^^ meta.field-2
+#               ^ punctuation.separator.sequence
+#                ^^^^^^^^^ meta.field-3
+#                         ^ punctuation.separator.sequence
+#                          ^^^^^^ meta.field-4
+#                                ^ punctuation.separator.sequence
+#                                 ^^^^ meta.field-5
+#                                     ^ punctuation.separator.sequence
+#                                      ^^^^^^^^ meta.field-1
+John	Doe	87	Citiville	TU43 6GH
+#^^^ meta.field-1
+#   ^ punctuation.separator.sequence
+#    ^^^ meta.field-2
+#       ^ punctuation.separator.sequence
+#        ^^ meta.field-3
+#          ^ punctuation.separator.sequence
+#           ^^^^^^^^^ meta.field-4
+#                    ^ punctuation.separator.sequence
+#                     ^^^^^^^^ meta.field-5
+Jane	Doe	22	"""Fun"" Street"	"The City	 County"	FA90 1XA
+#^^^ meta.field-1
+#   ^ punctuation.separator.sequence
+#    ^^^ meta.field-2
+#       ^ punctuation.separator.sequence
+#        ^^ meta.field-3
+#          ^ punctuation.separator.sequence
+#           ^^^^^^^^^^^^^^^^ meta.field-4 string.quoted.double
+#            ^^ constant.character.escape
+#                 ^^ constant.character.escape
+#                           ^ punctuation.separator.sequence
+#                            ^ meta.field-5 string.quoted.double punctuation.definition.string.begin
+#                             ^^^^^^^^^^^^^^^^^ meta.field-5 string.quoted.double
+#                                             ^ meta.field-5 string.quoted.double punctuation.definition.string.end
+#                                              ^ punctuation.separator.sequence
+#                                               ^^^^^^^^ meta.field-1
+Fred	Bloggs	"19"	"Street
+#^^^ meta.field-1
+#   ^ punctuation.separator.sequence
+#    ^^^^^^ meta.field-2
+#          ^ punctuation.separator.sequence
+#            ^^^ meta.field-3 string.quoted.double
+#               ^ punctuation.separator.sequence
+#                ^ meta.field-4 punctuation.definition.string.begin
+#                ^^^^^^^^ meta.field-4 string.quoted.double
+Spanning Multiple ""
+#^^^^^^^^^^^^^^^^^^^^ meta.field-4 string.quoted.double
+#                 ^^ constant.character.escape
+Lines"	Citee	P0stc0d3
+#^^^^^ meta.field-4 string.quoted.double
+#    ^ meta.field-4 punctuation.definition.string.end
+#     ^ punctuation.separator.sequence
+#      ^^^^^ meta.field-5
+#           ^ punctuation.separator.sequence
+#            ^^^^^^^^ meta.field-1
diff --git a/tests/syntax-tests/highlighted/CSV/comma_in_quotes.csv b/tests/syntax-tests/highlighted/CSV/comma_in_quotes.csv
index 2ad5b914..c4015719 100644
--- a/tests/syntax-tests/highlighted/CSV/comma_in_quotes.csv
+++ b/tests/syntax-tests/highlighted/CSV/comma_in_quotes.csv
@@ -1,7 +1,7 @@
-first,last,address,city,zip
-John,Doe,120 any st.,"Anytown, WW",08123
-a,b
-1,"ha 
-""ha"" 
-ha",120 any st.,"Anytown, WW",08123
-3,4,120 any st.,"Anytown, WW",08123
+first,last,address,city,zip
+John,Doe,120 any st.,"Anytown, WW",08123
+a,b
+1,"ha 
+""ha"" 
+ha",120 any st.,"Anytown, WW",08123
+3,4,120 any st.,"Anytown, WW",08123