Fix multi-byte characters in utils::extract_token_quoted()

1 files changed, 9 insertions, 0 deletions

diff --git a/test/utils.cpp b/test/utils.cpp
index 23efe9c9..d2038cc0 100644
--- a/test/utils.cpp
+++ b/test/utils.cpp
@@ -345,6 +345,15 @@ TEST_CASE("extract_token_quoted() processes escape sequences within quoted strin
 	REQUIRE(str == " remainder");
 }
 
+TEST_CASE("extract_token_quoted() works with Unicode strings too", "[utils]")
+{
+	std::string str = R"("привет мир" Юникода)";
+	const auto token = utils::extract_token_quoted(str);
+	REQUIRE(token.has_value());
+	REQUIRE(token.value() == "привет мир");
+	REQUIRE(str == " Юникода");
+}
+
 TEST_CASE("tokenize_nl() split a string into delimiters and fields", "[utils]")
 {
 	std::vector<std::string> tokens;