Add -I / --online-input for huge top-level arrays

4 files changed, 31 insertions, 11 deletions

diff --git a/jv.h b/jv.h
index 1362acca..92d0e058 100644
--- a/jv.h
+++ b/jv.h
@@ -130,6 +130,9 @@ void jv_nomem_handler(jv_nomem_handler_f, void *);
 
 jv jv_load_file(const char *, int);
 
+typedef enum {
+  JV_PARSE_EXPLODE_TOPLEVEL_ARRAY = 1
+} jv_parser_flags;
 struct jv_parser;
 struct jv_parser* jv_parser_new();
 void jv_parser_set_buf(struct jv_parser*, const char*, int, int);
diff --git a/jv_file.c b/jv_file.c
index 54ed36be..a6331607 100644
--- a/jv_file.c
+++ b/jv_file.c
@@ -18,7 +18,7 @@ jv jv_load_file(const char* filename, int raw) {
     data = jv_string("");
   } else {
     data = jv_array();
-    parser = jv_parser_new();
+    parser = jv_parser_new(0);
   }
   while (!feof(file) && !ferror(file)) {
     char buf[4096];
diff --git a/jv_parse.c b/jv_parse.c
index cc1e7b96..4667f1f9 100644
--- a/jv_parse.c
+++ b/jv_parse.c
@@ -28,6 +28,8 @@ struct jv_parser {
   int stackpos;
   int stacklen;
   jv next;
+
+  jv_parser_flags flags;
   
   char* tokenbuf;
   int tokenpos;
@@ -45,7 +47,8 @@ struct jv_parser {
 };
 
 
-static void parser_init(struct jv_parser* p) {
+static void parser_init(struct jv_parser* p, jv_parser_flags flags) {
+  p->flags = flags;
   p->stack = 0;
   p->stacklen = p->stackpos = 0;
   p->next = jv_invalid();
@@ -110,6 +113,9 @@ static pfunc token(struct jv_parser* p, char ch) {
     break;
 
   case ',':
+    if (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY) &&
+        jv_get_kind(p->stack[0]) == JV_KIND_ARRAY)
+      return 0;
     if (!jv_is_valid(p->next))
       return "Expected value before ','";
     if (p->stackpos == 0)
@@ -133,16 +139,22 @@ static pfunc token(struct jv_parser* p, char ch) {
     if (p->stackpos == 0 || jv_get_kind(p->stack[p->stackpos-1]) != JV_KIND_ARRAY)
       return "Unmatched ']'";
     if (jv_is_valid(p->next)) {
-      p->stack[p->stackpos-1] = jv_array_append(p->stack[p->stackpos-1], p->next);
-      p->next = jv_invalid();
+      if (p->stackpos != 1 || !(p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY)) {
+        p->stack[p->stackpos-1] = jv_array_append(p->stack[p->stackpos-1], p->next);
+        p->next = jv_invalid();
+      }
     } else {
       if (jv_array_length(jv_copy(p->stack[p->stackpos-1])) != 0) {
         // this case hits on input like [1,2,3,]
         return "Expected another array element";
       }
     }
-    jv_free(p->next);
-    p->next = p->stack[--p->stackpos];
+    if (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY)) {
+      jv_free(p->stack[--p->stackpos]);
+    } else {
+      jv_free(p->next);
+      p->next = p->stack[--p->stackpos];
+    }
     break;
 
   case '}':
@@ -315,7 +327,9 @@ static chclass classify(char c) {
 static const presult OK = "output produced";
 
 static int check_done(struct jv_parser* p, jv* out) {
-  if (p->stackpos == 0 && jv_is_valid(p->next)) {
+  if ((p->stackpos == 0 && jv_is_valid(p->next)) ||
+      (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY) &&
+       jv_get_kind(p->stack[0]) == JV_KIND_ARRAY && jv_is_valid(p->next))) {
     *out = p->next;
     p->next = jv_invalid();
     return 1;
@@ -370,9 +384,9 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
   return answer;
 }
 
-struct jv_parser* jv_parser_new() {
+struct jv_parser* jv_parser_new(jv_parser_flags flags) {
   struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser));
-  parser_init(p);
+  parser_init(p, flags);
   return p;
 }
 
@@ -444,7 +458,7 @@ jv jv_parser_next(struct jv_parser* p) {
 
 jv jv_parse_sized(const char* string, int length) {
   struct jv_parser parser;
-  parser_init(&parser);
+  parser_init(&parser, 0);
   jv_parser_set_buf(&parser, string, length, 0);
   jv value = jv_parser_next(&parser);
   if (jv_is_valid(value)) {
diff --git a/main.c b/main.c
index a7159219..c8da96a2 100644
--- a/main.c
+++ b/main.c
@@ -159,6 +159,7 @@ int main(int argc, char* argv[]) {
   ninput_files = 0;
   int further_args_are_files = 0;
   int jq_flags = 0;
+  jv_parser_flags parser_flags = 0;
   jv program_arguments = jv_array();
   for (int i=1; i<argc; i++) {
     if (further_args_are_files) {
@@ -196,6 +197,8 @@ int main(int argc, char* argv[]) {
       options |= FROM_FILE;
     } else if (isoption(argv[i], 'e', "exit-status")) {
       options |= EXIT_STATUS;
+    } else if (isoption(argv[i], 'I', "online-input")) {
+      parser_flags = JV_PARSE_EXPLODE_TOPLEVEL_ARRAY;
     } else if (isoption(argv[i], 0, "arg")) {
       if (i >= argc - 2) {
         fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname);
@@ -279,7 +282,7 @@ int main(int argc, char* argv[]) {
         slurped = jv_array();
       }
     }
-    struct jv_parser* parser = jv_parser_new();
+    struct jv_parser* parser = jv_parser_new(parser_flags);
     char buf[4096];
     while (read_more(buf, sizeof(buf))) {
       if (options & RAW_INPUT) {