merging upstream stedolan changes

1 files changed, 453 insertions, 236 deletions

diff --git a/builtin.c b/builtin.c
index bbda554d..b09bd961 100644
--- a/builtin.c
+++ b/builtin.c
@@ -1,51 +1,89 @@
 #include <string.h>
 #include "builtin.h"
+#include "bytecode.h"
 #include "compile.h"
 #include "jq_parser.h"
 #include "locfile.h"
+#include "jv_aux.h"
+#include "jv_unicode.h"
+
+
+
+typedef jv (*func_1)(jv);
+typedef jv (*func_2)(jv,jv);
+typedef jv (*func_3)(jv,jv,jv);
+typedef jv (*func_4)(jv,jv,jv,jv);
+typedef jv (*func_5)(jv,jv,jv,jv,jv);
+jv cfunction_invoke(struct cfunction* function, jv input[]) {
+  switch (function->nargs) {
+  case 1: return ((func_1)function->fptr)(input[0]);
+  case 2: return ((func_2)function->fptr)(input[0], input[1]);
+  case 3: return ((func_3)function->fptr)(input[0], input[1], input[2]);
+  case 4: return ((func_4)function->fptr)(input[0], input[1], input[2], input[3]);
+  case 5: return ((func_5)function->fptr)(input[0], input[1], input[2], input[3], input[4]);
+  default: return jv_invalid_with_msg(jv_string("Function takes too many arguments"));
+  }
+}
 
-enum {
-  CMP_OP_LESS,
-  CMP_OP_GREATER,
-  CMP_OP_LESSEQ,
-  CMP_OP_GREATEREQ
-} _cmp_op;
+static jv type_error(jv bad, const char* msg) {
+  jv err = jv_invalid_with_msg(jv_string_fmt("%s %s",
+                                             jv_kind_name(jv_get_kind(bad)),
+                                             msg));
+  jv_free(bad);
+  return err;
+}
 
-static void f_plus(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
-  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    output[0] = jv_number(jv_number_value(a) + 
-                          jv_number_value(b));
+static jv type_error2(jv bad1, jv bad2, const char* msg) {
+  jv err = jv_invalid_with_msg(jv_string_fmt("%s and %s %s",
+                                             jv_kind_name(jv_get_kind(bad1)),
+                                             jv_kind_name(jv_get_kind(bad2)),
+                                             msg));
+  jv_free(bad1);
+  jv_free(bad2);
+  return err;
+}
+
+static jv f_plus(jv input, jv a, jv b) {
+  jv_free(input);
+  if (jv_get_kind(a) == JV_KIND_NULL) {
+    jv_free(a);
+    return b;
+  } else if (jv_get_kind(b) == JV_KIND_NULL) {
+    jv_free(b);
+    return a;
+  } else if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
+    return jv_number(jv_number_value(a) + 
+                     jv_number_value(b));
   } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
-    output[0] = jv_string_concat(a, b);
+    return jv_string_concat(a, b);
   } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
-    output[0] = jv_array_concat(a, b);
+    return jv_array_concat(a, b);
   } else if (jv_get_kind(a) == JV_KIND_OBJECT && jv_get_kind(b) == JV_KIND_OBJECT) {
-    output[0] = jv_object_merge(a, b);
+    return jv_object_merge(a, b);
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to add %s and %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
+    return type_error2(a, b, "cannot be added");
+  }
+}
+
+static jv f_negate(jv input) {
+  if (jv_get_kind(input) != JV_KIND_NUMBER) {
+    return type_error(input, "cannot be negated");
   }
+  jv ret = jv_number(-jv_number_value(input));
+  jv_free(input);
+  return ret;
 }
 
-static void f_minus(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
+static jv f_minus(jv input, jv a, jv b) {
+  jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    output[0] = jv_number(jv_number_value(a) - jv_number_value(b));
+    return jv_number(jv_number_value(a) - jv_number_value(b));
   } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
     jv out = jv_array();
-    for (int i=0; i<jv_array_length(jv_copy(a)); i++) {
-      jv x = jv_array_get(jv_copy(a), i);
+    jv_array_foreach(a, i, x) {
       int include = 1;
-      for (int j=0; j<jv_array_length(jv_copy(b)); j++) {
-        if (jv_equal(jv_copy(x), jv_array_get(jv_copy(b), j))) {
+      jv_array_foreach(b, j, y) {
+        if (jv_equal(jv_copy(x), y)) {
           include = 0;
           break;
         }
@@ -56,279 +94,462 @@ static void f_minus(jv input[], jv output[]) {
     }
     jv_free(a);
     jv_free(b);
-    output[0] = out;
+    return out;
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to subtract %s and %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
+    return type_error2(a, b, "cannot be subtracted");
   }
 }
 
-static void f_multiply(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
+static jv f_multiply(jv input, jv a, jv b) {
+  jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    output[0] = jv_number(jv_number_value(a) * jv_number_value(b));
+    return jv_number(jv_number_value(a) * jv_number_value(b));
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to multiply %s and %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
+    return type_error2(a, b, "cannot be multiplied");
   }  
 }
 
-static void f_divide(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
+static jv f_divide(jv input, jv a, jv b) {
+  jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    output[0] = jv_number(jv_number_value(a) / jv_number_value(b));
+    return jv_number(jv_number_value(a) / jv_number_value(b));
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to divide %s by %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
+    return type_error2(a, b, "cannot be divided");
   }  
 }
 
-static void f_add(jv input[], jv output[]) {
-  jv array = input[0];
-  if (jv_get_kind(array) != JV_KIND_ARRAY) {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot add elements of an %s",
-                                                  jv_kind_name(jv_get_kind(array))));
-  } else if (jv_array_length(jv_copy(array)) == 0) {
-    output[0] = jv_null();
-  } else {
-    jv sum = jv_array_get(jv_copy(array), 0);
-    for (int i = 1; i < jv_array_length(jv_copy(array)); i++) {
-      jv x = jv_array_get(jv_copy(array), i);
-      jv add_args[] = {jv_null(), x, sum};
-      f_plus(add_args, &sum);
-    }
-    output[0] = sum;
-  }
-  jv_free(array);
+static jv f_equal(jv input, jv a, jv b) {
+  jv_free(input);
+  return jv_bool(jv_equal(a, b));
 }
 
-static void f_equal(jv input[], jv output[]) {
-  jv_free(input[0]);
-  output[0] = jv_bool(jv_equal(input[2], input[1]));
+static jv f_notequal(jv input, jv a, jv b) {
+  jv_free(input);
+  return jv_bool(!jv_equal(a, b));
 }
 
-static void f_notequal(jv input[], jv output[]) {
-  jv_free(input[0]);
-  output[0] = jv_bool(jv_equal(input[2], input[1]) == 0);
+enum cmp_op {
+  CMP_OP_LESS,
+  CMP_OP_GREATER,
+  CMP_OP_LESSEQ,
+  CMP_OP_GREATEREQ
+};
+
+static jv order_cmp(jv input, jv a, jv b, enum cmp_op op) {
+  jv_free(input);
+  int r = jv_cmp(a, b);
+  return jv_bool((op == CMP_OP_LESS && r < 0) ||
+                 (op == CMP_OP_LESSEQ && r <= 0) ||
+                 (op == CMP_OP_GREATEREQ && r >= 0) ||
+                 (op == CMP_OP_GREATER && r > 0));
 }
 
-static void order_cmp(jv input[], jv output[], int op) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
-  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    double da = jv_number_value(a);
-    double db = jv_number_value(b);
-    output[0] = jv_bool((op == CMP_OP_LESS && da < db) ||
-                        (op == CMP_OP_LESSEQ && da <= db) ||
-                        (op == CMP_OP_GREATEREQ && da >= db) ||
-                        (op == CMP_OP_GREATER && da > db));
-  } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to compare order of %s wrt %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
-  }
+static jv f_less(jv input, jv a, jv b) {
+  return order_cmp(input, a, b, CMP_OP_LESS);
 }
 
-static void f_less(jv input[], jv output[]) {
-  order_cmp(input, output, CMP_OP_LESS);
+static jv f_greater(jv input, jv a, jv b) {
+  return order_cmp(input, a, b, CMP_OP_GREATER);
 }
 
-static void f_greater(jv input[], jv output[]) {
-  order_cmp(input, output, CMP_OP_GREATER);
+static jv f_lesseq(jv input, jv a, jv b) {
+  return order_cmp(input, a, b, CMP_OP_LESSEQ);
 }
 
-static void f_lesseq(jv input[], jv output[]) {
-  order_cmp(input, output, CMP_OP_LESSEQ);
+static jv f_greatereq(jv input, jv a, jv b) {
+  return order_cmp(input, a, b, CMP_OP_GREATEREQ);
 }
 
-static void f_greatereq(jv input[], jv output[]) {
-  order_cmp(input, output, CMP_OP_GREATEREQ);
+static jv f_contains(jv a, jv b) {
+  if (jv_get_kind(a) == jv_get_kind(b)) {
+    return jv_bool(jv_contains(a, b));
+  } else {
+    return type_error2(a, b, "cannot have their containment checked");
+  }
 }
 
-static void f_contains(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
-  jv_kind akind = jv_get_kind(a);
+static jv f_tonumber(jv input) {
+  if (jv_get_kind(input) == JV_KIND_NUMBER) {
+    return input;
+  }
+  if (jv_get_kind(input) == JV_KIND_STRING) {
+    jv parsed = jv_parse(jv_string_value(input));
+    if (!jv_is_valid(parsed) || jv_get_kind(parsed) == JV_KIND_NUMBER) {
+      jv_free(input);
+      return parsed;
+    }
+  }
+  return type_error(input, "cannot be parsed as a number");
+}
 
-  if (akind == jv_get_kind(b)) {
-    output[0] = jv_bool(jv_contains(a, b));
-  } else {  
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Can only check containment of values of the same type."));
-    jv_free(a);
-    jv_free(b);
+static jv f_length(jv input) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY) {
+    return jv_number(jv_array_length(input));
+  } else if (jv_get_kind(input) == JV_KIND_OBJECT) {
+    return jv_number(jv_object_length(input));
+  } else if (jv_get_kind(input) == JV_KIND_STRING) {
+    return jv_number(jv_string_length(input));
+  } else if (jv_get_kind(input) == JV_KIND_NULL) {
+    jv_free(input);
+    return jv_number(0);
+  } else {
+    return type_error(input, "has no length");
   }
 }
 
-static void f_tonumber(jv input[], jv output[]) {
-  if (jv_get_kind(input[0]) == JV_KIND_NUMBER) {
-    output[0] = input[0];
-  } else if (jv_get_kind(input[0]) == JV_KIND_STRING) {
-    jv parsed = jv_parse(jv_string_value(input[0]));
-    if (!jv_is_valid(parsed)) {
-      jv_free(input[0]);
-      output[0] = parsed;
-    } else if (jv_get_kind(parsed) != JV_KIND_NUMBER) {
-      output[0] = jv_invalid_with_msg(jv_string_fmt("'%s' is not a number",
-                                                    jv_string_value(input[0])));
-      jv_free(input[0]);
+static jv f_tostring(jv input) {
+  if (jv_get_kind(input) == JV_KIND_STRING) {
+    return input;
+  } else {
+    return jv_dump_string(input, 0);
+  }
+}
+
+#define CHARS_ALPHANUM "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+
+static jv escape_string(jv input, const char* escapings) {
+
+  assert(jv_get_kind(input) == JV_KIND_STRING);
+  const char* lookup[128] = {0};
+  const char* p = escapings;
+  while (*p) {
+    lookup[(int)*p] = p+1;
+    p++;
+    p += strlen(p);
+    p++;
+  }
+
+  jv ret = jv_string("");
+  const char* i = jv_string_value(input);
+  const char* end = i + jv_string_length(jv_copy(input));
+  const char* cstart;
+  int c = 0;
+  while ((i = jvp_utf8_next((cstart = i), end, &c))) {
+    assert(c != -1);
+    if (c < 128 && lookup[c]) {
+      ret = jv_string_append_str(ret, lookup[c]);
     } else {
-      jv_free(input[0]);
-      output[0] = parsed;
+      ret = jv_string_append_buf(ret, cstart, i - cstart);
+    }
+  }
+  jv_free(input);
+  return ret;
+
+}
+
+static jv f_format(jv input, jv fmt) {
+  if (jv_get_kind(fmt) != JV_KIND_STRING) {
+    jv_free(input);
+    return type_error(fmt, "is not a valid format");
+  }
+  const char* fmt_s = jv_string_value(fmt);
+  if (!strcmp(fmt_s, "json")) {
+    jv_free(fmt);
+    return jv_dump_string(input, 0);
+  } else if (!strcmp(fmt_s, "text")) {
+    jv_free(fmt);
+    return f_tostring(input);
+  } else if (!strcmp(fmt_s, "csv")) {
+    jv_free(fmt);
+    if (jv_get_kind(input) != JV_KIND_ARRAY)
+      return type_error(input, "cannot be csv-formatted, only array");
+    jv line = jv_string("");
+    jv_array_foreach(input, i, x) {
+      if (i) line = jv_string_append_str(line, ",");
+      switch (jv_get_kind(x)) {
+      case JV_KIND_NULL:
+        /* null rendered as empty string */
+        jv_free(x);
+        break;
+      case JV_KIND_TRUE:
+      case JV_KIND_FALSE:
+        line = jv_string_concat(line, jv_dump_string(x, 0));
+        break;
+      case JV_KIND_NUMBER:
+        if (jv_number_value(x) != jv_number_value(x)) {
+          /* NaN, render as empty string */
+          jv_free(x);
+        } else {
+          line = jv_string_concat(line, jv_dump_string(x, 0));
+        }
+        break;
+      case JV_KIND_STRING: {
+        line = jv_string_append_str(line, "\"");
+        line = jv_string_concat(line, escape_string(x, "\"\"\"\0"));
+        line = jv_string_append_str(line, "\"");
+        break;
+      }
+      default:
+        jv_free(input);
+        jv_free(line);
+        return type_error(x, "is not valid in a csv row");
+      }
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "html")) {
+    jv_free(fmt);
+    return escape_string(f_tostring(input), "&&amp;\0<&lt;\0>&gt;\0'&apos;\0\"&quot;\0");
+  } else if (!strcmp(fmt_s, "uri")) {
+    jv_free(fmt);
+    input = f_tostring(input);
+
+    int unreserved[128] = {0};
+    const char* p = CHARS_ALPHANUM "-_.!~*'()";
+    while (*p) unreserved[(int)*p++] = 1;
+
+    jv line = jv_string("");
+    const char* s = jv_string_value(input);
+    for (int i=0; i<jv_string_length(jv_copy(input)); i++) {
+      unsigned ch = (unsigned)(unsigned char)*s;
+      if (ch < 128 && unreserved[ch]) {
+        line = jv_string_append_buf(line, s, 1);
+      } else {
+        line = jv_string_concat(line, jv_string_fmt("%%%02x", ch));
+      }
+      s++;
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "sh")) {
+    jv_free(fmt);
+    if (jv_get_kind(input) != JV_KIND_ARRAY)
+      input = jv_array_set(jv_array(), 0, input);
+    jv line = jv_string("");
+    jv_array_foreach(input, i, x) {
+      if (i) line = jv_string_append_str(line, " ");
+      switch (jv_get_kind(x)) {
+      case JV_KIND_NULL:
+      case JV_KIND_TRUE:
+      case JV_KIND_FALSE:
+      case JV_KIND_NUMBER:
+        line = jv_string_concat(line, jv_dump_string(x, 0));
+        break;
+
+      case JV_KIND_STRING: {
+        line = jv_string_append_str(line, "'");
+        line = jv_string_concat(line, escape_string(x, "''\\''\0"));
+        line = jv_string_append_str(line, "'");
+        break;
+      }
+
+      default:
+        jv_free(input);
+        jv_free(line);
+        return type_error(x, "can not be escaped for shell");
+      }
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "base64")) {
+    jv_free(fmt);
+    input = f_tostring(input);
+    jv line = jv_string("");
+    const char b64[64 + 1] = CHARS_ALPHANUM "+/";
+    const char* data = jv_string_value(input);
+    int len = jv_string_length(jv_copy(input));
+    for (int i=0; i<len; i+=3) {
+      uint32_t code = 0;
+      int n = len - i >= 3 ? 3 : len-i;
+      for (int j=0; j<3; j++) {
+        code <<= 8;
+        code |= j < n ? (unsigned)data[i+j] : 0;
+      }
+      char buf[4];
+      for (int j=0; j<4; j++) {
+        buf[j] = b64[(code >> (18 - j*6)) & 0x3f];
+      }
+      if (n < 3) buf[3] = '=';
+      if (n < 2) buf[2] = '=';
+      line = jv_string_append_buf(line, buf, sizeof(buf));
     }
+    jv_free(input);
+    return line;
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot parse %s as a number",
-                                                  jv_kind_name(jv_get_kind(input[0]))));
-    jv_free(input[0]);
-    return;
+    jv_free(input);
+    return jv_invalid_with_msg(jv_string_concat(fmt, jv_string(" is not a valid format")));
   }
 }
 
-static void f_length(jv input[], jv output[]) {
-  if (jv_get_kind(input[0]) == JV_KIND_ARRAY) {
-    output[0] = jv_number(jv_array_length(input[0]));
-  } else if (jv_get_kind(input[0]) == JV_KIND_OBJECT) {
-    output[0] = jv_number(jv_object_length(input[0]));
-  } else if (jv_get_kind(input[0]) == JV_KIND_STRING) {
-    output[0] = jv_number(jv_string_length(input[0]));
+static jv f_keys(jv input) {
+  if (jv_get_kind(input) == JV_KIND_OBJECT || jv_get_kind(input) == JV_KIND_ARRAY) {
+    return jv_keys(input);
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot get the length of %s",
-                                                  jv_kind_name(jv_get_kind(input[0]))));
-    jv_free(input[0]);
+    return type_error(input, "has no keys");
   }
 }
 
-static void f_tostring(jv input[], jv output[]) {
-  if (jv_get_kind(input[0]) == JV_KIND_STRING) {
-    output[0] = input[0];
+static jv f_sort(jv input){
+  if (jv_get_kind(input) == JV_KIND_ARRAY) {
+    return jv_sort(input, jv_copy(input));
   } else {
-    output[0] = jv_dump_string(input[0], 0);
+    return type_error(input, "cannot be sorted, as it is not an array");
   }
 }
 
-static int string_cmp(const void* pa, const void* pb){
-  const jv* a = pa;
-  const jv* b = pb;
-  int lena = jv_string_length(jv_copy(*a));
-  int lenb = jv_string_length(jv_copy(*b));
-  int minlen = lena < lenb ? lena : lenb;
-  int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen);
-  if (r == 0) r = lena - lenb;
-  return r;
-}
-
-static void f_keys(jv input[], jv output[]) {
-  if (jv_get_kind(input[0]) == JV_KIND_OBJECT) {
-    int nkeys = jv_object_length(jv_copy(input[0]));
-    jv* keys = malloc(sizeof(jv) * nkeys);
-    int kidx = 0;
-    jv_object_foreach(i, input[0]) {
-      keys[kidx++] = jv_object_iter_key(input[0], i);
-    }
-    qsort(keys, nkeys, sizeof(jv), string_cmp);
-    output[0] = jv_array_sized(nkeys);
-    for (int i = 0; i<nkeys; i++) {
-      output[0] = jv_array_append(output[0], keys[i]);
-    }
-    free(keys);
-    jv_free(input[0]);
-  } else if (jv_get_kind(input[0]) == JV_KIND_ARRAY) {
-    int n = jv_array_length(input[0]);
-    output[0] = jv_array();
-    for (int i=0; i<n; i++){
-      output[0] = jv_array_set(output[0], i, jv_number(i));
-    }
+static jv f_sort_by_impl(jv input, jv keys) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY && 
+      jv_get_kind(keys) == JV_KIND_ARRAY &&
+      jv_array_length(jv_copy(input)) == jv_array_length(jv_copy(keys))) {
+    return jv_sort(input, keys);
+  } else {
+    return type_error2(input, keys, "cannot be sorted, as they are not both arrays");
+  }
+}
+
+static jv f_group_by_impl(jv input, jv keys) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY && 
+      jv_get_kind(keys) == JV_KIND_ARRAY &&
+      jv_array_length(jv_copy(input)) == jv_array_length(jv_copy(keys))) {
+    return jv_group(input, keys);
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("'keys' only supports object, not %s",
-                                                  jv_kind_name(jv_get_kind(input[0]))));
-    jv_free(input[0]);
+    return type_error2(input, keys, "cannot be sorted, as they are not both arrays");
   }
 }
 
-static void f_type(jv input[], jv output[]) {
-  output[0] = jv_string(jv_kind_name(jv_get_kind(input[0])));
-  jv_free(input[0]);
+static jv minmax_by(jv values, jv keys, int is_min) {
+  if (jv_get_kind(values) != JV_KIND_ARRAY)
+    return type_error2(values, keys, "cannot be iterated over");
+  if (jv_get_kind(keys) != JV_KIND_ARRAY)
+    return type_error2(values, keys, "cannot be iterated over");
+  if (jv_array_length(jv_copy(values)) != jv_array_length(jv_copy(keys)))
+    return type_error2(values, keys, "have wrong length");
+
+  if (jv_array_length(jv_copy(values)) == 0) {
+    jv_free(values);
+    jv_free(keys);
+    return jv_null();
+  }
+  jv ret = jv_array_get(jv_copy(values), 0);
+  jv retkey = jv_array_get(jv_copy(keys), 0);
+  for (int i=1; i<jv_array_length(jv_copy(values)); i++) {
+    jv item = jv_array_get(jv_copy(keys), i);
+    int cmp = jv_cmp(jv_copy(item), jv_copy(retkey));
+    if ((cmp < 0) == (is_min == 1)) {
+      jv_free(retkey);
+      retkey = item;
+      jv_free(ret);
+      ret = jv_array_get(jv_copy(values), i);
+    } else {
+      jv_free(item);
+    }
+  }
+  jv_free(values);
+  jv_free(keys);
+  jv_free(retkey);
+  return ret;
 }
 
-static block j_empty() {
-  return gen_op_block_defn(CLOSURE_CREATE, "empty", gen_op_simple(BACKTRACK));
+static jv f_min(jv x) {
+  return minmax_by(x, jv_copy(x), 1);
 }
 
-static block j_false() {
-  return gen_op_block_defn(CLOSURE_CREATE, "false",
-                           block_join(gen_op_const(LOADK, jv_false()),
-                                      gen_op_simple(RET)));
+static jv f_max(jv x) {
+  return minmax_by(x, jv_copy(x), 0);
 }
 
-static block j_true() {
-  return gen_op_block_defn(CLOSURE_CREATE, "true",
-                           block_join(gen_op_const(LOADK, jv_true()),
-                                      gen_op_simple(RET)));
+static jv f_min_by_impl(jv x, jv y) {
+  return minmax_by(x, y, 1);
 }
 
-static block j_null() {
-  return gen_op_block_defn(CLOSURE_CREATE, "null",
-                           block_join(gen_op_const(LOADK, jv_null()),
-                                      gen_op_simple(RET)));
+static jv f_max_by_impl(jv x, jv y) {
+  return minmax_by(x, y, 0);
 }
 
-static block j_not() {
-  return gen_op_block_defn(CLOSURE_CREATE, "not",
-                           block_join(gen_condbranch(gen_op_const(LOADK, jv_false()),
-                                                     gen_op_const(LOADK, jv_true())),
-                                      gen_op_simple(RET)));
+
+static jv f_type(jv input) {
+  jv out = jv_string(jv_kind_name(jv_get_kind(input)));
+  jv_free(input);
+  return out;
+}
+
+static jv f_error(jv input, jv msg) {
+  jv_free(input);
+  msg = f_tostring(msg);
+  return jv_invalid_with_msg(msg);
 }
 
 static struct cfunction function_list[] = {
-  {f_plus, "_plus", CALL_BUILTIN_3_1},
-  {f_minus, "_minus", CALL_BUILTIN_3_1},
-  {f_multiply, "_multiply", CALL_BUILTIN_3_1},
-  {f_divide, "_divide", CALL_BUILTIN_3_1},
-  {f_tonumber, "tonumber", CALL_BUILTIN_1_1},
-  {f_tostring, "tostring", CALL_BUILTIN_1_1},
-  {f_keys, "keys", CALL_BUILTIN_1_1},
-  {f_equal, "_equal", CALL_BUILTIN_3_1},
-  {f_notequal, "_notequal", CALL_BUILTIN_3_1},
-  {f_less, "_less", CALL_BUILTIN_3_1},
-  {f_greater, "_greater", CALL_BUILTIN_3_1},
-  {f_lesseq, "_lesseq", CALL_BUILTIN_3_1},
-  {f_greatereq, "_greatereq", CALL_BUILTIN_3_1},
-  {f_contains, "_contains", CALL_BUILTIN_3_1},
-  {f_length, "length", CALL_BUILTIN_1_1},
-  {f_type, "type", CALL_BUILTIN_1_1},
-  {f_add, "add", CALL_BUILTIN_1_1},
+  {(cfunction_ptr)f_plus, "_plus", 3},
+  {(cfunction_ptr)f_negate, "_negate", 1},
+  {(cfunction_ptr)f_minus, "_minus", 3},
+  {(cfunction_ptr)f_multiply, "_multiply", 3},
+  {(cfunction_ptr)f_divide, "_divide", 3},
+  {(cfunction_ptr)f_tonumber, "tonumber", 1},
+  {(cfunction_ptr)f_tostring, "tostring", 1},
+  {(cfunction_ptr)f_keys, "keys", 1},
+  {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking
+  {(cfunction_ptr)jv_getpath, "getpath", 2},
+  {(cfunction_ptr)jv_delpaths, "delpaths", 2},
+  {(cfunction_ptr)f_equal, "_equal", 3},
+  {(cfunction_ptr)f_notequal, "_notequal", 3},
+  {(cfunction_ptr)f_less, "_less", 3},
+  {(cfunction_ptr)f_greater, "_greater", 3},
+  {(cfunction_ptr)f_lesseq, "_lesseq", 3},
+  {(cfunction_ptr)f_greatereq, "_greatereq", 3},
+  {(cfunction_ptr)f_contains, "contains", 2},
+  {(cfunction_ptr)f_length, "length", 1},
+  {(cfunction_ptr)f_type, "type", 1},
+  {(cfunction_ptr)f_sort, "sort", 1},
+  {(cfunction_ptr)f_sort_by_impl, "_sort_by_impl", 2},
+  {(cfunction_ptr)f_group_by_impl, "_group_by_impl", 2},
+  {(cfunction_ptr)f_min, "min", 1},
+  {(cfunction_ptr)f_max, "max", 1},
+  {(cfunction_ptr)f_min_by_impl, "_min_by_impl", 2},
+  {(cfunction_ptr)f_max_by_impl, "_max_by_impl", 2},
+  {(cfunction_ptr)f_error, "error", 2},
+  {(cfunction_ptr)f_format, "format", 2},
 };
 
-static struct symbol_table cbuiltins = {function_list, sizeof(function_list)/sizeof(function_list[0])};
+static struct symbol_table cbuiltins = 
+  {function_list, sizeof(function_list)/sizeof(function_list[0])};
 
-typedef block (*bytecoded_builtin)();
-static bytecoded_builtin bytecoded_builtins[] = {
-  j_empty,
-  j_false,
-  j_true,
-  j_null,
-  j_not,
-};
+struct bytecoded_builtin { const char* name; block code; };
+static block bind_bytecoded_builtins(block b) {
+  block builtins = gen_noop();
+  {
+    struct bytecoded_builtin builtin_defs[] = {
+      {"empty", gen_op_simple(BACKTRACK)},
+      {"false", gen_const(jv_false())},
+      {"true", gen_const(jv_true())},
+      {"null", gen_const(jv_null())},
+      {"not", gen_condbranch(gen_const(jv_false()),
+                             gen_const(jv_true()))}
+    };
+    for (unsigned i=0; i<sizeof(builtin_defs)/sizeof(builtin_defs[0]); i++) {
+      builtins = BLOCK(builtins, gen_function(builtin_defs[i].name, gen_noop(),
+                                              builtin_defs[i].code));
+    }
+  }
+  {
+    struct bytecoded_builtin builtin_def_1arg[] = {
+      {"path", BLOCK(gen_op_simple(DUP), 
+                     gen_call("arg", gen_noop()),
+                     gen_op_simple(GETPATH))},
+    };
+    for (unsigned i=0; i<sizeof(builtin_def_1arg)/sizeof(builtin_def_1arg[0]); i++) {
+      builtins = BLOCK(builtins, gen_function(builtin_def_1arg[i].name,
+                                              gen_op_block_unbound(CLOSURE_PARAM, "arg"),
+                                              builtin_def_1arg[i].code));
+    }
+  }
+  
+  return block_bind(builtins, b, OP_IS_CALL_PSEUDO);
+}
 
 static const char* jq_builtins[] = {
   "def map(f): [.[] | f];",
   "def select(f): if f then . else empty end;",
+  "def sort_by(f): _sort_by_impl(map([f]));",
+  "def group_by(f): _group_by_impl(map([f]));",
+  "def unique: group_by(.) | map(.[0]);",
+  "def max_by(f): _max_by_impl(map([f]));",
+  "def min_by(f): _min_by_impl(map([f]));",
+  "def add: fold null as $sum (.[] | $sum + .);",
+  "def del(f): delpaths([path(f)]);",
+  "def _assign(paths; value): value as $v | fold . as $obj (path(paths) as $p | $obj | setpath($p; $v));",
+  "def _modify(paths; update): fold . as $obj (path(paths) as $p | $obj | setpath($p; getpath($p) | update));",
+  "def recurse(f): ., (f | select(. != null) | recurse(f));",
 };
 
 
@@ -342,10 +563,6 @@ block builtins_bind(block b) {
     b = block_bind(funcs, b, OP_IS_CALL_PSEUDO);
     locfile_free(&src);
   }
-  block builtins = gen_noop();
-  for (unsigned i=0; i<sizeof(bytecoded_builtins)/sizeof(bytecoded_builtins[0]); i++) {
-    block_append(&builtins, bytecoded_builtins[i]());
-  }
-  b = block_bind(builtins, b, OP_IS_CALL_PSEUDO);
+  b = bind_bytecoded_builtins(b);
   return gen_cbinding(&cbuiltins, b);
 }