diff options
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | docs/content/3.manual/manual.yml | 68 | ||||
-rw-r--r-- | execute.c | 10 | ||||
-rw-r--r-- | jq.h | 3 | ||||
-rw-r--r-- | linker.c | 107 | ||||
-rw-r--r-- | main.c | 28 | ||||
-rw-r--r-- | tests/modules/c/c.jq | 13 | ||||
-rw-r--r-- | tests/modules/lib/jq/e/e.jq | 1 | ||||
-rw-r--r-- | tests/modules/lib/jq/f.jq | 1 | ||||
-rwxr-xr-x | tests/run | 15 |
10 files changed, 167 insertions, 80 deletions
@@ -10,6 +10,7 @@ tags jq +!tests/modules/lib/jq/ jq.1 # Something delightfully recursive happens otherwise diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 42c6c98c..cf78c7cd 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -2286,45 +2286,58 @@ sections: body: | jq has a library/module system. Modules are files whose names end - in `.jq`. Modules should start with a `module` directive. - Programs and modules must `import` their dependencies, if any. + in `.jq`. Modules should (but don't have to) start with a + `module` directive. Programs and modules must `import` their + dependencies, if any. - A search path is used to search for modules. Each directory in - the search path is tested twice: once with a directory named "any" - added on the end, and once with the jq version number as a - directory added on the end. + Modules imported by a program are searched for in a default search + path (see below). The `import` directive allows the importer to + alter this path. + + Paths in the a search path are subject to various substitutions. For paths starting with "~/", the user's home directory is substituted for "~". - For paths starting with "$ORIGIN/", the path of the directory of - the dependent is used. For main programs that's the path of the - jq executable's containing directory. + For paths starting with "$ORIGIN/", the path of the jq executable + is substituted for "$ORIGIN". + + For paths starting with "./" or paths that are ".", the path of + the including file is substituted for ".". For top-level programs + given on the command-line, the current directory is used. + + Import directives can optionally specify a search path to which + the default is appended. + + The default search path is the search path given to the `-L` + command-line option, else the "$JQ_LIBRARY_PATH", if set in the + environment, else `["~/.jq", "$ORIGIN/../lib/jq"]` (on Unix) or + `["~/.jq", "$ORIGIN/lib"]` (on Windows). + + The `-L` argument's and "$JQ_LIBRARY_PATH" environment variable's + string values are split on ":" on Unix, or ";" on Windows. + + Null and empty string path elements terminate search path + processing. Modules may be organized into a hierarchy, with name components separated by double-colons ("::"). For example: "foo::bar". Each component in the name corresponds to a directory on the - filesystem. Consecutive components with the same name are not - allowed to avoid ambiguities (e.g., "foo::foo"). + filesystem. - For example, a module named "foo::bar" would be searched for in - "foo/bar.jq" and "foo/bar/bar.jq" in the given search path. This - is intended to allow modules to be placed in a directory along - with, for example, version control files, README files, and so on, - but also to allow for single-file modules. + A module named "foo::bar" would be searched for in "foo/bar.jq" + and "foo/bar/bar.jq" in the given search path. This is intended to + allow modules to be placed in a directory along with, for example, + version control files, README files, and so on, but also to allow + for single-file modules. - The search path used is as follows: + Consecutive components with the same name are not allowed to avoid + ambiguities (e.g., "foo::foo"). - - the directory indicated in the search key of the `import` - directive metadata (see below) - - otherwise any directory(ies) listed in the `-L` command-line - argument - - otherwise any directory(ies) listed in the `$JQ_LIBRARY_PATH` - environment variable + For example, with `-L$HOME/.jq` a module `foo` can be found in + `$HOME/.jq/foo.jq` and `$HOME/.jq/foo/foo.jq`. - For example, with `-L$HOME/.jq` A module `foo` can be found in - `$HOME/.jq/any/foo.jq`, `$HOME/.jq/1.4/foo.jq`, - `$HOME/.jq/any/foo/foo.jq`, and in `$HOME/.jq/foo/foo.jq`. + If "$HOME/.jq" is a file, it is sourced into the main program. entries: - title: "`module NAME {<metadata>};`" @@ -2349,7 +2362,8 @@ sections: an object with keys like "version", "repo", "URI", and so on. The "search" key in the metadata, if present, should have a - string value consisting of a single directory path to search. + string or array value (array of strings); this is the search + path to be prefixed to the top-level search path. - title: "`modulemeta`" body: | @@ -1004,12 +1004,16 @@ int jq_compile(jq_state *jq, const char* str) { return jq_compile_args(jq, str, jv_array()); } -jv jq_get_lib_origin(jq_state *jq) { - return jq_get_attr(jq, jv_string("ORIGIN")); +jv jq_get_jq_origin(jq_state *jq) { + return jq_get_attr(jq, jv_string("JQ_ORIGIN")); +} + +jv jq_get_prog_origin(jq_state *jq) { + return jq_get_attr(jq, jv_string("PROGRAM_ORIGIN")); } jv jq_get_lib_dirs(jq_state *jq) { - return jq_get_attr(jq, jv_string("LIB_DIRS")); + return jq_get_attr(jq, jv_string("JQ_LIBRARY_PATH")); } void jq_set_attrs(jq_state *jq, jv attrs) { @@ -24,7 +24,8 @@ void jq_teardown(jq_state **); void jq_set_attrs(jq_state *, jv); jv jq_get_attrs(jq_state *); -jv jq_get_lib_origin(jq_state *); +jv jq_get_jq_origin(jq_state *); +jv jq_get_prog_origin(jq_state *); jv jq_get_lib_dirs(jq_state *); void jq_set_attr(jq_state *, jv, jv); jv jq_get_attr(jq_state *, jv); @@ -28,31 +28,19 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_ // 1. lib_path // 2. -L paths passed in on the command line (from jq_state*) // 3. JQ_LIBRARY_PATH environment variable -jv build_lib_search_chain(jq_state *jq, jv lib_path) { - assert(jv_get_kind(lib_path) == JV_KIND_STRING); - - jv out_paths = jv_array(); - if (jv_string_length_bytes(jv_copy(lib_path))) - out_paths = jv_array_append(out_paths, lib_path); - else - jv_free(lib_path); - jv lib_dirs = jq_get_lib_dirs(jq); - jv_array_foreach(lib_dirs, i, path) { - if (jv_string_length_bytes(jv_copy(path)) == 0) { - jv_free(path); - continue; - } +jv build_lib_search_chain(jq_state *jq, jv search_path) { + assert(jv_get_kind(search_path) == JV_KIND_ARRAY); + jv expanded = jv_array(); + jv_array_foreach(search_path, i, path) { path = expand_path(path); - if (jv_is_valid(path)) { - out_paths = jv_array_append(out_paths, path); - } else { - jv emsg = jv_invalid_get_msg(path); - jq_report_error(jq, jv_string_fmt("jq: warning: skipping search path: %s\n",jv_string_value(emsg))); - jv_free(emsg); - } + if (jv_is_valid(path)) + expanded = jv_array_append(expanded, expand_path(path)); + else + jv_free(path); } - jv_free(lib_dirs); - return out_paths; + jv_free(search_path); + expanded = jv_array_concat(expanded, jq_get_lib_dirs(jq)); + return expanded; } static jv name2relpath(jv name) { @@ -77,7 +65,7 @@ static jv name2relpath(jv name) { } static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) { - assert(jv_get_kind(lib_search_path) == JV_KIND_STRING); + assert(jv_get_kind(lib_search_path) == JV_KIND_ARRAY); assert(jv_get_kind(lib_name) == JV_KIND_STRING); jv rel_path = name2relpath(jv_copy(lib_name)); @@ -89,9 +77,18 @@ static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) { struct stat st; int ret; - jv lib_search_paths = build_lib_search_chain(jq, expand_path(lib_search_path)); + jv lib_search_paths = build_lib_search_chain(jq, lib_search_path); jv_array_foreach(lib_search_paths, i, spath) { + if (jv_get_kind(spath) == JV_KIND_NULL) { + jv_free(spath); + break; + } + if (jv_get_kind(spath) != JV_KIND_STRING || + strcmp(jv_string_value(spath), "") == 0) { + jv_free(spath); + continue; /* XXX report non-strings in search path?? */ + } jv testpath = jq_realpath(jv_string_fmt("%s/%s.jq", jv_string_value(spath), jv_string_value(rel_path))); @@ -125,7 +122,18 @@ static int version_matches(jq_state *jq, block importer, block module) { return 1; } -static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, struct lib_loading_state *lib_state) { +static jv default_search(jv value) { + if (!jv_is_valid(value)) { + jv_free(value); + return JV_ARRAY(jv_string("."), jv_string("$ORIGIN")); + } + if (jv_get_kind(value) != JV_KIND_ARRAY) + return JV_ARRAY(value); + return value; +} + +// XXX Split this into a util that takes a callback, and then... +static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block *src_block, struct lib_loading_state *lib_state) { jv deps = block_take_imports(src_block); block bk = *src_block; int nerrors = 0; @@ -137,26 +145,38 @@ static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, s jv_free(as); as = jv_string(""); } - jv search = jv_object_get(dep, jv_string("search")); - if (!jv_is_valid(search)) { - jv_free(search); - search = jv_string(""); - } - if (strncmp("$ORIGIN/",jv_string_value(search),8) == 0) { - jv tsearch = jv_string_fmt("%s/%s", - jv_string_value(lib_origin), - jv_string_value(search) + sizeof ("$ORIGIN/") - 1); - jv_free(search); - search = tsearch; + jv search = default_search(jv_object_get(dep, jv_string("search"))); + jv_array_foreach(search, k, search_elt) { + if (strcmp(".",jv_string_value(search_elt)) == 0) { + jv tsearch = jv_copy(lib_origin); + jv_free(search_elt); + search = jv_array_set(search, k, tsearch); + } else if (strncmp("./",jv_string_value(search_elt),sizeof("./")-1) == 0) { + jv tsearch = jv_string_fmt("%s/%s", + jv_string_value(lib_origin), + jv_string_value(search_elt) + sizeof ("./") - 1); + jv_free(search_elt); + search = jv_array_set(search, k, tsearch); + } else if (strncmp("$ORIGIN/",jv_string_value(search_elt),sizeof("$ORIGIN/")-1) == 0) { + jv tsearch = jv_string_fmt("%s/%s", + jv_string_value(jq_origin), + jv_string_value(search_elt) + sizeof ("$ORIGIN/") - 1); + jv_free(search_elt); + search = jv_array_set(search, k, tsearch); + } else { + jv_free(search_elt); + } } jv lib_path = find_lib(jq, name, search); + // XXX ...move the rest of this into a callback. if (!jv_is_valid(lib_path)) { jv emsg = jv_invalid_get_msg(lib_path); jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg))); jv_free(emsg); - jv_free(lib_origin); jv_free(as); jv_free(deps); + jv_free(jq_origin); + jv_free(lib_origin); return 1; } uint64_t state_idx = 0; @@ -186,6 +206,7 @@ static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, s jv_free(as); } jv_free(lib_origin); + jv_free(jq_origin); jv_free(deps); return nerrors; } @@ -208,7 +229,9 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_ lib_state->names[state_idx] = strdup(jv_string_value(lib_path)); lib_state->defs[state_idx] = program; char *lib_origin = strdup(jv_string_value(lib_path)); - nerrors += process_dependencies(jq, jv_string(dirname(lib_origin)), &lib_state->defs[state_idx], lib_state); + nerrors += process_dependencies(jq, jq_get_jq_origin(jq), + jv_string(dirname(lib_origin)), + &lib_state->defs[state_idx], lib_state); free(lib_origin); *out_block = lib_state->defs[state_idx]; } @@ -219,8 +242,10 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_ return nerrors; } +// FIXME It'd be nice to have an option to search the same search path +// as we do in process_dependencies. jv load_module_meta(jq_state *jq, jv modname) { - jv lib_path = find_lib(jq, modname, jv_string("")); + jv lib_path = find_lib(jq, modname, jv_array()); jv meta = jv_null(); jv data = jv_load_file(jv_string_value(lib_path), 1); if (jv_is_valid(data)) { @@ -249,7 +274,7 @@ int load_program(jq_state *jq, struct locfile* src, block *out_block) { if (nerrors) return nerrors; - nerrors = process_dependencies(jq, jq_get_lib_origin(jq), &program, &lib_state); + nerrors = process_dependencies(jq, jq_get_jq_origin(jq), jq_get_prog_origin(jq), &program, &lib_state); block libs = gen_noop(); for (uint64_t i = 0; i < lib_state.ct; ++i) { free(lib_state.names[i]); @@ -10,6 +10,7 @@ #include "jv.h" #include "jq.h" #include "jv_alloc.h" +#include "util.h" #include "version.h" int jq_testsuite(int argc, char* argv[]); @@ -210,7 +211,7 @@ int main(int argc, char* argv[]) { int jq_flags = 0; size_t short_opts = 0; jv program_arguments = jv_array(); - jv lib_search_paths = jv_array(); + jv lib_search_paths = jv_null(); for (int i=1; i<argc; i++, short_opts = 0) { if (further_args_are_files) { input_filenames[ninput_files++] = argv[i]; @@ -225,6 +226,8 @@ int main(int argc, char* argv[]) { } } else { if (argv[i][1] == 'L') { + if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) + lib_search_paths = jv_array(); if (argv[i][2] != 0) { // -Lname (faster check than strlen) lib_search_paths = jv_array_append(lib_search_paths, jv_string(argv[i]+2)); } else if (i >= argc - 1) { @@ -376,7 +379,8 @@ int main(int argc, char* argv[]) { } char *penv = getenv("JQ_LIBRARY_PATH"); - if (penv) { + if (penv && jv_get_kind(lib_search_paths) == JV_KIND_NULL) { + // Use $JQ_LIBRARY_PATH #ifdef WIN32 #define PATH_ENV_SEPARATOR ";" #else @@ -384,15 +388,22 @@ int main(int argc, char* argv[]) { #endif lib_search_paths = jv_array_concat(lib_search_paths,jv_string_split(jv_string(penv),jv_string(PATH_ENV_SEPARATOR))); #undef PATH_ENV_SEPARATOR + } else if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) { + // Use compiled-in default JQ_LIBRARY_PATH +#ifdef WIN32 + lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/lib")); +#else + lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/../lib/jq")); +#endif } - jq_set_attr(jq, jv_string("LIB_DIRS"), lib_search_paths); + jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_search_paths); char *origin = strdup(argv[0]); if (origin == NULL) { fprintf(stderr, "Error: out of memory\n"); exit(1); } - jq_set_attr(jq, jv_string("ORIGIN"), jv_string(dirname(origin))); + jq_set_attr(jq, jv_string("JQ_ORIGIN"), jv_string(dirname(origin))); free(origin); if (strchr(JQ_VERSION, '-') == NULL) @@ -436,6 +447,12 @@ int main(int argc, char* argv[]) { } if (options & FROM_FILE) { + char *program_origin = strdup(program); + if (program_origin == NULL) { + perror("malloc"); + exit(2); + } + jv data = jv_load_file(program, 1); if (!jv_is_valid(data)) { data = jv_invalid_get_msg(data); @@ -444,9 +461,12 @@ int main(int argc, char* argv[]) { ret = 2; goto out; } + jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string(dirname(program_origin)))); compiled = jq_compile_args(jq, jv_string_value(data), program_arguments); + free(program_origin); jv_free(data); } else { + jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string("."))); // XXX is this good? compiled = jq_compile_args(jq, program, program_arguments); } if (!compiled){ diff --git a/tests/modules/c/c.jq b/tests/modules/c/c.jq index 2d355ed3..d5000067 100644 --- a/tests/modules/c/c.jq +++ b/tests/modules/c/c.jq @@ -1,5 +1,14 @@ module c {whatever:null}; import a as foo; -import d as d {search:"$ORIGIN/"}; +import d as d {search:"./"}; +import d {search:"./"}; +import e as e {search:"./../lib/jq"}; +import f as f {search:"./../lib/jq"}; def a: 0; -def c: foo::a + "c" + d::meh; +def c: + if meh != d::meh then error("import into namespace doesn't work") + elif foo::a != "a" then error("foo::a didn't work as expected") + elif d::meh != "meh" then error("d::meh didn't work as expected") + elif e::bah != "bah" then error("e::bah didn't work as expected") + elif f::f != "f is here" then error("f::f didn't work as expected") + else foo::a + "c" + d::meh + e::bah end; diff --git a/tests/modules/lib/jq/e/e.jq b/tests/modules/lib/jq/e/e.jq new file mode 100644 index 00000000..84b6a8cc --- /dev/null +++ b/tests/modules/lib/jq/e/e.jq @@ -0,0 +1 @@ +def bah: "bah"; diff --git a/tests/modules/lib/jq/f.jq b/tests/modules/lib/jq/f.jq new file mode 100644 index 00000000..1018b198 --- /dev/null +++ b/tests/modules/lib/jq/f.jq @@ -0,0 +1 @@ +def f: "f is here"; @@ -137,12 +137,12 @@ if ! $VALGRIND $Q ./jq -ner -L $mods 'import a as foo; import b as bar; import a exit 1 fi -if ! $VALGRIND $Q ./jq -ner -L $mods 'import c as foo; [foo::a, foo::c] | . == [0,"acmeh"]' > /dev/null; then +if ! $VALGRIND $Q ./jq -ner -L $mods 'import c as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then echo "Module system appears to be broken" 1>&2 exit 1 fi -if [ "`$VALGRIND $Q ./jq -cner -L $mods '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"$ORIGIN/","as":"d","name":"d"}]}' ]; then +if [ "`$VALGRIND $Q ./jq -cner -L $mods '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"./","as":"d","name":"d"},{"search":"./","name":"d"},{"search":"./../lib/jq","as":"e","name":"e"},{"search":"./../lib/jq","as":"f","name":"f"}]}' ]; then echo "modulemeta builtin appears to be broken" 1>&2 exit 1 fi @@ -171,3 +171,14 @@ if [ -n "$VALGRIND" ] && ! grep 'ERROR SUMMARY: 0 errors from 0 contexts' $d/out cat $d/out exit 1 fi + +if ! $VALGRIND ./jq -ner -L $mods -f $mods/test_bind_order.jq > $d/out 2>&1; then + echo "Import bind order is broken" 1>&2 + exit 1 +fi + +if [ -n "$VALGRIND" ] && ! grep 'ERROR SUMMARY: 0 errors from 0 contexts' $d/out > /dev/null; then + echo "Memory errors when programs have syntax errors" 1>&2 + cat $d/out + exit 1 +fi |