lua: introduced a lua lib

Goal is to improve separation between packages and utilities.
Can help with autocompletion/navigate nixpkgs faster.
Also it will help standardize how LUA_PATH is exported across packages,
so that one can more easily make lua changes across nixpkgs (for
    instance changing where lua modules are installed).

11 files changed, 162 insertions, 97 deletions

diff --git a/pkgs/applications/editors/neovim/tests.nix b/pkgs/applications/editors/neovim/tests.nix
index 5f4cda714a67..08342a1543d5 100644
--- a/pkgs/applications/editors/neovim/tests.nix
+++ b/pkgs/applications/editors/neovim/tests.nix
@@ -133,7 +133,7 @@ rec {
     configure.pathogen.pluginNames = [ "vim-nix" ];
   };
 
-  nvimWithLuaPackages = wrapNeovim2 "with-lua-packages" (makeNeovimConfig {
+  nvimWithLuaPackages = wrapNeovim2 "-with-lua-packages" (makeNeovimConfig {
     extraLuaPackages = ps: [ps.mpack];
     customRC = ''
       lua require("mpack")
@@ -141,6 +141,7 @@ rec {
   });
 
   nvim_with_lua_packages = runTest nvimWithLuaPackages ''
+    export HOME=$TMPDIR
     ${nvimWithLuaPackages}/bin/nvim -i NONE --noplugin -es
   '';
 })
diff --git a/pkgs/applications/editors/neovim/utils.nix b/pkgs/applications/editors/neovim/utils.nix
index 837c9a79ffe8..bf0d80d1f07d 100644
--- a/pkgs/applications/editors/neovim/utils.nix
+++ b/pkgs/applications/editors/neovim/utils.nix
@@ -78,8 +78,7 @@ let
         ++ (extraPython3Packages ps)
         ++ (lib.concatMap (f: f ps) pluginPython3Packages));
 
-      lua = neovim-unwrapped.lua;
-      luaEnv = lua.withPackages(ps: extraLuaPackages ps);
+      luaEnv = neovim-unwrapped.lua.withPackages(extraLuaPackages);
 
       # Mapping a boolean argument to a key that tells us whether to add or not to
       # add to nvim's 'embedded rc' this:
@@ -115,8 +114,8 @@ let
         ] ++ lib.optionals (binPath != "") [
           "--suffix" "PATH" ":" binPath
         ] ++ lib.optionals (luaEnv != null) [
-          "--prefix" "LUA_PATH" ";" "${luaEnv}/share/lua/${lua.luaversion}/?.lua"
-          "--prefix" "LUA_CPATH" ";" "${luaEnv}/lib/lua/${lua.luaversion}/?.so"
+          "--prefix" "LUA_PATH" ";" (neovim-unwrapped.lua.pkgs.lib.genLuaPathAbsStr luaEnv)
+          "--prefix" "LUA_CPATH" ";" (neovim-unwrapped.lua.pkgs.lib.genLuaCPathAbsStr luaEnv)
         ];
 
 
diff --git a/pkgs/development/interpreters/lua-5/build-lua-package.nix b/pkgs/development/interpreters/lua-5/build-lua-package.nix
index 5639b2a4bb90..64e872ad5f63 100644
--- a/pkgs/development/interpreters/lua-5/build-lua-package.nix
+++ b/pkgs/development/interpreters/lua-5/build-lua-package.nix
@@ -238,7 +238,7 @@ builtins.removeAttrs attrs ["disabled" "checkInputs" "externalDeps" "extraVariab
     inherit externalDeps;
   } // passthru;
 
-  meta = with lib.maintainers; {
+  meta = {
     platforms = lua.meta.platforms;
     # add extra maintainer(s) to every package
     maintainers = (meta.maintainers or []) ++ [ ];
diff --git a/pkgs/development/interpreters/lua-5/interpreter.nix b/pkgs/development/interpreters/lua-5/interpreter.nix
index 3476b2b648b6..c6b4f478f633 100644
--- a/pkgs/development/interpreters/lua-5/interpreter.nix
+++ b/pkgs/development/interpreters/lua-5/interpreter.nix
@@ -1,7 +1,8 @@
 { lib, stdenv, fetchurl, readline
 , compat ? false
 , callPackage
-, packageOverrides ? (self: super: {})
+, makeWrapper
+, packageOverrides ? (final: prev: {})
 , sourceVersion
 , hash
 , patches ? []
@@ -10,7 +11,9 @@
 , staticOnly ? stdenv.hostPlatform.isStatic
 }:
 let
-luaPackages = callPackage ../../lua-modules {lua=self; overrides=packageOverrides;};
+  luaPackages = callPackage ../../lua-modules {
+    lua=self; overrides=packageOverrides;
+  };
 
 plat = if stdenv.isLinux then "linux"
        else if stdenv.isDarwin then "macosx"
@@ -31,21 +34,32 @@ self = stdenv.mkDerivation rec {
     sha256 = hash;
   };
 
-  LuaPathSearchPaths    = luaPackages.getLuaPathList luaversion;
-  LuaCPathSearchPaths   = luaPackages.getLuaCPathList luaversion;
+  LuaPathSearchPaths    = luaPackages.lib.luaPathList;
+  LuaCPathSearchPaths   = luaPackages.lib.luaCPathList;
   setupHook = luaPackages.lua-setup-hook LuaPathSearchPaths LuaCPathSearchPaths;
 
+  nativeBuildInputs = [ makeWrapper ];
   buildInputs = [ readline ];
 
   inherit patches;
 
-  postPatch = lib.optionalString (!stdenv.isDarwin && !staticOnly) ''
+  # we can't pass flags to the lua makefile because for portability, everything is hardcoded
+  postPatch = ''
+    {
+      echo -e '
+        #undef  LUA_PATH_DEFAULT
+        #define LUA_PATH_DEFAULT "./share/lua/${luaversion}/?.lua;./?.lua;./?/init.lua"
+        #undef  LUA_CPATH_DEFAULT
+        #define LUA_CPATH_DEFAULT "./lib/lua/${luaversion}/?.so;./?.so;./lib/lua/${luaversion}/loadall.so"
+      '
+    } >> src/luaconf.h
+  '' + lib.optionalString (!stdenv.isDarwin && !staticOnly) ''
     # Add a target for a shared library to the Makefile.
     sed -e '1s/^/LUA_SO = liblua.so/' \
         -e 's/ALL_T *= */&$(LUA_SO) /' \
         -i src/Makefile
     cat ${./lua-dso.make} >> src/Makefile
-  '';
+  '' ;
 
   # see configurePhase for additional flags (with space)
   makeFlags = [
diff --git a/pkgs/development/interpreters/lua-5/wrap-lua.nix b/pkgs/development/interpreters/lua-5/wrap-lua.nix
index f00e0d5ac336..049afcd6116d 100644
--- a/pkgs/development/interpreters/lua-5/wrap-lua.nix
+++ b/pkgs/development/interpreters/lua-5/wrap-lua.nix
@@ -4,16 +4,13 @@
 , makeWrapper
 }:
 
-with lib;
-
 # defined in trivial-builders.nix
 # imported as wrapLua in lua-packages.nix and passed to build-lua-derivation to be used as buildInput
 makeSetupHook {
-      deps = makeWrapper;
-      substitutions.executable = lua.interpreter;
-      substitutions.lua = lua;
-      substitutions.LuaPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
-      substitutions.LuaCPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
-
+  deps = makeWrapper;
+  substitutions.executable = lua.interpreter;
+  substitutions.lua = lua;
+  substitutions.LuaPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
+  substitutions.LuaCPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
 } ./wrap.sh
 
diff --git a/pkgs/development/interpreters/lua-5/wrapper.nix b/pkgs/development/interpreters/lua-5/wrapper.nix
index 0459b67534d4..b9ac255d2432 100644
--- a/pkgs/development/interpreters/lua-5/wrapper.nix
+++ b/pkgs/development/interpreters/lua-5/wrapper.nix
@@ -11,13 +11,18 @@
 let
   env = let
     paths =  requiredLuaModules (extraLibs ++ [ lua ] );
-  in (buildEnv {
+  in buildEnv {
     name = "${lua.name}-env";
 
     inherit paths;
     inherit ignoreCollisions;
     extraOutputsToInstall = [ "out" ] ++ extraOutputsToInstall;
 
+    nativeBuildInputs = [
+      makeWrapper
+      (lua.pkgs.lua-setup-hook lua.LuaPathSearchPaths lua.LuaCPathSearchPaths)
+    ];
+
     # we create wrapper for the binaries in the different packages
     postBuild = ''
       if [ -L "$out/bin" ]; then
@@ -37,7 +42,12 @@ let
               rm -f "$out/bin/$prg"
               if [ -x "$prg" ]; then
                 nix_debug "Making wrapper $prg"
-                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --suffix LUA_PATH ';' "$LUA_PATH"   --suffix LUA_CPATH ';' "$LUA_CPATH" ${lib.concatStringsSep " " makeWrapperArgs}
+                makeWrapper "$path/bin/$prg" "$out/bin/$prg" \
+                  --set-default LUA_PATH ";;" \
+                  --suffix LUA_PATH ';' "$LUA_PATH" \
+                  --set-default LUA_CPATH ";;" \
+                  --suffix LUA_CPATH ';' "$LUA_CPATH" \
+                  ${lib.concatStringsSep " " makeWrapperArgs}
               fi
             fi
           done
@@ -62,8 +72,5 @@ let
         '';
     };
     };
-  }).overrideAttrs (_: {
-    # Add extra deps needed for postBuild hook.
-    nativeBuildInputs = [ makeWrapper lua ];
-  });
+  };
 in env
diff --git a/pkgs/development/interpreters/luajit/default.nix b/pkgs/development/interpreters/luajit/default.nix
index 6a163c4b562f..ac74610bcce9 100644
--- a/pkgs/development/interpreters/luajit/default.nix
+++ b/pkgs/development/interpreters/luajit/default.nix
@@ -10,7 +10,7 @@
 , extraMeta ? { }
 , callPackage
 , self
-, packageOverrides ? (self: super: { })
+, packageOverrides ? (final: prev: {})
 , enableFFI ? true
 , enableJIT ? true
 , enableJITDebugModule ? enableJIT
@@ -62,6 +62,15 @@ stdenv.mkDerivation rec {
       # passed by nixpkgs CC wrapper is insufficient on its own
       substituteInPlace src/Makefile --replace "#CCDEBUG= -g" "CCDEBUG= -g"
     fi
+
+    {
+      echo -e '
+        #undef  LUA_PATH_DEFAULT
+        #define LUA_PATH_DEFAULT "./share/lua/${luaversion}/?.lua;./?.lua;./?/init.lua"
+        #undef  LUA_CPATH_DEFAULT
+        #define LUA_CPATH_DEFAULT "./lib/lua/${luaversion}/?.so;./?.so;./lib/lua/${luaversion}/loadall.so"
+      '
+    } >> src/luaconf.h
   '';
 
   configurePhase = false;
@@ -88,15 +97,10 @@ stdenv.mkDerivation rec {
     ln -s "$out"/bin/luajit-* "$out"/bin/luajit
   '';
 
-  LuaPathSearchPaths = [
-    "lib/lua/${luaversion}/?.lua"
-    "share/lua/${luaversion}/?.lua"
-    "share/lua/${luaversion}/?/init.lua"
-    "lib/lua/${luaversion}/?/init.lua"
-    "share/${name}/?.lua"
-  ];
-  LuaCPathSearchPaths = [ "lib/lua/${luaversion}/?.so" "share/lua/${luaversion}/?.so" ];
-  setupHook = luaPackages.lua-setup-hook LuaPathSearchPaths LuaCPathSearchPaths;
+  LuaPathSearchPaths    = luaPackages.lib.luaPathList;
+  LuaCPathSearchPaths   = luaPackages.lib.luaCPathList;
+
+  setupHook = luaPackages.lua-setup-hook luaPackages.lib.luaPathList luaPackages.lib.luaCPathList;
 
   passthru = rec {
     buildEnv = callPackage ../lua-5/wrapper.nix {
diff --git a/pkgs/development/lua-modules/default.nix b/pkgs/development/lua-modules/default.nix
index 091b94f58f51..e4927ee30464 100644
--- a/pkgs/development/lua-modules/default.nix
+++ b/pkgs/development/lua-modules/default.nix
@@ -1,7 +1,7 @@
 # inspired by pkgs/development/haskell-modules/default.nix
 { pkgs, lib
 , lua
-, overrides ? (self: super: {})
+, overrides ? (final: prev: {})
 }:
 
 let
@@ -15,7 +15,7 @@ let
   overridenPackages = import ./overrides.nix { inherit pkgs; };
 
   generatedPackages = if (builtins.pathExists ./generated-packages.nix) then
-        pkgs.callPackage ./generated-packages.nix { } else (self: super: {});
+        pkgs.callPackage ./generated-packages.nix { } else (final: prev: {});
 
   extensible-self = lib.makeExtensible
     (extends overrides
@@ -24,7 +24,6 @@ let
               initialPackages
               )
           )
-    )
-          ;
+    );
 in
   extensible-self
diff --git a/pkgs/development/lua-modules/lib.nix b/pkgs/development/lua-modules/lib.nix
new file mode 100644
index 000000000000..0180897f77a6
--- /dev/null
+++ b/pkgs/development/lua-modules/lib.nix
@@ -0,0 +1,63 @@
+{ pkgs, lib, lua }:
+let
+  requiredLuaModules = drvs: with lib; let
+    modules =  filter hasLuaModule drvs;
+  in unique ([lua] ++ modules ++ concatLists (catAttrs "requiredLuaModules" modules));
+  # Check whether a derivation provides a lua module.
+  hasLuaModule = drv: drv ? luaModule;
+in
+rec {
+  inherit hasLuaModule requiredLuaModules;
+
+  luaPathList = [
+    "share/lua/${lua.luaversion}/?.lua"
+    "share/lua/${lua.luaversion}/?/init.lua"
+  ];
+  luaCPathList = [
+    "lib/lua/${lua.luaversion}/?.so"
+  ];
+
+  /* generate paths without a prefix
+  */
+  luaPathRelStr = lib.concatStringsSep ";" luaPathList;
+  luaCPathRelStr = lib.concatStringsSep ";" luaCPathList;
+
+  /* generate LUA_(C)PATH value for a specific derivation, i.e., with absolute paths
+  */
+  genLuaPathAbsStr = drv: lib.concatMapStringsSep ";" (x: "${drv}/${x}") luaPathList;
+  genLuaCPathAbsStr = drv: lib.concatMapStringsSep ";" (x: "${drv}/${x}") luaCPathList;
+
+  /* Generate a LUA_PATH with absolute paths
+  */
+  # genLuaPathAbs = drv:
+  #   lib.concatStringsSep ";" (map (x: "${drv}/x") luaPathList);
+
+  luaAtLeast = lib.versionAtLeast lua.luaversion;
+  luaOlder = lib.versionOlder lua.luaversion;
+  isLua51 = (lib.versions.majorMinor lua.version) == "5.1";
+  isLua52 = (lib.versions.majorMinor lua.version) == "5.2";
+  isLua53 = lua.luaversion == "5.3";
+  isLuaJIT = lib.getName lua == "luajit";
+
+  /* generates the relative path towards the folder where
+   seems stable even when using  lua_modules_path = ""
+
+   Example:
+    getDataFolder luaPackages.stdlib
+    => stdlib-41.2.2-1-rocks/stdlib/41.2.2-1/doc
+  */
+  getDataFolder = drv:
+    "${drv.pname}-${drv.version}-rocks/${drv.pname}/${drv.version}";
+
+  /* Convert derivation to a lua module.
+    so that luaRequireModules can be run later
+  */
+  toLuaModule = drv:
+    drv.overrideAttrs( oldAttrs: {
+      # Use passthru in order to prevent rebuilds when possible.
+      passthru = (oldAttrs.passthru or {})// {
+        luaModule = lua;
+        requiredLuaModules = requiredLuaModules drv.propagatedBuildInputs;
+      };
+    });
+}
diff --git a/pkgs/development/lua-modules/overrides.nix b/pkgs/development/lua-modules/overrides.nix
index a15bd6e53663..247cf89f3945 100644
--- a/pkgs/development/lua-modules/overrides.nix
+++ b/pkgs/development/lua-modules/overrides.nix
@@ -363,6 +363,12 @@ with super;
     '';
   });
 
+  # TODO just while testing, remove afterwards
+  # toVimPlugin should do it instead
+  gitsigns-nvim = super.gitsigns-nvim.overrideAttrs(oa: {
+    nativeBuildInputs = oa.nativeBuildInputs or [] ++ [ pkgs.vimUtils.vimGenDocHook ];
+  });
+
   # aliases
   cjson = super.lua-cjson;
 }
diff --git a/pkgs/top-level/lua-packages.nix b/pkgs/top-level/lua-packages.nix
index 246d62b903b3..359c8980e73b 100644
--- a/pkgs/top-level/lua-packages.nix
+++ b/pkgs/top-level/lua-packages.nix
@@ -18,85 +18,60 @@ let
   packages = ( self:
 
 let
-  luaAtLeast = lib.versionAtLeast lua.luaversion;
-  luaOlder = lib.versionOlder lua.luaversion;
-  isLua51 = (lib.versions.majorMinor lua.version) == "5.1";
-  isLua52 = (lib.versions.majorMinor lua.version) == "5.2";
-  isLua53 = lua.luaversion == "5.3";
-  isLuaJIT = lib.getName lua == "luajit";
 
-  lua-setup-hook = callPackage ../development/interpreters/lua-5/setup-hook.nix { };
-
-  # Check whether a derivation provides a lua module.
-  hasLuaModule = drv: drv ? luaModule ;
+  # a function of lua_path / lua_cpath
+  lua-setup-hook = callPackage ../development/interpreters/lua-5/setup-hook.nix {
+    inherit lib;
+  };
 
   callPackage = pkgs.newScope self;
 
-  requiredLuaModules = drvs: with lib; let
-    modules =  filter hasLuaModule drvs;
-  in unique ([lua] ++ modules ++ concatLists (catAttrs "requiredLuaModules" modules));
-
-  # Convert derivation to a lua module.
-  toLuaModule = drv:
-    drv.overrideAttrs( oldAttrs: {
-      # Use passthru in order to prevent rebuilds when possible.
-      passthru = (oldAttrs.passthru or {})// {
-        luaModule = lua;
-        requiredLuaModules = requiredLuaModules drv.propagatedBuildInputs;
-      };
-    });
-
-
-  platformString =
-    if stdenv.isDarwin then "macosx"
-    else if stdenv.isFreeBSD then "freebsd"
-    else if stdenv.isLinux then "linux"
-    else if stdenv.isSunOS then "solaris"
-    else throw "unsupported platform";
-
   buildLuaApplication = args: buildLuarocksPackage ({namePrefix="";} // args );
 
-  buildLuarocksPackage = with pkgs.lib; makeOverridable(callPackage ../development/interpreters/lua-5/build-lua-package.nix {
-    inherit toLuaModule;
+  buildLuarocksPackage = lib.makeOverridable(callPackage ../development/interpreters/lua-5/build-lua-package.nix {
     inherit lua;
+    inherit (pkgs) lib;
+    inherit (luaLib) toLuaModule;
   });
-in
-with self; {
 
-  getLuaPathList = majorVersion: [
-    "share/lua/${majorVersion}/?.lua"
-    "share/lua/${majorVersion}/?/init.lua"
-  ];
-  getLuaCPathList = majorVersion: [
-    "lib/lua/${majorVersion}/?.so"
-  ];
-
-  # helper functions for dealing with LUA_PATH and LUA_CPATH
-  getPath = drv: pathListForVersion:
-    lib.concatMapStringsSep ";" (path: "${drv}/${path}") (pathListForVersion lua.luaversion);
-  getLuaPath = drv: getPath drv getLuaPathList;
-  getLuaCPath = drv: getPath drv getLuaCPathList;
+  luaLib = import ../development/lua-modules/lib.nix {
+    inherit (pkgs) lib;
+    inherit pkgs lua;
+  };
 
   #define build lua package function
   buildLuaPackage = callPackage ../development/lua-modules/generic {
-    inherit lua writeText;
+    inherit writeText;
   };
 
+  getPath = drv: pathListForVersion:
+    lib.concatMapStringsSep ";" (path: "${drv}/${path}") pathListForVersion;
 
-  inherit toLuaModule hasLuaModule lua-setup-hook;
-  inherit buildLuarocksPackage buildLuaApplication;
-  inherit requiredLuaModules luaOlder luaAtLeast
-    isLua51 isLua52 isLua53 isLuaJIT lua callPackage;
+in
+{
+  # helper functions for dealing with LUA_PATH and LUA_CPATH
+  lib = luaLib;
+
+  getLuaPath = drv: luaLib.getPath drv (luaLib.luaPathList lua.luaversion) ;
+  getLuaCPath = drv: luaLib.getPath drv (luaLib.luaCPathList lua.luaversion) ;
+
+
+  inherit lua lua-setup-hook callPackage;
+  inherit buildLuaPackage buildLuarocksPackage buildLuaApplication;
+  inherit (luaLib) luaOlder luaAtLeast isLua51 isLua52 isLua53 isLuaJIT
+    requiredLuaModules toLuaModule hasLuaModule;
 
   # wraps programs in $out/bin with valid LUA_PATH/LUA_CPATH
   wrapLua = callPackage ../development/interpreters/lua-5/wrap-lua.nix {
-    inherit lua; inherit (pkgs) makeSetupHook makeWrapper;
+    inherit lua lib;
+    inherit (pkgs) makeSetupHook makeWrapper;
   };
 
   luarocks = callPackage ../development/tools/misc/luarocks {
-    inherit lua;
+    inherit lua lib;
   };
 
+  # a fork of luarocks used to generate nix lua derivations from rockspecs
   luarocks-nix = callPackage ../development/tools/misc/luarocks/luarocks-nix.nix { };
 
   luxio = buildLuaPackage {
@@ -132,7 +107,7 @@ with self; {
     };
   };
 
-  vicious = toLuaModule(stdenv.mkDerivation rec {
+  vicious = toLuaModule( stdenv.mkDerivation rec {
     pname = "vicious";
     version = "2.5.0";