buildGoModule: function for packaging Go modules

The function buildGoModule builds Go programs managed with Go modules. It builds
a Go module through a two phase build:

- An intermediate fetcher derivation. This derivation will be used to
  fetch all of the dependencies of the Go module.
- A final derivation will use the output of the intermediate derivation
  to build the binaries and produce the final output.

4 files changed, 401 insertions, 99 deletions

diff --git a/doc/languages-frameworks/go.xml b/doc/languages-frameworks/go.xml
index ab4c9f0f7c88..6f6e7925a1b5 100644
--- a/doc/languages-frameworks/go.xml
+++ b/doc/languages-frameworks/go.xml
@@ -3,12 +3,91 @@
          xml:id="sec-language-go">
  <title>Go</title>
 
- <para>
-  The function <varname>buildGoPackage</varname> builds standard Go programs.
- </para>
+ <section xml:id="ssec-go-modules">
+  <title>Go modules</title>
 
- <example xml:id='ex-buildGoPackage'>
-  <title>buildGoPackage</title>
+  <para>
+   The function <varname> buildGoModule </varname> builds Go programs managed
+   with Go modules. It builds a
+   <link xlink:href="https://github.com/golang/go/wiki/Modules">Go
+   modules</link> through a two phase build:
+   <itemizedlist>
+    <listitem>
+     <para>
+      An intermediate fetcher derivation. This derivation will be used to fetch
+      all of the dependencies of the Go module.
+     </para>
+    </listitem>
+    <listitem>
+     <para>
+      A final derivation will use the output of the intermediate derivation to
+      build the binaries and produce the final output.
+     </para>
+    </listitem>
+   </itemizedlist>
+  </para>
+
+  <example xml:id='ex-buildGoModule'>
+   <title>buildGoModule</title>
+<programlisting>
+pet = buildGoModule rec {
+  name = "pet-${version}";
+  version = "0.3.4";
+
+  src = fetchFromGitHub {
+    owner = "knqyf263";
+    repo = "pet";
+    rev = "v${version}";
+    sha256 = "0m2fzpqxk7hrbxsgqplkg7h2p7gv6s1miymv3gvw0cz039skag0s";
+  };
+
+  modSha256 = "1879j77k96684wi554rkjxydrj8g3hpp0kvxz03sd8dmwr3lh83j"; <co xml:id='ex-buildGoModule-1' />
+
+  subPackages = [ "." ]; <co xml:id='ex-buildGoModule-2' />
+
+  meta = with lib; {
+    description = "Simple command-line snippet manager, written in Go";
+    homepage = https://github.com/knqyf263/pet;
+    license = licenses.mit;
+    maintainers = with maintainers; [ kalbasit ];
+    platforms = platforms.linux ++ platforms.darwin;
+  };
+}
+</programlisting>
+  </example>
+
+  <para>
+   <xref linkend='ex-buildGoModule'/> is an example expression using
+   buildGoModule, the following arguments are of special significance to the
+   function:
+   <calloutlist>
+    <callout arearefs='ex-buildGoModule-1'>
+     <para>
+      <varname>modSha256</varname> is the hash of the output of the
+      intermediate fetcher derivation.
+     </para>
+    </callout>
+    <callout arearefs='ex-buildGoModule-2'>
+     <para>
+      <varname>subPackages</varname> limits the builder from building child
+      packages that have not been listed. If <varname>subPackages</varname> is
+      not specified, all child packages will be built.
+     </para>
+    </callout>
+   </calloutlist>
+  </para>
+ </section>
+
+ <section xml:id="ssec-go-legacy">
+  <title>Go legacy</title>
+
+  <para>
+   The function <varname> buildGoPackage </varname> builds legacy Go programs,
+   not supporting Go modules.
+  </para>
+
+  <example xml:id='ex-buildGoPackage'>
+   <title>buildGoPackage</title>
 <programlisting>
 deis = buildGoPackage rec {
   name = "deis-${version}";
@@ -29,56 +108,56 @@ deis = buildGoPackage rec {
   buildFlags = "--tags release"; <co xml:id='ex-buildGoPackage-4' />
 }
 </programlisting>
- </example>
-
- <para>
-  <xref linkend='ex-buildGoPackage'/> is an example expression using
-  buildGoPackage, the following arguments are of special significance to the
-  function:
-  <calloutlist>
-   <callout arearefs='ex-buildGoPackage-1'>
-    <para>
-     <varname>goPackagePath</varname> specifies the package's canonical Go
-     import path.
-    </para>
-   </callout>
-   <callout arearefs='ex-buildGoPackage-2'>
-    <para>
-     <varname>subPackages</varname> limits the builder from building child
-     packages that have not been listed. If <varname>subPackages</varname> is
-     not specified, all child packages will be built.
-    </para>
-    <para>
-     In this example only <literal>github.com/deis/deis/client</literal> will
-     be built.
-    </para>
-   </callout>
-   <callout arearefs='ex-buildGoPackage-3'>
-    <para>
-     <varname>goDeps</varname> is where the Go dependencies of a Go program are
-     listed as a list of package source identified by Go import path. It could
-     be imported as a separate <varname>deps.nix</varname> file for
-     readability. The dependency data structure is described below.
-    </para>
-   </callout>
-   <callout arearefs='ex-buildGoPackage-4'>
-    <para>
-     <varname>buildFlags</varname> is a list of flags passed to the go build
-     command.
-    </para>
-   </callout>
-  </calloutlist>
- </para>
-
- <para>
-  The <varname>goDeps</varname> attribute can be imported from a separate
-  <varname>nix</varname> file that defines which Go libraries are needed and
-  should be included in <varname>GOPATH</varname> for
-  <varname>buildPhase</varname>.
- </para>
-
- <example xml:id='ex-goDeps'>
-  <title>deps.nix</title>
+  </example>
+
+  <para>
+   <xref linkend='ex-buildGoPackage'/> is an example expression using
+   buildGoPackage, the following arguments are of special significance to the
+   function:
+   <calloutlist>
+    <callout arearefs='ex-buildGoPackage-1'>
+     <para>
+      <varname>goPackagePath</varname> specifies the package's canonical Go
+      import path.
+     </para>
+    </callout>
+    <callout arearefs='ex-buildGoPackage-2'>
+     <para>
+      <varname>subPackages</varname> limits the builder from building child
+      packages that have not been listed. If <varname>subPackages</varname> is
+      not specified, all child packages will be built.
+     </para>
+     <para>
+      In this example only <literal>github.com/deis/deis/client</literal> will
+      be built.
+     </para>
+    </callout>
+    <callout arearefs='ex-buildGoPackage-3'>
+     <para>
+      <varname>goDeps</varname> is where the Go dependencies of a Go program
+      are listed as a list of package source identified by Go import path. It
+      could be imported as a separate <varname>deps.nix</varname> file for
+      readability. The dependency data structure is described below.
+     </para>
+    </callout>
+    <callout arearefs='ex-buildGoPackage-4'>
+     <para>
+      <varname>buildFlags</varname> is a list of flags passed to the go build
+      command.
+     </para>
+    </callout>
+   </calloutlist>
+  </para>
+
+  <para>
+   The <varname>goDeps</varname> attribute can be imported from a separate
+   <varname>nix</varname> file that defines which Go libraries are needed and
+   should be included in <varname>GOPATH</varname> for
+   <varname>buildPhase</varname>.
+  </para>
+
+  <example xml:id='ex-goDeps'>
+   <title>deps.nix</title>
 <programlisting>
 [ <co xml:id='ex-goDeps-1' />
   {
@@ -101,60 +180,62 @@ deis = buildGoPackage rec {
   }
 ]
 </programlisting>
- </example>
-
- <para>
-  <calloutlist>
-   <callout arearefs='ex-goDeps-1'>
-    <para>
-     <varname>goDeps</varname> is a list of Go dependencies.
-    </para>
-   </callout>
-   <callout arearefs='ex-goDeps-2'>
-    <para>
-     <varname>goPackagePath</varname> specifies Go package import path.
-    </para>
-   </callout>
-   <callout arearefs='ex-goDeps-3'>
-    <para>
-     <varname>fetch type</varname> that needs to be used to get package source.
-     If <varname>git</varname> is used there should be <varname>url</varname>,
-     <varname>rev</varname> and <varname>sha256</varname> defined next to it.
-    </para>
-   </callout>
-  </calloutlist>
- </para>
-
- <para>
-  To extract dependency information from a Go package in automated way use
-  <link xlink:href="https://github.com/kamilchm/go2nix">go2nix</link>. It can
-  produce complete derivation and <varname>goDeps</varname> file for Go
-  programs.
- </para>
-
- <para>
-  <varname>buildGoPackage</varname> produces
-  <xref linkend='chap-multiple-output' xrefstyle="select: title" /> where
-  <varname>bin</varname> includes program binaries. You can test build a Go
-  binary as follows:
+  </example>
+
+  <para>
+   <calloutlist>
+    <callout arearefs='ex-goDeps-1'>
+     <para>
+      <varname>goDeps</varname> is a list of Go dependencies.
+     </para>
+    </callout>
+    <callout arearefs='ex-goDeps-2'>
+     <para>
+      <varname>goPackagePath</varname> specifies Go package import path.
+     </para>
+    </callout>
+    <callout arearefs='ex-goDeps-3'>
+     <para>
+      <varname>fetch type</varname> that needs to be used to get package
+      source. If <varname>git</varname> is used there should be
+      <varname>url</varname>, <varname>rev</varname> and
+      <varname>sha256</varname> defined next to it.
+     </para>
+    </callout>
+   </calloutlist>
+  </para>
+
+  <para>
+   To extract dependency information from a Go package in automated way use
+   <link xlink:href="https://github.com/kamilchm/go2nix">go2nix</link>. It can
+   produce complete derivation and <varname>goDeps</varname> file for Go
+   programs.
+  </para>
+
+  <para>
+   <varname>buildGoPackage</varname> produces
+   <xref linkend='chap-multiple-output' xrefstyle="select: title" /> where
+   <varname>bin</varname> includes program binaries. You can test build a Go
+   binary as follows:
 <screen>
     $ nix-build -A deis.bin
   </screen>
-  or build all outputs with:
+   or build all outputs with:
 <screen>
     $ nix-build -A deis.all
   </screen>
-  <varname>bin</varname> output will be installed by default with
-  <varname>nix-env -i</varname> or <varname>systemPackages</varname>.
- </para>
+   <varname>bin</varname> output will be installed by default with
+   <varname>nix-env -i</varname> or <varname>systemPackages</varname>.
+  </para>
 
- <para>
-  You may use Go packages installed into the active Nix profiles by adding the
-  following to your ~/.bashrc:
+  <para>
+   You may use Go packages installed into the active Nix profiles by adding the
+   following to your ~/.bashrc:
 <screen>
 for p in $NIX_PROFILES; do
     GOPATH="$p/share/go:$GOPATH"
 done
 </screen>
- </para>
+  </para>
+ </section>
 </section>
diff --git a/pkgs/development/go-modules/generic/default.nix b/pkgs/development/go-modules/generic/default.nix
new file mode 100644
index 000000000000..b4c18343aad3
--- /dev/null
+++ b/pkgs/development/go-modules/generic/default.nix
@@ -0,0 +1,214 @@
+{ go, cacert, git, lib, removeReferencesTo, stdenv }:
+
+{ name
+, src
+, buildInputs ? []
+, nativeBuildInputs ? []
+, passthru ? {}
+, patches ? []
+
+# modSha256 is the sha256 of the vendored dependencies
+, modSha256
+
+# We want parallel builds by default
+, enableParallelBuilding ? true
+
+# Disabled flag
+, disabled ? false
+
+# Do not enable this without good reason
+# IE: programs coupled with the compiler
+, allowGoReference ? false
+
+, meta ? {}
+
+, ... }@args':
+
+with builtins;
+
+let
+  args = removeAttrs args' [ "modSha256" "disabled" ];
+
+  removeReferences = [ ] ++ lib.optional (!allowGoReference) go;
+
+  removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}'';
+
+  go-modules = go.stdenv.mkDerivation {
+    name = "${name}-go-modules";
+
+    nativeBuildInputs = [ go git ];
+
+    inherit (args) src;
+    inherit (go) GOOS GOARCH;
+
+    patches = args.patches or [];
+
+    GO111MODULE = "on";
+
+    # XXX: Add support for other fetchers, such as hg, bzr and alike.
+    GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+    # Instruct Go where to find the cacert.
+    # SSL_CERT_FILE is used by Linux machines.
+    # NIX_SSL_CERT_FILE is used by Darwin machines based on
+    # pkgs/development/compilers/go/ssl-cert-file-1.9.patch.
+    NIX_SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+    SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+    impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
+      "GIT_PROXY_COMMAND" "SOCKS_SERVER"
+    ];
+
+    configurePhase = args.modConfigurePhase or ''
+      runHook preConfigure
+
+      export GOCACHE=$TMPDIR/go-cache
+      export GOPATH="$TMPDIR/go"
+
+      runHook postConfigure
+    '';
+
+    buildPhase = args.modBuildPhase or ''
+      runHook preBuild
+
+      go mod download
+
+      runHook postBuild
+    '';
+
+    installPhase = args.modInstallPhase or ''
+      runHook preInstall
+
+      cp -r "''${GOPATH}/pkg/mod/cache/download" $out
+
+      runHook postInstall
+    '';
+
+    dontFixup = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    outputHash = modSha256;
+  };
+
+  package = go.stdenv.mkDerivation (args // {
+    nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs;
+
+    inherit (go) GOOS GOARCH;
+
+    GO111MODULE = "on";
+
+    configurePhase = args.configurePhase or ''
+      runHook preConfigure
+
+      export GOCACHE=$TMPDIR/go-cache
+      export GOPATH="$TMPDIR/go"
+      export GOPROXY=file://${go-modules}
+
+      runHook postConfigure
+    '';
+
+    buildPhase = args.buildPhase or ''
+      runHook preBuild
+
+      buildGoDir() {
+        local d; local cmd;
+        cmd="$1"
+        d="$2"
+        . $TMPDIR/buildFlagsArray
+        echo "$d" | grep -q "\(/_\|examples\|Godeps\|testdata\)" && return 0
+        [ -n "$excludedPackages" ] && echo "$d" | grep -q "$excludedPackages" && return 0
+        local OUT
+        if ! OUT="$(go $cmd $buildFlags "''${buildFlagsArray[@]}" -v -p $NIX_BUILD_CORES $d 2>&1)"; then
+          if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then
+            echo "$OUT" >&2
+            return 1
+          fi
+        fi
+        if [ -n "$OUT" ]; then
+          echo "$OUT" >&2
+        fi
+        return 0
+      }
+
+      getGoDirs() {
+        local type;
+        type="$1"
+        if [ -n "$subPackages" ]; then
+          echo "./$subPackages"
+        else
+          find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique
+        fi
+      }
+
+      if (( "''${NIX_DEBUG:-0}" >= 1 )); then
+        buildFlagsArray+=(-x)
+      fi
+
+      if [ ''${#buildFlagsArray[@]} -ne 0 ]; then
+        declare -p buildFlagsArray > $TMPDIR/buildFlagsArray
+      else
+        touch $TMPDIR/buildFlagsArray
+      fi
+      if [ -z "$enableParallelBuilding" ]; then
+          export NIX_BUILD_CORES=1
+      fi
+      for pkg in $(getGoDirs ""); do
+        buildGoDir install "$pkg"
+      done
+    '' + lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+      # normalize cross-compiled builds w.r.t. native builds
+      (
+        dir=$GOPATH/bin/${go.GOOS}_${go.GOARCH}
+        if [[ -n "$(shopt -s nullglob; echo $dir/*)" ]]; then
+          mv $dir/* $dir/..
+        fi
+        if [[ -d $dir ]]; then
+          rmdir $dir
+        fi
+      )
+    '' + ''
+      runHook postBuild
+    '';
+
+    doCheck = args.doCheck or false;
+    checkPhase = args.checkPhase or ''
+      runHook preCheck
+
+      for pkg in $(getGoDirs test); do
+        buildGoDir test "$pkg"
+      done
+
+      runHook postCheck
+    '';
+
+    installPhase = args.installPhase or ''
+      runHook preInstall
+
+      mkdir -p $out
+      dir="$GOPATH/bin"
+      [ -e "$dir" ] && cp -r $dir $out
+
+      runHook postInstall
+    '';
+
+    preFixup = (args.preFixup or "") + ''
+      find $out/bin -type f -exec ${removeExpr removeReferences} '{}' + || true
+    '';
+
+    disallowedReferences = lib.optional (!allowGoReference) go;
+
+    passthru = passthru // { inherit go go-modules; };
+
+    meta = {
+      # Add default meta information
+      platforms = go.meta.platforms or lib.platforms.all;
+    } // meta // {
+      # add an extra maintainer to every package
+      maintainers = (meta.maintainers or []) ++
+                    [ lib.maintainers.kalbasit ];
+    };
+  });
+in if disabled then
+  throw "${package.name} not supported for go ${go.meta.branch}"
+else
+  package
diff --git a/pkgs/development/go-modules/tools/setup-hook.sh b/pkgs/development/go-modules/tools/setup-hook.sh
new file mode 100644
index 000000000000..e354e50c35aa
--- /dev/null
+++ b/pkgs/development/go-modules/tools/setup-hook.sh
@@ -0,0 +1 @@
+export GOTOOLDIR=@bin@/bin
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 2d9d4185bb40..a59625e0c954 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -13351,6 +13351,12 @@ in
 
   buildGoPackage = buildGo111Package;
 
+  buildGo111Module = callPackage ../development/go-modules/generic {
+    go = buildPackages.go_1_11;
+  };
+
+  buildGoModule = buildGo111Module;
+
   go2nix = callPackage ../development/tools/go2nix { };
 
   leaps = callPackage ../development/tools/leaps { };