diff options
author | Graham Christensen <graham@grahamc.com> | 2019-04-11 12:35:35 -0400 |
---|---|---|
committer | Graham Christensen <graham@grahamc.com> | 2019-04-11 19:21:38 -0400 |
commit | 817c933878d1eea16af8a0efcb62fff59c698cdb (patch) | |
tree | 60219d6b23ad1da894064ce1a2e9b91fdec869f8 /pkgs/stdenv | |
parent | 0a80d3dc822185a4d7063566f6a8bb40b5ef2913 (diff) |
check-env: don't execute check-meta.nix 15,000 times
Generated from https://github.com/NixOS/nix/pull/2761:
```
ns calls ns/call
- /home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:22:5 591200 15026 39.3451
+ /home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:22:5 8744 308 28.3896
```
more, generated by:
```
$ NIX_SHOW_STATS=1 NIX_COUNT_CALLS=1 nix-instantiate ./pkgs/top-level/release.nix -A unstable > before 2>&1
$ jq -r '.functions | map((.name + ":" + .file + ":" + (.line|tostring) + ":" + (.column|tostring) + " " + (.count|tostring))) | .[]' before | sort > before.list
```
applying this patch, then:
```
$ NIX_SHOW_STATS=1 NIX_COUNT_CALLS=1 nix-instantiate ./pkgs/top-level/release.nix -A unstable > after 2>&1
$ jq -r '.functions | map((.name + ":" + .file + ":" + (.line|tostring) + ":" + (.column|tostring) + " " + (.count|tostring))) | .[]' after | sort > after.list
```
and then diffing before.list and after.list to get:
```
calls
- :/home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:4:1 7513
+ :/home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:4:1 154
- mutuallyExclusive:/home/grahamc/projects/github.com/NixOS/nixpkgs/lib/lists.nix:658:23 7513
+ mutuallyExclusive:/home/grahamc/projects/github.com/NixOS/nixpkgs/lib/lists.nix:658:23 154
- mutuallyExclusive:/home/grahamc/projects/github.com/NixOS/nixpkgs/lib/lists.nix:658:26 7513
+ mutuallyExclusive:/home/grahamc/projects/github.com/NixOS/nixpkgs/lib/lists.nix:658:26 154
- onlyLicenses:/home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:21:18 15026
+ onlyLicenses:/home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:21:18 308
```
The following information is from `NIX_SHOW_STATS=1 GC_INITIAL_HEAP_SIZE=4g nix-env -f ./outpaths.nix -qaP --no-name --out-path --arg checkMeta true`:
| stat | before | after | Δ | Δ% |
|:---------------------------|---------------:|---------------:|:----------------|--------:|
| **cpuTime** | 179.915 | 145.543 | 🡖 34.372 | -19.10% |
| **envs-bytes** | 3,900,878,824 | 3,599,483,208 | 🡖 301,395,616 | -7.73% |
| **envs-elements** | 214,426,071 | 185,881,709 | 🡖 28,544,362 | -13.31% |
| **envs-number** | 136,591,891 | 132,026,846 | 🡖 4,565,045 | -3.34% |
| **gc-heapSize** | 11,400,048,640 | 12,314,890,240 | 🡕 914,841,600 | 8.02% |
| **gc-totalBytes** | 25,976,902,560 | 24,510,740,176 | 🡖 1,466,162,384 | -5.64% |
| **list-bytes** | 1,665,290,080 | 1,665,290,080 | 0 | |
| **list-concats** | 7,264,417 | 7,264,417 | 0 | |
| **list-elements** | 208,161,260 | 208,161,260 | 0 | |
| **nrAvoided** | 191,359,386 | 179,693,661 | 🡖 11,665,725 | -6.10% |
| **nrFunctionCalls** | 119,665,062 | 116,348,547 | 🡖 3,316,515 | -2.77% |
| **nrLookups** | 80,996,257 | 76,069,825 | 🡖 4,926,432 | -6.08% |
| **nrOpUpdateValuesCopied** | 213,930,649 | 213,930,649 | 0 | |
| **nrOpUpdates** | 12,025,937 | 12,025,937 | 0 | |
| **nrPrimOpCalls** | 88,105,604 | 86,451,598 | 🡖 1,654,006 | -1.88% |
| **nrThunks** | 196,842,044 | 175,126,701 | 🡖 21,715,343 | -11.03% |
| **sets-bytes** | 7,678,425,776 | 7,285,767,928 | 🡖 392,657,848 | -5.11% |
| **sets-elements** | 310,241,340 | 294,373,227 | 🡖 15,868,113 | -5.11% |
| **sets-number** | 29,079,202 | 27,601,310 | 🡖 1,477,892 | -5.08% |
| **sizes-Attr** | 24 | 24 | 0 | |
| **sizes-Bindings** | 8 | 8 | 0 | |
| **sizes-Env** | 16 | 16 | 0 | |
| **sizes-Value** | 24 | 24 | 0 | |
| **symbols-bytes** | 16,474,666 | 16,474,676 | 🡕 10 | 0.00% |
| **symbols-number** | 376,426 | 376,427 | 🡕 1 | 0.00% |
| **values-bytes** | 6,856,506,288 | 6,316,585,560 | 🡖 539,920,728 | -7.87% |
| **values-number** | 285,687,762 | 263,191,065 | 🡖 22,496,697 | -7.87% |
The following information is from `NIX_SHOW_STATS=1 GC_INITIAL_HEAP_SIZE=4g nix-instantiate ./nixos/release-combined.nix -A tested`:
| stat | before | after | Δ | Δ% |
|:---------------------------|---------------:|---------------:|:----------------|-------:|
| **cpuTime** | 256.071 | 237.531 | 🡖 18.54 | -7.24% |
| **envs-bytes** | 7,111,004,192 | 7,041,478,520 | 🡖 69,525,672 | -0.98% |
| **envs-elements** | 346,236,940 | 339,588,487 | 🡖 6,648,453 | -1.92% |
| **envs-number** | 271,319,292 | 270,298,164 | 🡖 1,021,128 | -0.38% |
| **gc-heapSize** | 8,995,291,136 | 10,110,009,344 | 🡕 1,114,718,208 | 12.39% |
| **gc-totalBytes** | 37,172,737,408 | 36,878,391,888 | 🡖 294,345,520 | -0.79% |
| **list-bytes** | 1,886,162,656 | 1,886,163,472 | 🡕 816 | 0.00% |
| **list-concats** | 6,898,114 | 6,898,114 | 0 | |
| **list-elements** | 235,770,332 | 235,770,434 | 🡕 102 | 0.00% |
| **nrAvoided** | 328,829,821 | 326,618,157 | 🡖 2,211,664 | -0.67% |
| **nrFunctionCalls** | 240,850,845 | 239,998,495 | 🡖 852,350 | -0.35% |
| **nrLookups** | 144,849,632 | 142,126,339 | 🡖 2,723,293 | -1.88% |
| **nrOpUpdateValuesCopied** | 251,032,504 | 251,032,504 | 0 | |
| **nrOpUpdates** | 17,903,110 | 17,903,110 | 0 | |
| **nrPrimOpCalls** | 140,674,913 | 139,485,975 | 🡖 1,188,938 | -0.85% |
| **nrThunks** | 294,643,131 | 288,678,022 | 🡖 5,965,109 | -2.02% |
| **sets-bytes** | 9,464,322,192 | 9,456,172,048 | 🡖 8,150,144 | -0.09% |
| **sets-elements** | 377,474,889 | 377,134,877 | 🡖 340,012 | -0.09% |
| **sets-number** | 50,615,607 | 50,616,875 | 🡕 1,268 | 0.00% |
| **sizes-Attr** | 24 | 24 | 0 | |
| **sizes-Bindings** | 8 | 8 | 0 | |
| **sizes-Env** | 16 | 16 | 0 | |
| **sizes-Value** | 24 | 24 | 0 | |
| **symbols-bytes** | 3,147,102 | 3,147,064 | 🡖 38 | -0.00% |
| **symbols-number** | 82,819 | 82,819 | 0 | |
| **values-bytes** | 11,147,448,768 | 10,996,111,512 | 🡖 151,337,256 | -1.36% |
| **values-number** | 464,477,032 | 458,171,313 | 🡖 6,305,719 | -1.36% |
Diffstat (limited to 'pkgs/stdenv')
-rw-r--r-- | pkgs/stdenv/generic/check-meta.nix | 12 | ||||
-rw-r--r-- | pkgs/stdenv/generic/make-derivation.nix | 16 |
2 files changed, 15 insertions, 13 deletions
diff --git a/pkgs/stdenv/generic/check-meta.nix b/pkgs/stdenv/generic/check-meta.nix index fe0c8cfad919..76e1c351ff40 100644 --- a/pkgs/stdenv/generic/check-meta.nix +++ b/pkgs/stdenv/generic/check-meta.nix @@ -1,7 +1,7 @@ # Checks derivation meta and attrs for problems (like brokenness, # licenses, etc). -{ lib, config, hostPlatform, meta }: +{ lib, config, hostPlatform }: let # If we're in hydra, we can dispense with the more verbose error @@ -76,7 +76,7 @@ let showLicense = license: license.shortName or "unknown"; - pos_str = meta.position or "«unknown-file»"; + pos_str = meta: meta.position or "«unknown-file»"; remediation = { unfree = remediate_whitelist "Unfree"; @@ -143,12 +143,12 @@ let ${lib.concatStrings (builtins.map (output: " - ${output}\n") missingOutputs)} ''; - handleEvalIssue = attrs: { reason , errormsg ? "" }: + handleEvalIssue = { meta, attrs }: { reason , errormsg ? "" }: let msg = if inHydra then "Failed to evaluate ${attrs.name or "«name-missing»"}: «${reason}»: ${errormsg}" else '' - Package ‘${attrs.name or "«name-missing»"}’ in ${pos_str} ${errormsg}, refusing to evaluate. + Package ‘${attrs.name or "«name-missing»"}’ in ${pos_str meta} ${errormsg}, refusing to evaluate. '' + (builtins.getAttr reason remediation) attrs; @@ -245,12 +245,12 @@ let { valid = false; reason = "unknown-meta"; errormsg = "has an invalid meta attrset:${lib.concatMapStrings (x: "\n\t - " + x) res}"; } else { valid = true; }; - assertValidity = attrs: let + assertValidity = { meta, attrs }: let validity = checkValidity attrs; in validity // { # Throw an error if trying to evaluate an non-valid derivation handled = if !validity.valid - then handleEvalIssue attrs (removeAttrs validity ["valid"]) + then handleEvalIssue { inherit meta attrs; } (removeAttrs validity ["valid"]) else true; }; diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index dfb7e95113b8..417c10363b52 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -1,6 +1,13 @@ { lib, config, stdenv }: -rec { +let + checkMeta = import ./check-meta.nix { + inherit lib config; + # Nix itself uses the `system` field of a derivation to decide where + # to build it. This is a bit confusing for cross compilation. + inherit (stdenv) hostPlatform; + }; +in rec { # `mkDerivation` wraps the builtin `derivation` function to # produce derivations that use this stdenv and its shell. # @@ -263,12 +270,7 @@ rec { __propagatedImpureHostDeps = computedPropagatedImpureHostDeps ++ __propagatedImpureHostDeps; }; - validity = import ./check-meta.nix { - inherit lib config meta; - # Nix itself uses the `system` field of a derivation to decide where - # to build it. This is a bit confusing for cross compilation. - inherit (stdenv) hostPlatform; - } attrs; + validity = checkMeta { inherit meta attrs; }; # The meta attribute is passed in the resulting attribute set, # but it's not part of the actual derivation, i.e., it's not |