terraform-providers: handle go modules (#104667)

* terraform-providers: handle go modules

More and more terraform providers are switching from vendored
dependencies to Go modules.

Let's say that you update the "aws" provider and it fails. You can run
it again with:

    ./update-provider aws --vendor

Any package that has the "vendorSha256" key will be considered as a go
module package.

The script now also supports adding new providers by using the
<owner>/<repo> format. Eg:

     ./update-provider hetznercloud/hcloud --vendor

* address comment

Fixes https://github.com/NixOS/nixpkgs/pull/104667#discussion_r529788569

* support the null use-case

* escape provider name as well

* fix typo

1 files changed, 144 insertions, 43 deletions

diff --git a/pkgs/applications/networking/cluster/terraform-providers/update-provider b/pkgs/applications/networking/cluster/terraform-providers/update-provider
index c7094e1c77e4..e1f1a0ef38a5 100755
--- a/pkgs/applications/networking/cluster/terraform-providers/update-provider
+++ b/pkgs/applications/networking/cluster/terraform-providers/update-provider
@@ -1,42 +1,94 @@
 #!/usr/bin/env nix-shell
-#! nix-shell -i bash -p coreutils curl jq
+#! nix-shell -i bash -p coreutils curl jq moreutils
 # shellcheck shell=bash
+# vim: ft=sh
+#
+# Update a terraform provider to the latest version advertised at the
+# provider source address.
 #
-# Update a terraform provider to the latest version advertised at
-# the provider source address.
 set -euo pipefail
 
-USAGE=$(cat<<DOC
-Specify the terraform provider name to update.
+show_usage() {
+  cat <<DOC
+Usage: ./update-provider [--force] [--vendor] [<owner>/]<provider>
+
+Update a single provider in the providers.json inventory file.
+
+For example to update 'terraform-providers.aws' run:
+
+    ./update-provider aws
+
+If the provider is not in the list already, use the form '<owner>/<provider>'
+to add the provider to the list:
 
-Example:
-To update nixpkgs.terraform-providers.aws run:
-./update-provider aws
+    ./update-provider hetznercloud/hcloud
+
+Options:
+
+  * --force: Force the update even if the version matches.
+  * --vendor: Switch from go package to go modules with vendor.
+  * --vendor-sha256 <sha256>: Override the SHA256 or "null".
 DOC
-)
+}
+
+force=
+provider=
+vendor=
+vendorSha256=
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+  -h | --help)
+    show_usage
+    exit
+    ;;
+  --force)
+    force=1
+    shift
+    ;;
+  --vendor)
+    force=1
+    vendor=1
+    shift
+    ;;
+  --vendor-sha256)
+    force=1
+    vendorSha256=$2
+    shift 2
+    ;;
+  *)
+    if [[ -n "$provider" ]]; then
+      echo "ERROR: provider name was passed two times: '$provider' and '$1'"
+      echo "Use --help for more info"
+      exit 1
+    fi
+    provider=$1
+    shift
+  esac
+done
 
-provider_name="${1:-}"
-if [ -z "$provider_name" ]; then
-  echo "No providers specified!"
+if [[ -z "$provider" ]]; then
+  echo "ERROR: No providers specified!"
   echo
-  echo "$USAGE"
+  show_usage
   exit 1
 fi
 
-provider_source_address="$(jq -r ".$provider_name.\"provider-source-address\"" providers.json)"
+provider_name=$(basename "$provider")
 
-if [ "$provider_source_address" == "null" ]; then
-  echo "No provider source address specified with provider: $provider_name"
-  exit 1
-fi
+# Usage: read_attr <key>
+read_attr() {
+  jq -r ".\"$provider_name\".\"$1\"" providers.json
+}
 
-# The provider source address (used inside Terraform `required_providers` block) is
-# used to compute the registry API endpoint
-#
-# registry.terraform.io/hashicorp/aws (provider source address)
-# registry.terraform.io/providers/hashicorp/aws (provider URL for the website)
-# registry.terraform.io/v1/providers/hashicorp/aws (provider URL for the JSON API)
-registry_response=$(curl -s https://"${provider_source_address/\///v1/providers/}")
+# Usage: update_attr <key> <value>
+update_attr() {
+  if [[ "$2" == "null" ]]; then
+    jq -S ".\"$provider_name\".\"$1\" = null" providers.json | sponge providers.json
+  else
+    jq -S ".\"$provider_name\".\"$1\" = \"$2\"" providers.json | sponge providers.json
+  fi
+}
 
 prefetch_github() {
   # of a given owner, repo and rev, fetch the tarball and return the output of
@@ -47,31 +99,80 @@ prefetch_github() {
   nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz"
 }
 
+old_source_address="$(read_attr provider-source-address)"
+old_vendor_sha256=$(read_attr vendorSha256)
+old_version=$(read_attr version)
+
+if [[ $provider =~ ^[^/]+/[^/]+$ ]]; then
+  source_address=registry.terraform.io/$provider
+else
+  source_address=$old_source_address
+fi
+if [[ "$source_address" == "null" ]]; then
+  echo "Could not find the source address for provider: $provider"
+  exit 1
+fi
+update_attr "provider-source-address" "$source_address"
+
+# The provider source address (used inside Terraform `required_providers` block) is
+# used to compute the registry API endpoint
+#
+# registry.terraform.io/hashicorp/aws (provider source address)
+# registry.terraform.io/providers/hashicorp/aws (provider URL for the website)
+# registry.terraform.io/v1/providers/hashicorp/aws (provider URL for the JSON API)
+registry_response=$(curl -s https://"${source_address/\///v1/providers/}")
+
+version="$(jq -r '.version' <<< "$registry_response")"
+if [[ "$old_version" = "$version" && "$force" != 1 && -z "$vendorSha256" && "$old_vendor_sha256" != "$vendorSha256" ]]; then
+  echo "$provider_name is already at version $version"
+  exit
+fi
+update_attr version "$version"
+
 provider_source_url="$(jq -r '.source' <<< "$registry_response")"
 
 org="$(echo "$provider_source_url" | cut -d '/' -f 4)"
+update_attr owner "$org"
 repo="$(echo "$provider_source_url" | cut -d '/' -f 5)"
+update_attr repo "$repo"
 rev="$(jq -r '.tag' <<< "$registry_response")"
-
+update_attr rev "$rev"
 sha256=$(prefetch_github "$org" "$repo" "$rev")
+update_attr sha256 "$sha256"
 
-version="$(jq -r '.version' <<< "$registry_response")"
+repo_root=$(git rev-parse --show-toplevel)
 
-updated_provider="$(mktemp)"
-cat <<EOF >> "$updated_provider"
-{
-  "$provider_name": {
-    "owner": "$org",
-    "repo": "$repo",
-    "rev": "$rev",
-    "sha256": "$sha256",
-    "version": "$version",
-    "provider-source-address": "$provider_source_address"
-  }
-}
-EOF
+if [[ -z "$vendorSha256" ]]; then
+  if [[ "$old_vendor_sha256" == null ]]; then
+    vendorSha256=null
+  elif [[ -n "$old_vendor_sha256" || "$vendor" = 1 ]]; then
+    echo "=== Calculating vendorSha256 ==="
+    update_attr vendorSha256 "0000000000000000000000000000000000000000000000000000000000000000"
+    # Hackish way to find out the desired sha256. First build, then extract the
+    # error message from the logs.
+    set +e
+    nix-build --no-out-link "$repo_root" -A "terraform-providers.$provider_name.go-modules" 2>vendor_log.txt
+    set -e
+    logs=$(< vendor_log.txt)
+    if ! [[ $logs =~ got:\ +([^\ ]+) ]]; then
+      echo "ERROR: could not find new hash in output:"
+      cat vendor_log.txt
+      rm -f vendor_log.txt
+      exit 1
+    fi
+    rm -f vendor_log.txt
+    vendorSha256=${BASH_REMATCH[1]}
+    # Deal with nix unstable
+    if [[ $vendorSha256 = sha256-* ]]; then
+      vendorSha256=$(nix to-base32 "$vendorSha256")
+    fi
+  fi
+fi
 
-original_provider_list="$(mktemp)"
-cat providers.json > "$original_provider_list"
+if [[ -n "$vendorSha256" ]]; then
+  update_attr vendorSha256 "$vendorSha256"
+fi
 
-jq --sort-keys --slurp '.[0] * .[1]' "$original_provider_list" "$updated_provider" > providers.json
+# Check that the provider builds
+echo "=== Building terraform-providers.$provider_name ==="
+nix-build "$repo_root" -A "terraform-providers.$provider_name"