Move all of NixOS to nixos/ in preparation of the repository merge

4 files changed, 381 insertions, 0 deletions

diff --git a/nixos/maintainers/option-usages.nix b/nixos/maintainers/option-usages.nix
new file mode 100644
index 000000000000..7413b9e18cec
--- /dev/null
+++ b/nixos/maintainers/option-usages.nix
@@ -0,0 +1,99 @@
+{ configuration ? import ../lib/from-env.nix "NIXOS_CONFIG" <nixos-config>
+
+# []: display all options
+# [<option names>]: display the selected options
+, displayOptions ? [
+    "hardware.pcmcia.enable"
+    "environment.systemPackages"
+    "boot.kernelModules"
+    "services.udev.packages"
+    "jobs"
+    "environment.etc"
+    "system.activationScripts"
+  ]
+}:
+
+# This file is used to generate a dot graph which contains all options and
+# there dependencies to track problems and their sources.
+
+let
+
+  evalFun = {
+    extraArgs ? {}
+  }: import ../lib/eval-config.nix {
+       modules = [ configuration ];
+       inherit extraArgs;
+     };
+
+  eval = evalFun {};
+  inherit (eval) pkgs;
+
+  reportNewFailures = old: new: with pkgs.lib;
+    let
+      filterChanges =
+        filter ({fst, snd}:
+          !(fst.config.success -> snd.config.success)
+        );
+
+      keepNames =
+        map ({fst, snd}:
+          assert fst.name == snd.name; snd.name
+        );
+     in
+       keepNames (
+         filterChanges (
+           zipLists (collect isOption old) (collect isOption new)
+         )
+       );
+
+
+  # Create a list of modules where each module contains only one failling
+  # options.
+  introspectionModules = with pkgs.lib;
+    let
+      setIntrospection = opt: rec {
+        name = opt.name;
+        path = splitString "." opt.name;
+        config = setAttrByPath path
+          (throw "Usage introspection of '${name}' by forced failure.");
+      };
+    in
+      map setIntrospection (collect isOption eval.options);
+
+  overrideConfig = thrower:
+    pkgs.lib.recursiveUpdateUntil (path: old: new:
+      path == thrower.path
+    ) eval.config thrower.config;
+
+
+  graph = with pkgs.lib;
+    map (thrower: {
+      option = thrower.name;
+      usedBy = reportNewFailures eval.options (evalFun {
+        extraArgs = {
+          config = overrideConfig thrower;
+        };
+      }).options;
+    }) introspectionModules;
+
+  graphToDot = graph: with pkgs.lib; ''
+    digraph "Option Usages" {
+      ${concatMapStrings ({option, usedBy}:
+          assert __trace option true;
+          if displayOptions == [] || elem option displayOptions then
+            concatMapStrings (user: ''
+              "${option}" -> "${user}"''
+            ) usedBy
+          else ""
+        ) graph}
+    }
+  '';
+
+in
+
+pkgs.texFunctions.dot2pdf {
+  dotGraph = pkgs.writeTextFile {
+    name = "option_usages.dot";
+    text = graphToDot graph;
+  };
+}
diff --git a/nixos/maintainers/scripts/ec2/create-ebs-amis.py b/nixos/maintainers/scripts/ec2/create-ebs-amis.py
new file mode 100755
index 000000000000..93971ac9504d
--- /dev/null
+++ b/nixos/maintainers/scripts/ec2/create-ebs-amis.py
@@ -0,0 +1,220 @@
+#! /usr/bin/env python
+
+import os
+import sys
+import time
+import argparse
+import nixops.util
+from nixops import deployment
+from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
+import boto.ec2
+
+parser = argparse.ArgumentParser(description='Create an EBS-backed NixOS AMI')
+parser.add_argument('--region', dest='region', required=True, help='EC2 region to create the image in')
+parser.add_argument('--keep', dest='keep', action='store_true', help='Keep NixOps machine after use')
+parser.add_argument('--hvm', dest='hvm', action='store_true', help='Create HVM image')
+parser.add_argument('--key', dest='key_name', action='store_true', help='Keypair used for HVM instance creation', default="rob")
+args = parser.parse_args()
+
+instance_type = "cc1.4xlarge" if args.hvm else "m1.small"
+ebs_size = 8 if args.hvm else 20
+
+
+# Start a NixOS machine in the given region.
+f = open("ebs-creator-config.nix", "w")
+f.write('''{{
+  resources.ec2KeyPairs.keypair.accessKeyId = "logicblox-dev";
+  resources.ec2KeyPairs.keypair.region = "{0}";
+
+  machine =
+    {{ pkgs, ... }}:
+    {{
+      deployment.ec2.accessKeyId = "logicblox-dev";
+      deployment.ec2.region = "{0}";
+      deployment.ec2.blockDeviceMapping."/dev/xvdg".size = pkgs.lib.mkOverride 10 {1};
+    }};
+}}
+'''.format(args.region, ebs_size))
+f.close()
+
+db = deployment.open_database(deployment.get_default_state_file())
+try:
+    depl = deployment.open_deployment(db, "ebs-creator")
+except Exception:
+    depl = deployment.create_deployment(db)
+    depl.name = "ebs-creator"
+depl.auto_response = "y"
+depl.nix_exprs = [os.path.abspath("./ebs-creator.nix"), os.path.abspath("./ebs-creator-config.nix")]
+if not args.keep: depl.destroy_resources()
+depl.deploy(allow_reboot=True)
+
+m = depl.machines['machine']
+
+
+# Do the installation.
+device="/dev/xvdg"
+if args.hvm:
+    m.run_command('parted -s /dev/xvdg -- mklabel msdos')
+    m.run_command('parted -s /dev/xvdg -- mkpart primary ext2 1M -1s')
+    device="/dev/xvdg1"
+
+m.run_command("if mountpoint -q /mnt; then umount /mnt; fi")
+m.run_command("mkfs.ext4 -L nixos {0}".format(device))
+m.run_command("mkdir -p /mnt")
+m.run_command("mount {0} /mnt".format(device))
+m.run_command("touch /mnt/.ebs")
+m.run_command("mkdir -p /mnt/etc/nixos")
+m.run_command("nix-channel --add http://nixos.org/channels/nixos-unstable")
+m.run_command("nix-channel --update")
+m.run_command("nixos-rebuild switch")
+version = m.run_command("nixos-version", capture_stdout=True).replace('"', '').rstrip()
+print >> sys.stderr, "NixOS version is {0}".format(version)
+m.upload_file("./amazon-base-config.nix", "/mnt/etc/nixos/configuration.nix")
+m.run_command("nixos-install")
+if args.hvm:
+    m.run_command('cp /mnt/nix/store/*-grub-0.97*/lib/grub/i386-pc/* /mnt/boot/grub')
+    m.run_command('sed -i "s|hd0|hd0,0|" /mnt/boot/grub/menu.lst')
+    m.run_command('echo "(hd1) /dev/xvdg" > device.map')
+    m.run_command('echo -e "root (hd1,0)\nsetup (hd1)" | grub --device-map=device.map --batch')
+
+
+m.run_command("umount /mnt")
+
+
+if args.hvm:
+    ami_name = "nixos-{0}-x86_64-ebs-hvm".format(version)
+    description = "NixOS {0} (x86_64; EBS root; hvm)".format(version)
+else:
+    ami_name = "nixos-{0}-x86_64-ebs".format(version)
+    description = "NixOS {0} (x86_64; EBS root)".format(version)
+
+
+# Wait for the snapshot to finish.
+def check():
+    status = snapshot.update()
+    print >> sys.stderr, "snapshot status is {0}".format(status)
+    return status == '100%'
+
+m.connect()
+volume = m._conn.get_all_volumes([], filters={'attachment.instance-id': m.resource_id, 'attachment.device': "/dev/sdg"})[0]
+if args.hvm:
+    instance = m._conn.run_instances( image_id="ami-6a9e4503"
+                                    , instance_type=instance_type
+                                    , key_name=args.key_name
+                                    , placement=m.zone
+                                    , security_groups=["eelco-test"]).instances[0]
+    nixops.util.check_wait(lambda: instance.update() == 'running', max_tries=120)
+    instance.stop()
+    nixops.util.check_wait(lambda: instance.update() == 'stopped', max_tries=120)
+    old_root_volume = m._conn.get_all_volumes([], filters={'attachment.instance-id': instance.id, 'attachment.device': "/dev/sda1"})[0]
+    old_root_volume.detach()
+    volume.detach()
+    nixops.util.check_wait(lambda: volume.update() == 'available', max_tries=120)
+    nixops.util.check_wait(lambda: old_root_volume.update() == 'available', max_tries=120)
+    volume.attach(instance.id, '/dev/sda1')
+    nixops.util.check_wait(lambda: volume.update() == 'in-use', max_tries=120)
+
+    ami_id = m._conn.create_image(instance.id, ami_name, description)
+    time.sleep(5)
+    image = m._conn.get_all_images([ami_id])[0]
+    nixops.util.check_wait(lambda: image.update() == 'available', max_tries=120)
+    instance.terminate()
+
+else:
+    # Create a snapshot.
+    snapshot = volume.create_snapshot(description=description)
+    print >> sys.stderr, "created snapshot {0}".format(snapshot.id)
+
+    nixops.util.check_wait(check, max_tries=120)
+
+    m._conn.create_tags([snapshot.id], {'Name': ami_name})
+
+    if not args.keep: depl.destroy_resources()
+
+     # Register the image.
+    aki = m._conn.get_all_images(filters={'manifest-location': '*pv-grub-hd0_1.03-x86_64*'})[0]
+    print >> sys.stderr, "using kernel image {0} - {1}".format(aki.id, aki.location)
+
+    block_map = BlockDeviceMapping()
+    block_map['/dev/sda'] = BlockDeviceType(snapshot_id=snapshot.id, delete_on_termination=True)
+    block_map['/dev/sdb'] = BlockDeviceType(ephemeral_name="ephemeral0")
+    block_map['/dev/sdc'] = BlockDeviceType(ephemeral_name="ephemeral1")
+    block_map['/dev/sdd'] = BlockDeviceType(ephemeral_name="ephemeral2")
+    block_map['/dev/sde'] = BlockDeviceType(ephemeral_name="ephemeral3")
+
+    ami_id = m._conn.register_image(
+        name=ami_name,
+        description=description,
+        architecture="x86_64",
+        root_device_name="/dev/sda",
+        kernel_id=aki.id,
+        block_device_map=block_map)
+
+print >> sys.stderr, "registered AMI {0}".format(ami_id)
+
+print >> sys.stderr, "sleeping a bit..."
+time.sleep(30)
+
+print >> sys.stderr, "setting image name..."
+m._conn.create_tags([ami_id], {'Name': ami_name})
+
+print >> sys.stderr, "making image public..."
+image = m._conn.get_all_images(image_ids=[ami_id])[0]
+image.set_launch_permissions(user_ids=[], group_names=["all"])
+
+# Do a test deployment to make sure that the AMI works.
+f = open("ebs-test.nix", "w")
+f.write(
+    '''
+    {{
+      network.description = "NixOS EBS test";
+
+      resources.ec2KeyPairs.keypair.accessKeyId = "logicblox-dev";
+      resources.ec2KeyPairs.keypair.region = "{0}";
+
+      machine = {{ config, pkgs, resources, ... }}: {{
+        deployment.targetEnv = "ec2";
+        deployment.ec2.accessKeyId = "logicblox-dev";
+        deployment.ec2.region = "{0}";
+        deployment.ec2.instanceType = "{2}";
+        deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name;
+        deployment.ec2.securityGroups = [ "admin" ];
+        deployment.ec2.ami = "{1}";
+      }};
+    }}
+    '''.format(args.region, ami_id, instance_type))
+f.close()
+
+test_depl = deployment.create_deployment(db)
+test_depl.auto_response = "y"
+test_depl.name = "ebs-creator-test"
+test_depl.nix_exprs = [os.path.abspath("./ebs-test.nix")]
+test_depl.deploy(create_only=True)
+test_depl.machines['machine'].run_command("nixos-version")
+
+if args.hvm:
+    image_type = 'hvm'
+else:
+    image_type = 'ebs'
+
+# Log the AMI ID.
+f = open("{0}.{1}.ami-id".format(args.region, image_type), "w")
+f.write("{0}".format(ami_id))
+f.close()
+
+for dest in [ 'us-east-1', 'us-west-1', 'us-west-2', 'eu-west-1']:
+    if args.region != dest:
+        print >> sys.stderr, "copying image from region {0} to {1}".format(args.region, dest)
+        conn = boto.ec2.connect_to_region(dest)
+        copy_image = conn.copy_image(args.region, ami_id, ami_name, description=None, client_token=None)
+
+        # Log the AMI ID.
+        f = open("{0}.{1}.ami-id".format(dest, image_type), "w")
+        f.write("{0}".format(copy_image.image_id))
+        f.close()
+
+
+if not args.keep:
+    test_depl.destroy_resources()
+    test_depl.delete()
+
diff --git a/nixos/maintainers/scripts/ec2/create-s3-amis.sh b/nixos/maintainers/scripts/ec2/create-s3-amis.sh
new file mode 100755
index 000000000000..1aaac2832394
--- /dev/null
+++ b/nixos/maintainers/scripts/ec2/create-s3-amis.sh
@@ -0,0 +1,49 @@
+#! /bin/sh -e
+
+nixos=$(nix-instantiate --find-file nixos)
+export NIXOS_CONFIG=$(dirname $(readlink -f $0))/amazon-base-config.nix
+
+version=$(nix-instantiate --eval-only '<nixos>' -A config.system.nixosVersion | sed s/'"'//g)
+echo "NixOS version is $version"
+
+buildAndUploadFor() {
+    system="$1"
+    arch="$2"
+
+    echo "building $system image..."
+    nix-build '<nixos>' \
+        -A config.system.build.amazonImage --argstr system "$system" -o ec2-ami
+
+    ec2-bundle-image -i ./ec2-ami/nixos.img --user "$AWS_ACCOUNT" --arch "$arch" \
+        -c "$EC2_CERT" -k "$EC2_PRIVATE_KEY"
+
+    for region in eu-west-1 us-east-1 us-west-1 us-west-2; do
+        echo "uploading $system image for $region..."
+
+        name=nixos-$version-$arch-s3
+        bucket="$(echo $name-$region | tr '[A-Z]_' '[a-z]-')"
+
+        if [ "$region" = eu-west-1 ]; then s3location=EU;
+        elif [ "$region" = us-east-1 ]; then s3location=US;
+        else s3location="$region"
+        fi
+
+        ec2-upload-bundle -b "$bucket" -m /tmp/nixos.img.manifest.xml \
+            -a "$EC2_ACCESS_KEY" -s "$EC2_SECRET_KEY" --location "$s3location" \
+            --url http://s3.amazonaws.com
+
+        kernel=$(ec2-describe-images -o amazon --filter "manifest-location=*pv-grub-hd0_1.03-$arch*" --region "$region" | cut -f 2)
+        echo "using PV-GRUB kernel $kernel"
+
+        ami=$(ec2-register "$bucket/nixos.img.manifest.xml" -n "$name" -d "NixOS $system r$revision" \
+            --region "$region" --kernel "$kernel" | cut -f 2)
+
+        echo "AMI ID is $ami"
+
+        echo $ami >> $region.s3.ami-id
+
+        ec2-modify-image-attribute --region "$region" "$ami" -l -a all
+    done
+}
+
+buildAndUploadFor x86_64-linux x86_64
diff --git a/nixos/maintainers/scripts/ec2/ebs-creator.nix b/nixos/maintainers/scripts/ec2/ebs-creator.nix
new file mode 100644
index 000000000000..43dbd86f6754
--- /dev/null
+++ b/nixos/maintainers/scripts/ec2/ebs-creator.nix
@@ -0,0 +1,13 @@
+{
+  network.description = "NixOS EBS creator";
+
+  machine =
+    { config, pkgs, resources, ... }:
+    { deployment.targetEnv = "ec2";
+      deployment.ec2.instanceType = "m1.large";
+      deployment.ec2.securityGroups = [ "admin" ];
+      deployment.ec2.ebsBoot = false;
+      deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name;
+      environment.systemPackages = [ pkgs.parted ];
+    };
+}