Merging against master - updating smokingpig, rebase was going to be messy

7 files changed, 237 insertions, 43 deletions

diff --git a/nixos/tests/bittorrent.nix b/nixos/tests/bittorrent.nix
index 5aded554f4e8..3a718a798315 100644
--- a/nixos/tests/bittorrent.nix
+++ b/nixos/tests/bittorrent.nix
@@ -11,7 +11,7 @@ import ./make-test.nix ({ pkgs, ... }:
 let
 
   # Some random file to serve.
-  file = pkgs.nixUnstable.src;
+  file = pkgs.hello.src;
 
   miniupnpdConf = nodes: pkgs.writeText "miniupnpd.conf"
     ''
diff --git a/nixos/tests/installer.nix b/nixos/tests/installer.nix
index 1df2c651f9bc..35dd00fe630f 100644
--- a/nixos/tests/installer.nix
+++ b/nixos/tests/installer.nix
@@ -115,8 +115,8 @@ let
 
       # Did the swap device get activated?
       # uncomment once https://bugs.freedesktop.org/show_bug.cgi?id=86930 is resolved
-      #$machine->waitForUnit("swap.target");
-      $machine->waitUntilSucceeds("cat /proc/swaps | grep -q /dev");
+      $machine->waitForUnit("swap.target");
+      $machine->succeed("cat /proc/swaps | grep -q /dev");
 
       # Check whether the channel works.
       $machine->succeed("nix-env -iA nixos.procps >&2");
diff --git a/nixos/tests/kubernetes.nix b/nixos/tests/kubernetes.nix
index 273bd3c80c19..dcd25e211971 100644
--- a/nixos/tests/kubernetes.nix
+++ b/nixos/tests/kubernetes.nix
@@ -59,6 +59,7 @@ in {
             virtualisation.diskSize = 2048;
 
             programs.bash.enableCompletion = true;
+            environment.systemPackages = with pkgs; [ netcat bind ];
 
             services.kubernetes.roles = ["master" "node"];
             virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0";
diff --git a/nixos/tests/mesos.nix b/nixos/tests/mesos.nix
index 3610603aeba2..6e9af126f032 100644
--- a/nixos/tests/mesos.nix
+++ b/nixos/tests/mesos.nix
@@ -1,32 +1,91 @@
-import ./make-test.nix ({ pkgs, ...} : {
-  name = "simple";
+import ./make-test.nix ({ pkgs, ...} : rec {
+  name = "mesos";
   meta = with pkgs.stdenv.lib.maintainers; {
-    maintainers = [ offline ];
+    maintainers = [ offline kamilchm cstrahan ];
   };
 
-  machine = { config, pkgs, ... }: {
-    services.zookeeper.enable = true;
-    virtualisation.docker.enable = true;
-    services.mesos = {
-      slave = {
-        enable = true;
-        master = "zk://localhost:2181/mesos";
-        attributes = {
-          tag1 = "foo";
-          tag2 = "bar";
-        };
+  nodes = {
+    master = { config, pkgs, ... }: {
+      networking.firewall.enable = false;
+      services.zookeeper.enable = true;
+      services.mesos.master = {
+          enable = true;
+          zk = "zk://master:2181/mesos";
       };
-      master = {
-        enable = true;
-        zk = "zk://localhost:2181/mesos";
+    };
+
+    slave = { config, pkgs, ... }: {
+      networking.firewall.enable = false;
+      networking.nat.enable = true;
+      virtualisation.docker.enable = true;
+      services.mesos = {
+        slave = {
+          enable = true;
+          master = "master:5050";
+          dockerRegistry = registry;
+          executorEnvironmentVariables = {
+            PATH = "/run/current-system/sw/bin";
+          };
+        };
       };
     };
   };
 
+  simpleDocker = pkgs.dockerTools.buildImage {
+    name = "echo";
+    contents = [ pkgs.stdenv.shellPackage pkgs.coreutils ];
+    config = {
+      Env = [
+        # When shell=true, mesos invokes "sh -c '<cmd>'", so make sure "sh" is
+        # on the PATH.
+        "PATH=${pkgs.stdenv.shellPackage}/bin:${pkgs.coreutils}/bin"
+      ];
+      Entrypoint = [ "echo" ];
+    };
+  };
+
+  registry = pkgs.runCommand "registry" { } ''
+    mkdir -p $out
+    cp ${simpleDocker} $out/echo:latest.tar
+  '';
+
+  testFramework = pkgs.pythonPackages.buildPythonPackage {
+    name = "mesos-tests";
+    propagatedBuildInputs = [ pkgs.mesos ];
+    catchConflicts = false;
+    src = ./mesos_test.py;
+    phases = [ "installPhase" "fixupPhase" ];
+    installPhase = ''
+      mkdir $out
+      cp $src $out/mesos_test.py
+      chmod +x $out/mesos_test.py
+
+      echo "done" > test.result
+      tar czf $out/test.tar.gz test.result
+    '';
+  };
+
   testScript =
     ''
       startAll;
-      $machine->waitForUnit("mesos-master.service");
-      $machine->waitForUnit("mesos-slave.service");
+      $master->waitForUnit("mesos-master.service");
+      $slave->waitForUnit("mesos-slave.service");
+
+      $master->waitForOpenPort(5050);
+      $slave->waitForOpenPort(5051);
+
+      # is slave registred? 
+      $master->waitUntilSucceeds("curl -s --fail http://master:5050/master/slaves".
+                                 " | grep -q \"\\\"hostname\\\":\\\"slave\\\"\"");
+
+      # try to run docker image 
+      $master->succeed("${pkgs.mesos}/bin/mesos-execute --master=master:5050".
+                       " --resources=\"cpus:0.1;mem:32\" --name=simple-docker".
+                       " --containerizer=mesos --docker_image=echo:latest".
+                       " --shell=true --command=\"echo done\" | grep -q TASK_FINISHED");
+
+      # simple command with .tar.gz uri
+      $master->succeed("${testFramework}/mesos_test.py master ".
+                       "${testFramework}/test.tar.gz");
     '';
 })
diff --git a/nixos/tests/mesos_test.py b/nixos/tests/mesos_test.py
new file mode 100644
index 000000000000..be8bb32e49a7
--- /dev/null
+++ b/nixos/tests/mesos_test.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+import uuid
+import time
+import subprocess
+import os
+
+import sys
+
+from mesos.interface import Scheduler
+from mesos.native import MesosSchedulerDriver
+from mesos.interface import mesos_pb2
+
+def log(msg):
+    process = subprocess.Popen("systemd-cat", stdin=subprocess.PIPE)
+    (out,err) = process.communicate(msg)
+
+class NixosTestScheduler(Scheduler):
+    def __init__(self):
+        self.master_ip = sys.argv[1]
+        self.download_uri = sys.argv[2]
+
+    def resourceOffers(self, driver, offers):
+        log("XXX got resource offer")
+
+        offer = offers[0]
+        task = self.new_task(offer)
+        uri = task.command.uris.add()
+        uri.value = self.download_uri
+        task.command.value = "cat test.result"
+        driver.launchTasks(offer.id, [task])
+
+    def statusUpdate(self, driver, update):
+        log("XXX status update")
+        if update.state == mesos_pb2.TASK_FAILED:
+            log("XXX test task failed with message: " + update.message)
+            driver.stop()
+            sys.exit(1)
+        elif update.state == mesos_pb2.TASK_FINISHED:
+            driver.stop()
+            sys.exit(0)
+
+    def new_task(self, offer):
+        task = mesos_pb2.TaskInfo()
+        id = uuid.uuid4()
+        task.task_id.value = str(id)
+        task.slave_id.value = offer.slave_id.value
+        task.name = "task {}".format(str(id))
+
+        cpus = task.resources.add()
+        cpus.name = "cpus"
+        cpus.type = mesos_pb2.Value.SCALAR
+        cpus.scalar.value = 0.1
+
+        mem = task.resources.add()
+        mem.name = "mem"
+        mem.type = mesos_pb2.Value.SCALAR
+        mem.scalar.value = 32
+
+        return task
+
+if __name__ == '__main__':
+    log("XXX framework started")
+
+    framework = mesos_pb2.FrameworkInfo()
+    framework.user = "root"
+    framework.name = "nixos-test-framework"
+    driver = MesosSchedulerDriver(
+        NixosTestScheduler(),
+        framework,
+        sys.argv[1] + ":5050"
+    )
+    driver.run()
diff --git a/nixos/tests/nat.nix b/nixos/tests/nat.nix
index 4fbf64462682..74e20bff8d81 100644
--- a/nixos/tests/nat.nix
+++ b/nixos/tests/nat.nix
@@ -3,34 +3,47 @@
 # client on the inside network, a server on the outside network, and a
 # router connected to both that performs Network Address Translation
 # for the client.
-import ./make-test.nix ({ pkgs, withFirewall, ... }:
+import ./make-test.nix ({ pkgs, lib, withFirewall, withConntrackHelpers ? false, ... }:
   let
     unit = if withFirewall then "firewall" else "nat";
   in
   {
-    name = "nat${if withFirewall then "WithFirewall" else "Standalone"}";
-  meta = with pkgs.stdenv.lib.maintainers; {
+    name = "nat" + (if withFirewall then "WithFirewall" else "Standalone")
+                 + (lib.optionalString withConntrackHelpers "withConntrackHelpers");
+    meta = with pkgs.stdenv.lib.maintainers; {
       maintainers = [ eelco chaoflow rob wkennington ];
     };
 
     nodes =
       { client =
           { config, pkgs, nodes, ... }:
-          { virtualisation.vlans = [ 1 ];
-            networking.firewall.allowPing = true;
-            networking.defaultGateway =
-              (pkgs.lib.head nodes.router.config.networking.interfaces.eth2.ip4).address;
-          };
+          lib.mkMerge [
+            { virtualisation.vlans = [ 1 ];
+              networking.firewall.allowPing = true;
+              networking.defaultGateway =
+                (pkgs.lib.head nodes.router.config.networking.interfaces.eth2.ip4).address;
+            }
+            (lib.optionalAttrs withConntrackHelpers {
+              networking.firewall.connectionTrackingModules = [ "ftp" ];
+              networking.firewall.autoLoadConntrackHelpers = true;
+            })
+          ];
 
         router =
           { config, pkgs, ... }:
-          { virtualisation.vlans = [ 2 1 ];
-            networking.firewall.enable = withFirewall;
-            networking.firewall.allowPing = true;
-            networking.nat.enable = true;
-            networking.nat.internalIPs = [ "192.168.1.0/24" ];
-            networking.nat.externalInterface = "eth1";
-          };
+          lib.mkMerge [
+            { virtualisation.vlans = [ 2 1 ];
+              networking.firewall.enable = withFirewall;
+              networking.firewall.allowPing = true;
+              networking.nat.enable = true;
+              networking.nat.internalIPs = [ "192.168.1.0/24" ];
+              networking.nat.externalInterface = "eth1";
+            }
+            (lib.optionalAttrs withConntrackHelpers {
+              networking.firewall.connectionTrackingModules = [ "ftp" ];
+              networking.firewall.autoLoadConntrackHelpers = true;
+            })
+          ];
 
         server =
           { config, pkgs, ... }:
@@ -66,7 +79,8 @@ import ./make-test.nix ({ pkgs, withFirewall, ... }:
         $client->succeed("curl -v ftp://server/foo.txt >&2");
 
         # Test whether active FTP works.
-        $client->succeed("curl -v -P - ftp://server/foo.txt >&2");
+        $client->${if withConntrackHelpers then "succeed" else "fail"}(
+          "curl -v -P - ftp://server/foo.txt >&2");
 
         # Test ICMP.
         $client->succeed("ping -c 1 router >&2");
diff --git a/nixos/tests/networking.nix b/nixos/tests/networking.nix
index 17d4a878d3a4..83103f35d482 100644
--- a/nixos/tests/networking.nix
+++ b/nixos/tests/networking.nix
@@ -10,29 +10,61 @@ let
       vlanIfs = range 1 (length config.virtualisation.vlans);
     in {
       virtualisation.vlans = [ 1 2 3 ];
+      boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = true;
       networking = {
         useDHCP = false;
         useNetworkd = networkd;
         firewall.allowPing = true;
+        firewall.checkReversePath = true;
+        firewall.allowedUDPPorts = [ 547 ];
         interfaces = mkOverride 0 (listToAttrs (flip map vlanIfs (n:
           nameValuePair "eth${toString n}" {
             ipAddress = "192.168.${toString n}.1";
             prefixLength = 24;
+            ipv6Address = "fd00:1234:5678:${toString n}::1";
+            ipv6PrefixLength = 64;
           })));
       };
-      services.dhcpd = {
+      services.dhcpd4 = {
         enable = true;
         interfaces = map (n: "eth${toString n}") vlanIfs;
         extraConfig = ''
-          option subnet-mask 255.255.255.0;
+          authoritative;
         '' + flip concatMapStrings vlanIfs (n: ''
           subnet 192.168.${toString n}.0 netmask 255.255.255.0 {
-            option broadcast-address 192.168.${toString n}.255;
             option routers 192.168.${toString n}.1;
+            # XXX: technically it's _not guaranteed_ that IP addresses will be
+            # issued from the first item in range onwards! We assume that in
+            # our tests however.
             range 192.168.${toString n}.2 192.168.${toString n}.254;
           }
         '');
       };
+      services.radvd = {
+        enable = true;
+        config = flip concatMapStrings vlanIfs (n: ''
+          interface eth${toString n} {
+            AdvSendAdvert on;
+            AdvManagedFlag on;
+            AdvOtherConfigFlag on;
+
+            prefix fd00:1234:5678:${toString n}::/64 {
+              AdvAutonomous off;
+            };
+          };
+        '');
+      };
+      services.dhcpd6 = {
+        enable = true;
+        interfaces = map (n: "eth${toString n}") vlanIfs;
+        extraConfig = ''
+          authoritative;
+        '' + flip concatMapStrings vlanIfs (n: ''
+          subnet6 fd00:1234:5678:${toString n}::/64 {
+            range6 fd00:1234:5678:${toString n}::2 fd00:1234:5678:${toString n}::2;
+          }
+        '');
+      };
     };
 
   testCases = {
@@ -108,8 +140,14 @@ let
           useNetworkd = networkd;
           firewall.allowPing = true;
           useDHCP = true;
-          interfaces.eth1.ip4 = mkOverride 0 [ ];
-          interfaces.eth2.ip4 = mkOverride 0 [ ];
+          interfaces.eth1 = {
+            ip4 = mkOverride 0 [ ];
+            ip6 = mkOverride 0 [ ];
+          };
+          interfaces.eth2 = {
+            ip4 = mkOverride 0 [ ];
+            ip6 = mkOverride 0 [ ];
+          };
         };
       };
       testScript = { nodes, ... }:
@@ -121,21 +159,31 @@ let
 
           # Wait until we have an ip address on each interface
           $client->waitUntilSucceeds("ip addr show dev eth1 | grep -q '192.168.1'");
+          $client->waitUntilSucceeds("ip addr show dev eth1 | grep -q 'fd00:1234:5678:1:'");
           $client->waitUntilSucceeds("ip addr show dev eth2 | grep -q '192.168.2'");
+          $client->waitUntilSucceeds("ip addr show dev eth2 | grep -q 'fd00:1234:5678:2:'");
 
           # Test vlan 1
           $client->waitUntilSucceeds("ping -c 1 192.168.1.1");
           $client->waitUntilSucceeds("ping -c 1 192.168.1.2");
+          $client->waitUntilSucceeds("ping6 -c 1 fd00:1234:5678:1::1");
+          $client->waitUntilSucceeds("ping6 -c 1 fd00:1234:5678:1::2");
 
           $router->waitUntilSucceeds("ping -c 1 192.168.1.1");
           $router->waitUntilSucceeds("ping -c 1 192.168.1.2");
+          $router->waitUntilSucceeds("ping6 -c 1 fd00:1234:5678:1::1");
+          $router->waitUntilSucceeds("ping6 -c 1 fd00:1234:5678:1::2");
 
           # Test vlan 2
           $client->waitUntilSucceeds("ping -c 1 192.168.2.1");
           $client->waitUntilSucceeds("ping -c 1 192.168.2.2");
+          $client->waitUntilSucceeds("ping6 -c 1 fd00:1234:5678:2::1");
+          $client->waitUntilSucceeds("ping6 -c 1 fd00:1234:5678:2::2");
 
           $router->waitUntilSucceeds("ping -c 1 192.168.2.1");
           $router->waitUntilSucceeds("ping -c 1 192.168.2.2");
+          $router->waitUntilSucceeds("ping6 -c 1 fd00:1234:5678:2::1");
+          $router->waitUntilSucceeds("ping6 -c 1 fd00:1234:5678:2::2");
         '';
     };
     dhcpOneIf = {