diff options
author | Jaka Hudoklin <jakahudoklin@gmail.com> | 2016-11-24 23:10:01 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-11-24 23:10:01 +0100 |
commit | 3b500d37f503dd63da7a76ade9765e0b81c796cb (patch) | |
tree | 7a0e90f3b20e0575267773b2632856f74bfc721f /nixos/tests | |
parent | e29e2467201be02de2ae67bf1916b2114ffada23 (diff) | |
parent | 5bc7ae7adb66639dbff676db51454ed2e5f8ad87 (diff) |
Merge pull request #19023 from offlinehacker/kube-update
WIP: kubernetes update package and module
Diffstat (limited to 'nixos/tests')
-rw-r--r-- | nixos/tests/kubernetes.nix | 550 |
1 files changed, 388 insertions, 162 deletions
diff --git a/nixos/tests/kubernetes.nix b/nixos/tests/kubernetes.nix index b19ea67b0baf..273bd3c80c19 100644 --- a/nixos/tests/kubernetes.nix +++ b/nixos/tests/kubernetes.nix @@ -1,182 +1,408 @@ -# This test runs two node kubernetes cluster and checks if simple redis pod works +{ system ? builtins.currentSystem }: -import ./make-test.nix ({ pkgs, ...} : rec { - name = "kubernetes"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ offline ]; +with import ../lib/testing.nix { inherit system; }; +with import ../lib/qemu-flags.nix; +with pkgs.lib; + +let + redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON { + kind = "Pod"; + apiVersion = "v1"; + metadata.name = "redis"; + metadata.labels.name = "redis"; + spec.containers = [{ + name = "redis"; + image = "redis"; + args = ["--bind" "0.0.0.0"]; + imagePullPolicy = "Never"; + ports = [{ + name = "redis-server"; + containerPort = 6379; + }]; + }]; + }); + + redisService = pkgs.writeText "redis-service.json" (builtins.toJSON { + kind = "Service"; + apiVersion = "v1"; + metadata.name = "redis"; + spec = { + ports = [{port = 6379; targetPort = 6379;}]; + selector = {name = "redis";}; + }; + }); + + redisImage = pkgs.dockerTools.buildImage { + name = "redis"; + tag = "latest"; + contents = pkgs.redis; + config.Entrypoint = "/bin/redis-server"; }; - redisMaster = builtins.toFile "redis-master-pod.yaml" '' - id: redis-master-pod - kind: Pod - apiVersion: v1beta1 - desiredState: - manifest: - version: v1beta1 - id: redis-master-pod - containers: - - name: master - image: master:5000/nix - cpu: 100 - ports: - - name: redis-server - containerPort: 6379 - hostPort: 6379 - volumeMounts: - - name: nix-store - mountPath: /nix/store - readOnly: true - volumeMounts: - - name: system-profile - mountPath: /bin - readOnly: true - command: - - /bin/redis-server - volumes: - - name: nix-store - source: - hostDir: - path: /nix/store - - name: system-profile - source: - hostDir: - path: /run/current-system/sw/bin - labels: - name: redis - role: master + testSimplePod = '' + $kubernetes->execute("docker load < ${redisImage}"); + $kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}"); + $kubernetes->succeed("kubectl create -f ${redisService}"); + $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running"); + $kubernetes->succeed("nc -z \$\(dig \@10.10.0.1 redis.default.svc.cluster.local +short\) 6379"); ''; +in { + # This test runs kubernetes on a single node + trivial = makeTest { + name = "kubernetes-trivial"; - nodes = { - master = - { config, pkgs, lib, nodes, ... }: - { - virtualisation.memorySize = 768; - services.kubernetes = { - roles = ["master" "node"]; - dockerCfg = ''{"master:5000":{}}''; - controllerManager.machines = ["master" "node"]; - apiserver.address = "0.0.0.0"; - verbose = true; - }; - virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000"; + nodes = { + kubernetes = + { config, pkgs, lib, nodes, ... }: + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 2048; - services.etcd = { - listenPeerUrls = ["http://0.0.0.0:7001"]; - initialAdvertisePeerUrls = ["http://master:7001"]; - initialCluster = ["master=http://master:7001" "node=http://node:7001"]; - }; - services.dockerRegistry.enable = true; - services.dockerRegistry.host = "0.0.0.0"; - services.dockerRegistry.port = 5000; + programs.bash.enableCompletion = true; - virtualisation.vlans = [ 1 2 ]; - networking.bridges = { - cbr0.interfaces = [ "eth2" ]; - }; - networking.interfaces = { - cbr0 = { - ipAddress = "10.10.0.1"; - prefixLength = 24; - }; - eth2.ip4 = lib.mkOverride 0 [ ]; + services.kubernetes.roles = ["master" "node"]; + virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0"; + + networking.bridges.cbr0.interfaces = []; + networking.interfaces.cbr0 = {}; }; - networking.localCommands = '' - ip route add 10.10.0.0/16 dev cbr0 - ip route flush cache - ''; - networking.extraHosts = "127.0.0.1 master"; + }; + + testScript = '' + startAll; + + $kubernetes->waitUntilSucceeds("kubectl get nodes | grep kubernetes | grep Ready"); + + ${testSimplePod} + ''; + }; + + cluster = let + runWithOpenSSL = file: cmd: pkgs.runCommand file { + buildInputs = [ pkgs.openssl ]; + } cmd; + + ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048"; + ca_pem = runWithOpenSSL "ca.pem" '' + openssl req \ + -x509 -new -nodes -key ${ca_key} \ + -days 10000 -out $out -subj "/CN=etcd-ca" + ''; + etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048"; + etcd_csr = runWithOpenSSL "etcd.csr" '' + openssl req \ + -new -key ${etcd_key} \ + -out $out -subj "/CN=etcd" \ + -config ${openssl_cnf} + ''; + etcd_cert = runWithOpenSSL "etcd.pem" '' + openssl x509 \ + -req -in ${etcd_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} \ + -CAcreateserial -out $out \ + -days 365 -extensions v3_req \ + -extfile ${openssl_cnf} + ''; + + etcd_client_key = runWithOpenSSL "etcd-client-key.pem" + "openssl genrsa -out $out 2048"; + + etcd_client_csr = runWithOpenSSL "etcd-client-key.pem" '' + openssl req \ + -new -key ${etcd_client_key} \ + -out $out -subj "/CN=etcd-client" \ + -config ${client_openssl_cnf} + ''; + + etcd_client_cert = runWithOpenSSL "etcd-client.crt" '' + openssl x509 \ + -req -in ${etcd_client_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ + -out $out -days 365 -extensions v3_req \ + -extfile ${client_openssl_cnf} + ''; - networking.firewall.enable = false; - #networking.firewall.allowedTCPPorts = [ 4001 7001 ]; + apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048"; - environment.systemPackages = [ pkgs.redis ]; + apiserver_csr = runWithOpenSSL "apiserver.csr" '' + openssl req \ + -new -key ${apiserver_key} \ + -out $out -subj "/CN=kube-apiserver" \ + -config ${apiserver_cnf} + ''; + + apiserver_cert = runWithOpenSSL "apiserver.pem" '' + openssl x509 \ + -req -in ${apiserver_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ + -out $out -days 365 -extensions v3_req \ + -extfile ${apiserver_cnf} + ''; + + worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048"; + + worker_csr = runWithOpenSSL "worker.csr" '' + openssl req \ + -new -key ${worker_key} \ + -out $out -subj "/CN=kube-worker" \ + -config ${worker_cnf} + ''; + + worker_cert = runWithOpenSSL "worker.pem" '' + openssl x509 \ + -req -in ${worker_csr} \ + -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \ + -out $out -days 365 -extensions v3_req \ + -extfile ${worker_cnf} + ''; + + openssl_cnf = pkgs.writeText "openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = digitalSignature, keyEncipherment + extendedKeyUsage = serverAuth + subjectAltName = @alt_names + [alt_names] + DNS.1 = etcd1 + DNS.2 = etcd2 + DNS.3 = etcd3 + IP.1 = 127.0.0.1 + ''; + + client_openssl_cnf = pkgs.writeText "client-openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = digitalSignature, keyEncipherment + extendedKeyUsage = clientAuth + ''; + + apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + subjectAltName = @alt_names + [alt_names] + DNS.1 = kubernetes + DNS.2 = kubernetes.default + DNS.3 = kubernetes.default.svc + DNS.4 = kubernetes.default.svc.cluster.local + IP.1 = 10.10.10.1 + ''; + + worker_cnf = pkgs.writeText "worker-openssl.cnf" '' + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + subjectAltName = @alt_names + [alt_names] + DNS.1 = kubeWorker1 + DNS.2 = kubeWorker2 + ''; + + etcdNodeConfig = { + virtualisation.memorySize = 128; + + services = { + etcd = { + enable = true; + keyFile = etcd_key; + certFile = etcd_cert; + trustedCaFile = ca_pem; + peerClientCertAuth = true; + listenClientUrls = ["https://0.0.0.0:2379"]; + listenPeerUrls = ["https://0.0.0.0:2380"]; }; + }; - node = - { config, pkgs, lib, nodes, ... }: - { - services.kubernetes = { - roles = ["node"]; - dockerCfg = ''{"master:5000":{}}''; - kubelet.apiServers = ["master:8080"]; - verbose = true; - }; - virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000"; - services.etcd = { - listenPeerUrls = ["http://0.0.0.0:7001"]; - initialAdvertisePeerUrls = ["http://node:7001"]; - initialCluster = ["master=http://master:7001" "node=http://node:7001"]; - }; + environment.variables = { + ETCDCTL_CERT_FILE = "${etcd_client_cert}"; + ETCDCTL_KEY_FILE = "${etcd_client_key}"; + ETCDCTL_CA_FILE = "${ca_pem}"; + ETCDCTL_PEERS = "https://127.0.0.1:2379"; + }; - virtualisation.vlans = [ 1 2 ]; - networking.bridges = { - cbr0.interfaces = [ "eth2" ]; - }; - networking.interfaces = { - cbr0 = { - ipAddress = "10.10.1.1"; - prefixLength = 24; - }; - eth2.ip4 = lib.mkOverride 0 [ ]; - }; - networking.localCommands = '' - ip route add 10.10.0.0/16 dev cbr0 - ip route flush cache - ''; - networking.extraHosts = "127.0.0.1 node"; + networking.firewall.allowedTCPPorts = [ 2379 2380 ]; + }; - networking.firewall.enable = false; - #networking.firewall.allowedTCPPorts = [ 4001 7001 ]; + kubeConfig = { + virtualisation.diskSize = 2048; + programs.bash.enableCompletion = true; - environment.systemPackages = [ pkgs.redis ]; + services.flannel = { + enable = true; + network = "10.10.0.0/16"; + iface = "eth1"; + etcd = { + endpoints = ["https://etcd1:2379" "https://etcd2:2379" "https://etcd3:2379"]; + keyFile = etcd_client_key; + certFile = etcd_client_cert; + caFile = ca_pem; }; + }; + + # vxlan + networking.firewall.allowedUDPPorts = [ 8472 ]; - client = - { config, pkgs, nodes, ... }: - { - virtualisation.docker.enable = true; - virtualisation.docker.extraOptions = "--insecure-registry master:5000"; - environment.systemPackages = [ pkgs.kubernetes ]; - environment.etc."test/redis-master-pod.yaml".source = redisMaster; - environment.etc."test/pause".source = "${pkgs.kubernetes}/bin/kube-pause"; - environment.etc."test/Dockerfile".source = pkgs.writeText "Dockerfile" '' - FROM scratch - ADD pause / - ENTRYPOINT ["/pause"] - ''; + systemd.services.docker.after = ["flannel.service"]; + systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; + virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; + + services.kubernetes.verbose = true; + services.kubernetes.etcd = { + servers = ["https://etcd1:2379" "https://etcd2:2379" "https://etcd3:2379"]; + keyFile = etcd_client_key; + certFile = etcd_client_cert; + caFile = ca_pem; + }; + + environment.systemPackages = [ pkgs.bind pkgs.tcpdump pkgs.utillinux ]; + }; + + kubeMasterConfig = {pkgs, ...}: { + require = [kubeConfig]; + + # kube apiserver + networking.firewall.allowedTCPPorts = [ 443 ]; + + virtualisation.memorySize = 512; + + services.kubernetes = { + roles = ["master"]; + scheduler.leaderElect = true; + controllerManager.leaderElect = true; + + apiserver = { + publicAddress = "0.0.0.0"; + advertiseAddress = "192.168.1.8"; + tlsKeyFile = apiserver_key; + tlsCertFile = apiserver_cert; + clientCaFile = ca_pem; + kubeletClientCaFile = ca_pem; + kubeletClientKeyFile = worker_key; + kubeletClientCertFile = worker_cert; }; - }; + }; + }; - testScript = '' - startAll; - - $master->waitForUnit("kubernetes-apiserver.service"); - $master->waitForUnit("kubernetes-scheduler.service"); - $master->waitForUnit("kubernetes-controller-manager.service"); - $master->waitForUnit("kubernetes-kubelet.service"); - $master->waitForUnit("kubernetes-proxy.service"); - - $node->waitForUnit("kubernetes-kubelet.service"); - $node->waitForUnit("kubernetes-proxy.service"); - - $master->waitUntilSucceeds("kubectl get minions | grep master"); - $master->waitUntilSucceeds("kubectl get minions | grep node"); - - $client->waitForUnit("docker.service"); - $client->succeed("tar cv --files-from /dev/null | docker import - nix"); - $client->succeed("docker tag nix master:5000/nix"); - $master->waitForUnit("docker-registry.service"); - $client->succeed("docker push master:5000/nix"); - $client->succeed("mkdir -p /root/pause"); - $client->succeed("cp /etc/test/pause /root/pause/"); - $client->succeed("cp /etc/test/Dockerfile /root/pause/"); - $client->succeed("cd /root/pause && docker build -t master:5000/pause ."); - $client->succeed("docker push master:5000/pause"); - - subtest "simple pod", sub { - $client->succeed("kubectl create -f ${redisMaster} -s http://master:8080"); - $client->waitUntilSucceeds("kubectl get pods -s http://master:8080 | grep redis-master | grep -i running"); - } + kubeWorkerConfig = { pkgs, ... }: { + require = [kubeConfig]; - ''; -}) + virtualisation.memorySize = 512; + + # kubelet + networking.firewall.allowedTCPPorts = [ 10250 ]; + + services.kubernetes = { + roles = ["node"]; + kubeconfig = { + server = "https://kubernetes:443"; + caFile = ca_pem; + certFile = worker_cert; + keyFile = worker_key; + }; + kubelet = { + tlsKeyFile = worker_key; + tlsCertFile = worker_cert; + }; + }; + }; + in makeTest { + name = "kubernetes-cluster"; + + nodes = { + etcd1 = { config, pkgs, nodes, ... }: { + require = [etcdNodeConfig]; + services.etcd = { + advertiseClientUrls = ["https://etcd1:2379"]; + initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"]; + initialAdvertisePeerUrls = ["https://etcd1:2380"]; + }; + }; + + etcd2 = { config, pkgs, ... }: { + require = [etcdNodeConfig]; + services.etcd = { + advertiseClientUrls = ["https://etcd2:2379"]; + initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"]; + initialAdvertisePeerUrls = ["https://etcd2:2380"]; + }; + }; + + etcd3 = { config, pkgs, ... }: { + require = [etcdNodeConfig]; + services.etcd = { + advertiseClientUrls = ["https://etcd3:2379"]; + initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"]; + initialAdvertisePeerUrls = ["https://etcd3:2380"]; + }; + }; + + kubeMaster1 = { config, pkgs, lib, nodes, ... }: { + require = [kubeMasterConfig]; + }; + + kubeMaster2 = { config, pkgs, lib, nodes, ... }: { + require = [kubeMasterConfig]; + }; + + # Kubernetes TCP load balancer + kubernetes = { config, pkgs, ... }: { + # kubernetes + networking.firewall.allowedTCPPorts = [ 443 ]; + + services.haproxy.enable = true; + services.haproxy.config = '' + global + log 127.0.0.1 local0 notice + user haproxy + group haproxy + + defaults + log global + retries 2 + timeout connect 3000 + timeout server 5000 + timeout client 5000 + + listen kubernetes + bind 0.0.0.0:443 + mode tcp + option ssl-hello-chk + balance roundrobin + server kube-master-1 kubeMaster1:443 check + server kube-master-2 kubeMaster2:443 check + ''; + }; + + kubeWorker1 = { config, pkgs, lib, nodes, ... }: { + require = [kubeWorkerConfig]; + }; + + kubeWorker2 = { config, pkgs, lib, nodes, ... }: { + require = [kubeWorkerConfig]; + }; + }; + + testScript = '' + startAll; + + ${testSimplePod} + ''; + }; +} |