diff options
author | Allan Espinosa <allan.espinosa@outlook.com> | 2018-05-20 21:09:31 -0400 |
---|---|---|
committer | Allan Espinosa <allan.espinosa@outlook.com> | 2018-06-29 23:14:02 -0400 |
commit | 0c10b2baa6bf61c8ddaed7cdb6c2f2dbaab42662 (patch) | |
tree | ef0f30b3a0b9e1f6a6f29c5f3913f904f8f58a66 /nixos/tests/hadoop | |
parent | 978397d358ba94791bef736f19d3171a13bc03dc (diff) |
nixos/hadoop: add hadoop module (hdfs, yarn)
Diffstat (limited to 'nixos/tests/hadoop')
-rw-r--r-- | nixos/tests/hadoop/hdfs.nix | 54 | ||||
-rw-r--r-- | nixos/tests/hadoop/yarn.nix | 46 |
2 files changed, 100 insertions, 0 deletions
diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix new file mode 100644 index 000000000000..4206c940c1af --- /dev/null +++ b/nixos/tests/hadoop/hdfs.nix @@ -0,0 +1,54 @@ +import ../make-test.nix ({pkgs, ...}: { + nodes = { + namenode = {pkgs, config, ...}: { + services.hadoop = { + package = pkgs.hadoop_3_1; + hdfs.namenode.enabled = true; + coreSite = { + "fs.defaultFS" = "hdfs://namenode:8020"; + }; + hdfsSite = { + "dfs.replication" = 1; + "dfs.namenode.rpc-bind-host" = "0.0.0.0"; + "dfs.namenode.http-bind-host" = "0.0.0.0"; + }; + }; + networking.firewall.allowedTCPPorts = [ + 9870 # namenode.http-address + 8020 # namenode.rpc-address + ]; + }; + datanode = {pkgs, config, ...}: { + services.hadoop = { + package = pkgs.hadoop_3_1; + hdfs.datanode.enabled = true; + coreSite = { + "fs.defaultFS" = "hdfs://namenode:8020"; + }; + }; + networking.firewall.allowedTCPPorts = [ + 9864 # datanode.http.address + 9866 # datanode.address + 9867 # datanode.ipc.address + ]; + }; + }; + + testScript = '' + startAll + + $namenode->waitForUnit("hdfs-namenode"); + $namenode->waitForUnit("network.target"); + $namenode->waitForOpenPort(8020); + $namenode->waitForOpenPort(9870); + + $datanode->waitForUnit("hdfs-datanode"); + $datanode->waitForUnit("network.target"); + $datanode->waitForOpenPort(9864); + $datanode->waitForOpenPort(9866); + $datanode->waitForOpenPort(9867); + + $namenode->succeed("curl http://namenode:9870"); + $datanode->succeed("curl http://datanode:9864"); + ''; +}) diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix new file mode 100644 index 000000000000..e97cc1acc902 --- /dev/null +++ b/nixos/tests/hadoop/yarn.nix @@ -0,0 +1,46 @@ +import ../make-test.nix ({pkgs, ...}: { + nodes = { + resourcemanager = {pkgs, config, ...}: { + services.hadoop.package = pkgs.hadoop_3_1; + services.hadoop.yarn.resourcemanager.enabled = true; + services.hadoop.yarnSite = { + "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler"; + }; + networking.firewall.allowedTCPPorts = [ + 8088 # resourcemanager.webapp.address + 8031 # resourcemanager.resource-tracker.address + ]; + }; + nodemanager = {pkgs, config, ...}: { + services.hadoop.package = pkgs.hadoop_3_1; + services.hadoop.yarn.nodemanager.enabled = true; + services.hadoop.yarnSite = { + "yarn.resourcemanager.hostname" = "resourcemanager"; + "yarn.nodemanager.log-dirs" = "/tmp/userlogs"; + "yarn.nodemanager.address" = "0.0.0.0:8041"; + }; + networking.firewall.allowedTCPPorts = [ + 8042 # nodemanager.webapp.address + 8041 # nodemanager.address + ]; + }; + + }; + + testScript = '' + startAll; + + $resourcemanager->waitForUnit("yarn-resourcemanager"); + $resourcemanager->waitForUnit("network.target"); + $resourcemanager->waitForOpenPort(8031); + $resourcemanager->waitForOpenPort(8088); + + $nodemanager->waitForUnit("yarn-nodemanager"); + $nodemanager->waitForUnit("network.target"); + $nodemanager->waitForOpenPort(8042); + $nodemanager->waitForOpenPort(8041); + + $resourcemanager->succeed("curl http://localhost:8088"); + $nodemanager->succeed("curl http://localhost:8042"); + ''; +}) |