From fd184d63e92958f3ec4d9472d52d079fc800c153 Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 8 Jun 2022 04:26:03 +0200 Subject: syncstorage-rs: init at 0.11.0 --- pkgs/servers/syncstorage-rs/default.nix | 63 +++++++++++++++++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 ++ 2 files changed, 65 insertions(+) create mode 100644 pkgs/servers/syncstorage-rs/default.nix diff --git a/pkgs/servers/syncstorage-rs/default.nix b/pkgs/servers/syncstorage-rs/default.nix new file mode 100644 index 000000000000..17a7bb799fdf --- /dev/null +++ b/pkgs/servers/syncstorage-rs/default.nix @@ -0,0 +1,63 @@ +{ fetchFromGitHub +, rustPlatform +, pkg-config +, python3 +, openssl +, cmake +, libmysqlclient +, makeBinaryWrapper +, lib +}: + +let + pyFxADeps = python3.withPackages (p: [ + p.setuptools # imports pkg_resources + # remainder taken from requirements.txt + p.pyfxa + p.tokenlib + p.cryptography + ]); +in + +rustPlatform.buildRustPackage rec { + pname = "syncstorage-rs"; + version = "0.12.0"; + + src = fetchFromGitHub { + owner = "mozilla-services"; + repo = pname; + rev = version; + hash = "sha256-VfIpjpBS7LXe32fxIFp7xmbm40VwxUdHIEm5PnMpd4s="; + }; + + nativeBuildInputs = [ + cmake + makeBinaryWrapper + pkg-config + python3 + ]; + + buildInputs = [ + libmysqlclient + openssl + ]; + + preFixup = '' + wrapProgram $out/bin/syncstorage \ + --prefix PATH : ${lib.makeBinPath [ pyFxADeps ]} + ''; + + cargoSha256 = "sha256-JXxArKA/2SIYJvjNA1yZHR9xDKt3N2U7HVMP/6M3BxE="; + + buildFeatures = [ "grpcio/openssl" ]; + + # almost all tests need a DB to test against + doCheck = false; + + meta = { + description = "Mozilla Sync Storage built with Rust"; + homepage = "https://github.com/mozilla-services/syncstorage-rs"; + license = lib.licenses.mpl20; + maintainers = with lib.maintainers; [ pennae ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 72a7e29afb85..4c5ac95eed72 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -22153,6 +22153,8 @@ with pkgs; sympa = callPackage ../servers/mail/sympa { }; + syncstorage-rs = callPackage ../servers/syncstorage-rs { }; + system-sendmail = lowPrio (callPackage ../servers/mail/system-sendmail { }); # PulseAudio daemons -- cgit v1.2.3 From bd1978e911776c85370d853f324289b5c243670a Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 8 Jun 2022 04:26:34 +0200 Subject: nixos/firefox-syncserver: init --- .../from_md/release-notes/rl-2211.section.xml | 7 + nixos/doc/manual/release-notes/rl-2211.section.md | 1 + nixos/modules/module-list.nix | 1 + .../services/networking/firefox-syncserver.md | 55 ++++ .../services/networking/firefox-syncserver.nix | 328 +++++++++++++++++++++ .../services/networking/firefox-syncserver.xml | 77 +++++ 6 files changed, 469 insertions(+) create mode 100644 nixos/modules/services/networking/firefox-syncserver.md create mode 100644 nixos/modules/services/networking/firefox-syncserver.nix create mode 100644 nixos/modules/services/networking/firefox-syncserver.xml diff --git a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml index 5c29f98b28df..a4f312f6fa63 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml @@ -48,6 +48,13 @@ virtualisation.appvm. + + + syncstorage-rs, + a self-hostable sync server for Firefox. Available as + services.firefox-syncserver. + + dragonflydb, diff --git a/nixos/doc/manual/release-notes/rl-2211.section.md b/nixos/doc/manual/release-notes/rl-2211.section.md index 624bde2c83d7..9a2db0de3fa4 100644 --- a/nixos/doc/manual/release-notes/rl-2211.section.md +++ b/nixos/doc/manual/release-notes/rl-2211.section.md @@ -24,6 +24,7 @@ In addition to numerous new and upgraded packages, this release has the followin ## New Services {#sec-release-22.11-new-services} - [appvm](https://github.com/jollheef/appvm), Nix based app VMs. Available as [virtualisation.appvm](options.html#opt-virtualisation.appvm.enable). +- [syncstorage-rs](https://github.com/mozilla-services/syncstorage-rs), a self-hostable sync server for Firefox. Available as [services.firefox-syncserver](options.html#opt-services.firefox-syncserver.enable). - [dragonflydb](https://dragonflydb.io/), a modern replacement for Redis and Memcached. Available as [services.dragonflydb](#opt-services.dragonflydb.enable). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index d67602a26761..5c4c217e0a3f 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -773,6 +773,7 @@ ./services/networking/eternal-terminal.nix ./services/networking/fakeroute.nix ./services/networking/ferm.nix + ./services/networking/firefox-syncserver.nix ./services/networking/fireqos.nix ./services/networking/firewall.nix ./services/networking/flannel.nix diff --git a/nixos/modules/services/networking/firefox-syncserver.md b/nixos/modules/services/networking/firefox-syncserver.md new file mode 100644 index 000000000000..3ee863343ece --- /dev/null +++ b/nixos/modules/services/networking/firefox-syncserver.md @@ -0,0 +1,55 @@ +# Firefox Sync server {#module-services-firefox-syncserver} + +A storage server for Firefox Sync that you can easily host yourself. + +## Quickstart {#module-services-firefox-syncserver-quickstart} + +The absolute minimal configuration for the sync server looks like this: + +```nix +services.mysql.package = pkgs.mariadb; + +services.firefox-syncserver = { + enable = true; + secrets = builtins.toFile "sync-secrets" '' + SYNC_MASTER_SECRET=this-secret-is-actually-leaked-to-/nix/store + ''; + singleNode = { + enable = true; + hostname = "localhost"; + url = "http://localhost:5000"; + }; +}; +``` + +This will start a sync server that is only accessible locally. Once the services is +running you can navigate to `about:config` in your Firefox profile and set +`identity.sync.tokenserver.uri` to `http://localhost:5000/1.0/sync/1.5`. Your browser +will now use your local sync server for data storage. + +::: {.warning} +This configuration should never be used in production. It is not encrypted and +stores its secrets in a world-readable location. +::: + +## More detailed setup {#module-services-firefox-syncserver-configuration} + +The `firefox-syncserver` service provides a number of options to make setting up +small deployment easier. These are grouped under the `singleNode` element of the +option tree and allow simple configuration of the most important parameters. + +Single node setup is split into two kinds of options: those that affect the sync +server itself, and those that affect its surroundings. Options that affect the +sync server are `capacity`, which configures how many accounts may be active on +this instance, and `url`, which holds the URL under which the sync server can be +accessed. The `url` can be configured automatically when using nginx. + +Options that affect the surroundings of the sync server are `enableNginx`, +`enableTLS` and `hostnam`. If `enableNginx` is set the sync server module will +automatically add an nginx virtual host to the system using `hostname` as the +domain and set `url` accordingly. If `enableTLS` is set the module will also +enable ACME certificates on the new virtual host and force all connections to +be made via TLS. + +For actual deployment it is also recommended to store the `secrets` file in a +secure location. diff --git a/nixos/modules/services/networking/firefox-syncserver.nix b/nixos/modules/services/networking/firefox-syncserver.nix new file mode 100644 index 000000000000..254d5c1dc670 --- /dev/null +++ b/nixos/modules/services/networking/firefox-syncserver.nix @@ -0,0 +1,328 @@ +{ config, pkgs, lib, options, ... }: + +let + cfg = config.services.firefox-syncserver; + opt = options.services.firefox-syncserver; + defaultDatabase = "firefox_syncserver"; + defaultUser = "firefox-syncserver"; + + dbIsLocal = cfg.database.host == "localhost"; + dbURL = "mysql://${cfg.database.user}@${cfg.database.host}/${cfg.database.name}"; + + format = pkgs.formats.toml {}; + settings = { + database_url = dbURL; + human_logs = true; + tokenserver = { + node_type = "mysql"; + database_url = dbURL; + fxa_email_domain = "api.accounts.firefox.com"; + fxa_oauth_server_url = "https://oauth.accounts.firefox.com/v1"; + run_migrations = true; + } // lib.optionalAttrs cfg.singleNode.enable { + # Single-node mode is likely to be used on small instances with little + # capacity. The default value (0.1) can only ever release capacity when + # accounts are removed if the total capacity is 10 or larger to begin + # with. + # https://github.com/mozilla-services/syncstorage-rs/issues/1313#issuecomment-1145293375 + node_capacity_release_rate = 1; + }; + }; + configFile = format.generate "syncstorage.toml" (lib.recursiveUpdate settings cfg.settings); +in + +{ + options = { + services.firefox-syncserver = { + enable = lib.mkEnableOption '' + the Firefox Sync storage service. + + Out of the box this will not be very useful unless you also configure at least + one service and one nodes by inserting them into the mysql database manually, e.g. + by running + + + INSERT INTO `services` (`id`, `service`, `pattern`) VALUES ('1', 'sync-1.5', '{node}/1.5/{uid}'); + INSERT INTO `nodes` (`id`, `service`, `node`, `available`, `current_load`, + `capacity`, `downed`, `backoff`) + VALUES ('1', '1', 'https://mydomain.tld', '1', '0', '10', '0', '0'); + + + does this automatically when enabled + ''; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.syncstorage-rs; + defaultText = lib.literalExpression "pkgs.syncstorage-rs"; + description = '' + Package to use. + ''; + }; + + database.name = lib.mkOption { + # the mysql module does not allow `-quoting without resorting to shell + # escaping, so we restrict db names for forward compaitiblity should this + # behavior ever change. + type = lib.types.strMatching "[a-z_][a-z0-9_]*"; + default = defaultDatabase; + description = '' + Database to use for storage. Will be created automatically if it does not exist + and config.${opt.database.createLocally} is set. + ''; + }; + + database.user = lib.mkOption { + type = lib.types.str; + default = defaultUser; + description = '' + Username for database connections. + ''; + }; + + database.host = lib.mkOption { + type = lib.types.str; + default = "localhost"; + description = '' + Database host name. localhost is treated specially and inserts + systemd dependencies, other hostnames or IP addresses of the local machine do not. + ''; + }; + + database.createLocally = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether to create database and user on the local machine if they do not exist. + This includes enabling unix domain socket authentication for the configured user. + ''; + }; + + logLevel = lib.mkOption { + type = lib.types.str; + default = "error"; + description = '' + Log level to run with. This can be a simple log level like error + or trace, or a more complicated logging expression. + ''; + }; + + secrets = lib.mkOption { + type = lib.types.path; + description = '' + A file containing the various secrets. Should be in the format expected by systemd's + EnvironmentFile directory. Two secrets are currently available: + SYNC_MASTER_SECRET and + SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET. + ''; + }; + + singleNode = { + enable = lib.mkEnableOption "auto-configuration for a simple single-node setup"; + + enableTLS = lib.mkEnableOption "automatic TLS setup"; + + enableNginx = lib.mkEnableOption "nginx virtualhost definitions"; + + hostname = lib.mkOption { + type = lib.types.str; + description = '' + Host name to use for this service. + ''; + }; + + capacity = lib.mkOption { + type = lib.types.ints.unsigned; + default = 10; + description = '' + How many sync accounts are allowed on this server. Setting this value + equal to or less than the number of currently active accounts will + effectively deny service to accounts not yet registered here. + ''; + }; + + url = lib.mkOption { + type = lib.types.str; + default = "${if cfg.singleNode.enableTLS then "https" else "http"}://${cfg.singleNode.hostname}"; + defaultText = lib.literalExpression '' + ''${if cfg.singleNode.enableTLS then "https" else "http"}://''${config.${opt.singleNode.hostname}} + ''; + description = '' + URL of the host. If you are not using the automatic webserver proxy setup you will have + to change this setting or your sync server may not be functional. + ''; + }; + }; + + settings = lib.mkOption { + type = lib.types.submodule { + freeformType = format.type; + + options = { + port = lib.mkOption { + type = lib.types.port; + default = 5000; + description = '' + Port to bind to. + ''; + }; + + tokenserver.enabled = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether to enable the token service as well. + ''; + }; + }; + }; + default = { }; + description = '' + Settings for the sync server. These take priority over values computed + from NixOS options. + + See the doc comments on the Settings structs in + + and + + for available options. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + services.mysql = lib.mkIf cfg.database.createLocally { + enable = true; + ensureDatabases = [ cfg.database.name ]; + ensureUsers = [{ + name = cfg.database.user; + ensurePermissions = { + "${cfg.database.name}.*" = "all privileges"; + }; + }]; + }; + + systemd.services.firefox-syncserver = { + wantedBy = [ "multi-user.target" ]; + requires = lib.mkIf dbIsLocal [ "mysql.service" ]; + after = lib.mkIf dbIsLocal [ "mysql.service" ]; + environment.RUST_LOG = cfg.logLevel; + serviceConfig = { + User = defaultUser; + Group = defaultUser; + ExecStart = "${cfg.package}/bin/syncstorage --config ${configFile}"; + Stderr = "journal"; + EnvironmentFile = lib.mkIf (cfg.secrets != null) "${cfg.secrets}"; + + # hardening + RemoveIPC = true; + CapabilityBoundingSet = [ "" ]; + DynamicUser = true; + NoNewPrivileges = true; + PrivateDevices = true; + ProtectClock = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + # syncstorage-rs uses python-cffi internally, and python-cffi does not + # work with MemoryDenyWriteExecute=true + MemoryDenyWriteExecute = false; + RestrictNamespaces = true; + RestrictSUIDSGID = true; + ProtectHostname = true; + LockPersonality = true; + ProtectKernelTunables = true; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + RestrictRealtime = true; + ProtectSystem = "strict"; + ProtectProc = "invisible"; + ProcSubset = "pid"; + ProtectHome = true; + PrivateUsers = true; + PrivateTmp = true; + SystemCallFilter = [ "@system-service" "~ @privileged @resources" ]; + UMask = "0077"; + }; + }; + + systemd.services.firefox-syncserver-setup = lib.mkIf cfg.singleNode.enable { + wantedBy = [ "firefox-syncserver.service" ]; + requires = [ "firefox-syncserver.service" ] ++ lib.optional dbIsLocal "mysql.service"; + after = [ "firefox-syncserver.service" ] ++ lib.optional dbIsLocal "mysql.service"; + path = [ config.services.mysql.package ]; + script = '' + set -euo pipefail + shopt -s inherit_errexit + + schema_configured() { + mysql ${cfg.database.name} -Ne 'SHOW TABLES' | grep -q services + } + + services_configured() { + [ 1 != $(mysql ${cfg.database.name} -Ne 'SELECT COUNT(*) < 1 FROM `services`') ] + } + + create_services() { + mysql ${cfg.database.name} <<"EOF" + BEGIN; + + INSERT INTO `services` (`id`, `service`, `pattern`) + VALUES (1, 'sync-1.5', '{node}/1.5/{uid}'); + INSERT INTO `nodes` (`id`, `service`, `node`, `available`, `current_load`, + `capacity`, `downed`, `backoff`) + VALUES (1, 1, '${cfg.singleNode.url}', ${toString cfg.singleNode.capacity}, + 0, ${toString cfg.singleNode.capacity}, 0, 0); + + COMMIT; + EOF + } + + update_nodes() { + mysql ${cfg.database.name} <<"EOF" + UPDATE `nodes` + SET `capacity` = ${toString cfg.singleNode.capacity} + WHERE `id` = 1; + EOF + } + + for (( try = 0; try < 60; try++ )); do + if ! schema_configured; then + sleep 2 + elif services_configured; then + update_nodes + exit 0 + else + create_services + exit 0 + fi + done + + echo "Single-node setup failed" + exit 1 + ''; + }; + + services.nginx.virtualHosts = lib.mkIf cfg.singleNode.enableNginx { + ${cfg.singleNode.hostname} = { + enableACME = cfg.singleNode.enableTLS; + forceSSL = cfg.singleNode.enableTLS; + locations."/" = { + proxyPass = "http://localhost:${toString cfg.settings.port}"; + # source mentions that this header should be set + extraConfig = '' + add_header X-Content-Type-Options nosniff; + ''; + }; + }; + }; + }; + + meta = { + maintainers = with lib.maintainers; [ pennae ]; + # Don't edit the docbook xml directly, edit the md and generate it: + # `pandoc firefox-syncserver.md -t docbook --top-level-division=chapter --extract-media=media -f markdown+smart > firefox-syncserver.xml` + doc = ./firefox-syncserver.xml; + }; +} diff --git a/nixos/modules/services/networking/firefox-syncserver.xml b/nixos/modules/services/networking/firefox-syncserver.xml new file mode 100644 index 000000000000..66c812266951 --- /dev/null +++ b/nixos/modules/services/networking/firefox-syncserver.xml @@ -0,0 +1,77 @@ + + Firefox Sync server + + A storage server for Firefox Sync that you can easily host yourself. + +
+ Quickstart + + The absolute minimal configuration for the sync server looks like + this: + + +services.mysql.package = pkgs.mariadb; + +services.firefox-syncserver = { + enable = true; + secrets = builtins.toFile "sync-secrets" '' + SYNC_MASTER_SECRET=this-secret-is-actually-leaked-to-/nix/store + ''; + singleNode = { + enable = true; + hostname = "localhost"; + url = "http://localhost:5000"; + }; +}; + + + This will start a sync server that is only accessible locally. + Once the services is running you can navigate to + about:config in your Firefox profile and set + identity.sync.tokenserver.uri to + http://localhost:5000/1.0/sync/1.5. Your + browser will now use your local sync server for data storage. + + + + This configuration should never be used in production. It is not + encrypted and stores its secrets in a world-readable location. + + +
+
+ More detailed setup + + The firefox-syncserver service provides a + number of options to make setting up small deployment easier. + These are grouped under the singleNode element + of the option tree and allow simple configuration of the most + important parameters. + + + Single node setup is split into two kinds of options: those that + affect the sync server itself, and those that affect its + surroundings. Options that affect the sync server are + capacity, which configures how many accounts + may be active on this instance, and url, which + holds the URL under which the sync server can be accessed. The + url can be configured automatically when using + nginx. + + + Options that affect the surroundings of the sync server are + enableNginx, enableTLS and + hostnam. If enableNginx is + set the sync server module will automatically add an nginx virtual + host to the system using hostname as the domain + and set url accordingly. If + enableTLS is set the module will also enable + ACME certificates on the new virtual host and force all + connections to be made via TLS. + + + For actual deployment it is also recommended to store the + secrets file in a secure location. + +
+ -- cgit v1.2.3