diff options
author | Martin Weinelt <mweinelt@users.noreply.github.com> | 2022-08-12 01:33:26 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-12 01:33:26 +0200 |
commit | a58668f0a0738be43c4b6c4b7c95e5b61e0122c3 (patch) | |
tree | ed546cbc1727e2cfce71b3d6af823bd2635d8432 | |
parent | 9e0d0d4751bb268316e704216ffa4d451c41f0c7 (diff) | |
parent | bd1978e911776c85370d853f324289b5c243670a (diff) |
Merge pull request #176835 from pennae/syncserver
-rw-r--r-- | nixos/doc/manual/from_md/release-notes/rl-2211.section.xml | 7 | ||||
-rw-r--r-- | nixos/doc/manual/release-notes/rl-2211.section.md | 1 | ||||
-rw-r--r-- | nixos/modules/module-list.nix | 1 | ||||
-rw-r--r-- | nixos/modules/services/networking/firefox-syncserver.md | 55 | ||||
-rw-r--r-- | nixos/modules/services/networking/firefox-syncserver.nix | 328 | ||||
-rw-r--r-- | nixos/modules/services/networking/firefox-syncserver.xml | 77 | ||||
-rw-r--r-- | pkgs/servers/syncstorage-rs/default.nix | 63 | ||||
-rw-r--r-- | pkgs/top-level/all-packages.nix | 2 |
8 files changed, 534 insertions, 0 deletions
diff --git a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml index aaa85138dfa1..87066e6cdf26 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml @@ -126,6 +126,13 @@ </listitem> <listitem> <para> + <link xlink:href="https://github.com/mozilla-services/syncstorage-rs">syncstorage-rs</link>, + a self-hostable sync server for Firefox. Available as + <link xlink:href="options.html#opt-services.firefox-syncserver.enable">services.firefox-syncserver</link>. + </para> + </listitem> + <listitem> + <para> <link xlink:href="https://dragonflydb.io/">dragonflydb</link>, a modern replacement for Redis and Memcached. Available as <link linkend="opt-services.dragonflydb.enable">services.dragonflydb</link>. diff --git a/nixos/doc/manual/release-notes/rl-2211.section.md b/nixos/doc/manual/release-notes/rl-2211.section.md index e1253d46190f..be0d17f9054d 100644 --- a/nixos/doc/manual/release-notes/rl-2211.section.md +++ b/nixos/doc/manual/release-notes/rl-2211.section.md @@ -55,6 +55,7 @@ In addition to numerous new and upgraded packages, this release has the followin ## New Services {#sec-release-22.11-new-services} - [appvm](https://github.com/jollheef/appvm), Nix based app VMs. Available as [virtualisation.appvm](options.html#opt-virtualisation.appvm.enable). +- [syncstorage-rs](https://github.com/mozilla-services/syncstorage-rs), a self-hostable sync server for Firefox. Available as [services.firefox-syncserver](options.html#opt-services.firefox-syncserver.enable). - [dragonflydb](https://dragonflydb.io/), a modern replacement for Redis and Memcached. Available as [services.dragonflydb](#opt-services.dragonflydb.enable). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 4c51210bc200..6e979561fa03 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -783,6 +783,7 @@ ./services/networking/expressvpn.nix ./services/networking/fakeroute.nix ./services/networking/ferm.nix + ./services/networking/firefox-syncserver.nix ./services/networking/fireqos.nix ./services/networking/firewall.nix ./services/networking/flannel.nix diff --git a/nixos/modules/services/networking/firefox-syncserver.md b/nixos/modules/services/networking/firefox-syncserver.md new file mode 100644 index 000000000000..3ee863343ece --- /dev/null +++ b/nixos/modules/services/networking/firefox-syncserver.md @@ -0,0 +1,55 @@ +# Firefox Sync server {#module-services-firefox-syncserver} + +A storage server for Firefox Sync that you can easily host yourself. + +## Quickstart {#module-services-firefox-syncserver-quickstart} + +The absolute minimal configuration for the sync server looks like this: + +```nix +services.mysql.package = pkgs.mariadb; + +services.firefox-syncserver = { + enable = true; + secrets = builtins.toFile "sync-secrets" '' + SYNC_MASTER_SECRET=this-secret-is-actually-leaked-to-/nix/store + ''; + singleNode = { + enable = true; + hostname = "localhost"; + url = "http://localhost:5000"; + }; +}; +``` + +This will start a sync server that is only accessible locally. Once the services is +running you can navigate to `about:config` in your Firefox profile and set +`identity.sync.tokenserver.uri` to `http://localhost:5000/1.0/sync/1.5`. Your browser +will now use your local sync server for data storage. + +::: {.warning} +This configuration should never be used in production. It is not encrypted and +stores its secrets in a world-readable location. +::: + +## More detailed setup {#module-services-firefox-syncserver-configuration} + +The `firefox-syncserver` service provides a number of options to make setting up +small deployment easier. These are grouped under the `singleNode` element of the +option tree and allow simple configuration of the most important parameters. + +Single node setup is split into two kinds of options: those that affect the sync +server itself, and those that affect its surroundings. Options that affect the +sync server are `capacity`, which configures how many accounts may be active on +this instance, and `url`, which holds the URL under which the sync server can be +accessed. The `url` can be configured automatically when using nginx. + +Options that affect the surroundings of the sync server are `enableNginx`, +`enableTLS` and `hostnam`. If `enableNginx` is set the sync server module will +automatically add an nginx virtual host to the system using `hostname` as the +domain and set `url` accordingly. If `enableTLS` is set the module will also +enable ACME certificates on the new virtual host and force all connections to +be made via TLS. + +For actual deployment it is also recommended to store the `secrets` file in a +secure location. diff --git a/nixos/modules/services/networking/firefox-syncserver.nix b/nixos/modules/services/networking/firefox-syncserver.nix new file mode 100644 index 000000000000..254d5c1dc670 --- /dev/null +++ b/nixos/modules/services/networking/firefox-syncserver.nix @@ -0,0 +1,328 @@ +{ config, pkgs, lib, options, ... }: + +let + cfg = config.services.firefox-syncserver; + opt = options.services.firefox-syncserver; + defaultDatabase = "firefox_syncserver"; + defaultUser = "firefox-syncserver"; + + dbIsLocal = cfg.database.host == "localhost"; + dbURL = "mysql://${cfg.database.user}@${cfg.database.host}/${cfg.database.name}"; + + format = pkgs.formats.toml {}; + settings = { + database_url = dbURL; + human_logs = true; + tokenserver = { + node_type = "mysql"; + database_url = dbURL; + fxa_email_domain = "api.accounts.firefox.com"; + fxa_oauth_server_url = "https://oauth.accounts.firefox.com/v1"; + run_migrations = true; + } // lib.optionalAttrs cfg.singleNode.enable { + # Single-node mode is likely to be used on small instances with little + # capacity. The default value (0.1) can only ever release capacity when + # accounts are removed if the total capacity is 10 or larger to begin + # with. + # https://github.com/mozilla-services/syncstorage-rs/issues/1313#issuecomment-1145293375 + node_capacity_release_rate = 1; + }; + }; + configFile = format.generate "syncstorage.toml" (lib.recursiveUpdate settings cfg.settings); +in + +{ + options = { + services.firefox-syncserver = { + enable = lib.mkEnableOption '' + the Firefox Sync storage service. + + Out of the box this will not be very useful unless you also configure at least + one service and one nodes by inserting them into the mysql database manually, e.g. + by running + + <programlisting> + INSERT INTO `services` (`id`, `service`, `pattern`) VALUES ('1', 'sync-1.5', '{node}/1.5/{uid}'); + INSERT INTO `nodes` (`id`, `service`, `node`, `available`, `current_load`, + `capacity`, `downed`, `backoff`) + VALUES ('1', '1', 'https://mydomain.tld', '1', '0', '10', '0', '0'); + </programlisting> + + <option>${opt.singleNode.enable}</option> does this automatically when enabled + ''; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.syncstorage-rs; + defaultText = lib.literalExpression "pkgs.syncstorage-rs"; + description = '' + Package to use. + ''; + }; + + database.name = lib.mkOption { + # the mysql module does not allow `-quoting without resorting to shell + # escaping, so we restrict db names for forward compaitiblity should this + # behavior ever change. + type = lib.types.strMatching "[a-z_][a-z0-9_]*"; + default = defaultDatabase; + description = '' + Database to use for storage. Will be created automatically if it does not exist + and <literal>config.${opt.database.createLocally}</literal> is set. + ''; + }; + + database.user = lib.mkOption { + type = lib.types.str; + default = defaultUser; + description = '' + Username for database connections. + ''; + }; + + database.host = lib.mkOption { + type = lib.types.str; + default = "localhost"; + description = '' + Database host name. <literal>localhost</literal> is treated specially and inserts + systemd dependencies, other hostnames or IP addresses of the local machine do not. + ''; + }; + + database.createLocally = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether to create database and user on the local machine if they do not exist. + This includes enabling unix domain socket authentication for the configured user. + ''; + }; + + logLevel = lib.mkOption { + type = lib.types.str; + default = "error"; + description = '' + Log level to run with. This can be a simple log level like <literal>error</literal> + or <literal>trace</literal>, or a more complicated logging expression. + ''; + }; + + secrets = lib.mkOption { + type = lib.types.path; + description = '' + A file containing the various secrets. Should be in the format expected by systemd's + <literal>EnvironmentFile</literal> directory. Two secrets are currently available: + <literal>SYNC_MASTER_SECRET</literal> and + <literal>SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET</literal>. + ''; + }; + + singleNode = { + enable = lib.mkEnableOption "auto-configuration for a simple single-node setup"; + + enableTLS = lib.mkEnableOption "automatic TLS setup"; + + enableNginx = lib.mkEnableOption "nginx virtualhost definitions"; + + hostname = lib.mkOption { + type = lib.types.str; + description = '' + Host name to use for this service. + ''; + }; + + capacity = lib.mkOption { + type = lib.types.ints.unsigned; + default = 10; + description = '' + How many sync accounts are allowed on this server. Setting this value + equal to or less than the number of currently active accounts will + effectively deny service to accounts not yet registered here. + ''; + }; + + url = lib.mkOption { + type = lib.types.str; + default = "${if cfg.singleNode.enableTLS then "https" else "http"}://${cfg.singleNode.hostname}"; + defaultText = lib.literalExpression '' + ''${if cfg.singleNode.enableTLS then "https" else "http"}://''${config.${opt.singleNode.hostname}} + ''; + description = '' + URL of the host. If you are not using the automatic webserver proxy setup you will have + to change this setting or your sync server may not be functional. + ''; + }; + }; + + settings = lib.mkOption { + type = lib.types.submodule { + freeformType = format.type; + + options = { + port = lib.mkOption { + type = lib.types.port; + default = 5000; + description = '' + Port to bind to. + ''; + }; + + tokenserver.enabled = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether to enable the token service as well. + ''; + }; + }; + }; + default = { }; + description = '' + Settings for the sync server. These take priority over values computed + from NixOS options. + + See the doc comments on the <literal>Settings</literal> structs in + <link xlink:href="https://github.com/mozilla-services/syncstorage-rs/blob/master/syncstorage/src/settings.rs" /> + and + <link xlink:href="https://github.com/mozilla-services/syncstorage-rs/blob/master/syncstorage/src/tokenserver/settings.rs" /> + for available options. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + services.mysql = lib.mkIf cfg.database.createLocally { + enable = true; + ensureDatabases = [ cfg.database.name ]; + ensureUsers = [{ + name = cfg.database.user; + ensurePermissions = { + "${cfg.database.name}.*" = "all privileges"; + }; + }]; + }; + + systemd.services.firefox-syncserver = { + wantedBy = [ "multi-user.target" ]; + requires = lib.mkIf dbIsLocal [ "mysql.service" ]; + after = lib.mkIf dbIsLocal [ "mysql.service" ]; + environment.RUST_LOG = cfg.logLevel; + serviceConfig = { + User = defaultUser; + Group = defaultUser; + ExecStart = "${cfg.package}/bin/syncstorage --config ${configFile}"; + Stderr = "journal"; + EnvironmentFile = lib.mkIf (cfg.secrets != null) "${cfg.secrets}"; + + # hardening + RemoveIPC = true; + CapabilityBoundingSet = [ "" ]; + DynamicUser = true; + NoNewPrivileges = true; + PrivateDevices = true; + ProtectClock = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + # syncstorage-rs uses python-cffi internally, and python-cffi does not + # work with MemoryDenyWriteExecute=true + MemoryDenyWriteExecute = false; + RestrictNamespaces = true; + RestrictSUIDSGID = true; + ProtectHostname = true; + LockPersonality = true; + ProtectKernelTunables = true; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + RestrictRealtime = true; + ProtectSystem = "strict"; + ProtectProc = "invisible"; + ProcSubset = "pid"; + ProtectHome = true; + PrivateUsers = true; + PrivateTmp = true; + SystemCallFilter = [ "@system-service" "~ @privileged @resources" ]; + UMask = "0077"; + }; + }; + + systemd.services.firefox-syncserver-setup = lib.mkIf cfg.singleNode.enable { + wantedBy = [ "firefox-syncserver.service" ]; + requires = [ "firefox-syncserver.service" ] ++ lib.optional dbIsLocal "mysql.service"; + after = [ "firefox-syncserver.service" ] ++ lib.optional dbIsLocal "mysql.service"; + path = [ config.services.mysql.package ]; + script = '' + set -euo pipefail + shopt -s inherit_errexit + + schema_configured() { + mysql ${cfg.database.name} -Ne 'SHOW TABLES' | grep -q services + } + + services_configured() { + [ 1 != $(mysql ${cfg.database.name} -Ne 'SELECT COUNT(*) < 1 FROM `services`') ] + } + + create_services() { + mysql ${cfg.database.name} <<"EOF" + BEGIN; + + INSERT INTO `services` (`id`, `service`, `pattern`) + VALUES (1, 'sync-1.5', '{node}/1.5/{uid}'); + INSERT INTO `nodes` (`id`, `service`, `node`, `available`, `current_load`, + `capacity`, `downed`, `backoff`) + VALUES (1, 1, '${cfg.singleNode.url}', ${toString cfg.singleNode.capacity}, + 0, ${toString cfg.singleNode.capacity}, 0, 0); + + COMMIT; + EOF + } + + update_nodes() { + mysql ${cfg.database.name} <<"EOF" + UPDATE `nodes` + SET `capacity` = ${toString cfg.singleNode.capacity} + WHERE `id` = 1; + EOF + } + + for (( try = 0; try < 60; try++ )); do + if ! schema_configured; then + sleep 2 + elif services_configured; then + update_nodes + exit 0 + else + create_services + exit 0 + fi + done + + echo "Single-node setup failed" + exit 1 + ''; + }; + + services.nginx.virtualHosts = lib.mkIf cfg.singleNode.enableNginx { + ${cfg.singleNode.hostname} = { + enableACME = cfg.singleNode.enableTLS; + forceSSL = cfg.singleNode.enableTLS; + locations."/" = { + proxyPass = "http://localhost:${toString cfg.settings.port}"; + # source mentions that this header should be set + extraConfig = '' + add_header X-Content-Type-Options nosniff; + ''; + }; + }; + }; + }; + + meta = { + maintainers = with lib.maintainers; [ pennae ]; + # Don't edit the docbook xml directly, edit the md and generate it: + # `pandoc firefox-syncserver.md -t docbook --top-level-division=chapter --extract-media=media -f markdown+smart > firefox-syncserver.xml` + doc = ./firefox-syncserver.xml; + }; +} diff --git a/nixos/modules/services/networking/firefox-syncserver.xml b/nixos/modules/services/networking/firefox-syncserver.xml new file mode 100644 index 000000000000..66c812266951 --- /dev/null +++ b/nixos/modules/services/networking/firefox-syncserver.xml @@ -0,0 +1,77 @@ +<chapter xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="module-services-firefox-syncserver"> + <title>Firefox Sync server</title> + <para> + A storage server for Firefox Sync that you can easily host yourself. + </para> + <section xml:id="module-services-firefox-syncserver-quickstart"> + <title>Quickstart</title> + <para> + The absolute minimal configuration for the sync server looks like + this: + </para> + <programlisting language="nix"> +services.mysql.package = pkgs.mariadb; + +services.firefox-syncserver = { + enable = true; + secrets = builtins.toFile "sync-secrets" '' + SYNC_MASTER_SECRET=this-secret-is-actually-leaked-to-/nix/store + ''; + singleNode = { + enable = true; + hostname = "localhost"; + url = "http://localhost:5000"; + }; +}; +</programlisting> + <para> + This will start a sync server that is only accessible locally. + Once the services is running you can navigate to + <literal>about:config</literal> in your Firefox profile and set + <literal>identity.sync.tokenserver.uri</literal> to + <literal>http://localhost:5000/1.0/sync/1.5</literal>. Your + browser will now use your local sync server for data storage. + </para> + <warning> + <para> + This configuration should never be used in production. It is not + encrypted and stores its secrets in a world-readable location. + </para> + </warning> + </section> + <section xml:id="module-services-firefox-syncserver-configuration"> + <title>More detailed setup</title> + <para> + The <literal>firefox-syncserver</literal> service provides a + number of options to make setting up small deployment easier. + These are grouped under the <literal>singleNode</literal> element + of the option tree and allow simple configuration of the most + important parameters. + </para> + <para> + Single node setup is split into two kinds of options: those that + affect the sync server itself, and those that affect its + surroundings. Options that affect the sync server are + <literal>capacity</literal>, which configures how many accounts + may be active on this instance, and <literal>url</literal>, which + holds the URL under which the sync server can be accessed. The + <literal>url</literal> can be configured automatically when using + nginx. + </para> + <para> + Options that affect the surroundings of the sync server are + <literal>enableNginx</literal>, <literal>enableTLS</literal> and + <literal>hostnam</literal>. If <literal>enableNginx</literal> is + set the sync server module will automatically add an nginx virtual + host to the system using <literal>hostname</literal> as the domain + and set <literal>url</literal> accordingly. If + <literal>enableTLS</literal> is set the module will also enable + ACME certificates on the new virtual host and force all + connections to be made via TLS. + </para> + <para> + For actual deployment it is also recommended to store the + <literal>secrets</literal> file in a secure location. + </para> + </section> +</chapter> diff --git a/pkgs/servers/syncstorage-rs/default.nix b/pkgs/servers/syncstorage-rs/default.nix new file mode 100644 index 000000000000..17a7bb799fdf --- /dev/null +++ b/pkgs/servers/syncstorage-rs/default.nix @@ -0,0 +1,63 @@ +{ fetchFromGitHub +, rustPlatform +, pkg-config +, python3 +, openssl +, cmake +, libmysqlclient +, makeBinaryWrapper +, lib +}: + +let + pyFxADeps = python3.withPackages (p: [ + p.setuptools # imports pkg_resources + # remainder taken from requirements.txt + p.pyfxa + p.tokenlib + p.cryptography + ]); +in + +rustPlatform.buildRustPackage rec { + pname = "syncstorage-rs"; + version = "0.12.0"; + + src = fetchFromGitHub { + owner = "mozilla-services"; + repo = pname; + rev = version; + hash = "sha256-VfIpjpBS7LXe32fxIFp7xmbm40VwxUdHIEm5PnMpd4s="; + }; + + nativeBuildInputs = [ + cmake + makeBinaryWrapper + pkg-config + python3 + ]; + + buildInputs = [ + libmysqlclient + openssl + ]; + + preFixup = '' + wrapProgram $out/bin/syncstorage \ + --prefix PATH : ${lib.makeBinPath [ pyFxADeps ]} + ''; + + cargoSha256 = "sha256-JXxArKA/2SIYJvjNA1yZHR9xDKt3N2U7HVMP/6M3BxE="; + + buildFeatures = [ "grpcio/openssl" ]; + + # almost all tests need a DB to test against + doCheck = false; + + meta = { + description = "Mozilla Sync Storage built with Rust"; + homepage = "https://github.com/mozilla-services/syncstorage-rs"; + license = lib.licenses.mpl20; + maintainers = with lib.maintainers; [ pennae ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 68cdd2a8958e..018e6dfc9084 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -22805,6 +22805,8 @@ with pkgs; sympa = callPackage ../servers/mail/sympa { }; + syncstorage-rs = callPackage ../servers/syncstorage-rs { }; + system-sendmail = lowPrio (callPackage ../servers/mail/system-sendmail { }); # PulseAudio daemons |