From 55c3f56dc53de2451718d3fdba16d2197b48b509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Wed, 31 Aug 2022 12:01:34 +0200 Subject: Implement generating thin-edge binaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flake now exports a working `lib.mkThinEdge` function with which users can generate a working binary with only the components they wish to include. It works as follows: - It pulls in all the dependencies required for the thin-edge CLI - It then pulls in and patches the sources given. Any path references to tedge_api/lib are patched to their nix-store equivalencies. - The Cargo.lock is generated by downloading the cargo index and overriding its location to be effectively offline - A temporary Rust project is created, where all the different components are then referenced and pulled in. - This Rust project is then compiled using `crane` (currently a private fork with some hacks due to a deficiency in Nix relating to importing store paths in the TOML reader) - crane automatically copies the resulting binary to the output Signed-off-by: Marcel Müller --- lib/nix-thin-edge-builder/flake.lock | 17 +-- lib/nix-thin-edge-builder/flake.nix | 162 ++++++++++++++--------- lib/nix-thin-edge-builder/writeCargoRegistry.nix | 20 +++ lib/nix-thin-edge-builder/writeCargoToml.nix | 47 ++++--- lib/nix-thin-edge-builder/writeSrcMainRs.nix | 70 +++++++++- 5 files changed, 228 insertions(+), 88 deletions(-) create mode 100644 lib/nix-thin-edge-builder/writeCargoRegistry.nix diff --git a/lib/nix-thin-edge-builder/flake.lock b/lib/nix-thin-edge-builder/flake.lock index 31cda59b..d181e45d 100644 --- a/lib/nix-thin-edge-builder/flake.lock +++ b/lib/nix-thin-edge-builder/flake.lock @@ -9,15 +9,16 @@ ] }, "locked": { - "lastModified": 1660447363, - "narHash": "sha256-Y1OtGWEb0bgFlp9YULaiYcgfEoHVIz1NuXwzlFZ/0HE=", - "owner": "ipetkov", + "lastModified": 1661841194, + "narHash": "sha256-uTKGw9NVaJ/PlEoJzrnqYLU3ADOvGiXqeXr5pTX2WPE=", + "owner": "TheNeikos", "repo": "crane", - "rev": "5548a68f5d4d60a2315ab1232e6fba5528e71dc8", + "rev": "1bb502992605a5ffe5085987aa8924ae15c57349", "type": "github" }, "original": { - "owner": "ipetkov", + "owner": "TheNeikos", + "ref": "fix/use_unsafe", "repo": "crane", "type": "github" } @@ -70,11 +71,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1661300288, - "narHash": "sha256-R3FTmbhGhJ4bZZYFn/7KZjoFemhuSYCjPdPLzYSJKpI=", + "lastModified": 1661353537, + "narHash": "sha256-1E2IGPajOsrkR49mM5h55OtYnU0dGyre6gl60NXKITE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "eb569cf5cc4ff90eb78896c04ee1fd377acc7e1b", + "rev": "0e304ff0d9db453a4b230e9386418fd974d5804a", "type": "github" }, "original": { diff --git a/lib/nix-thin-edge-builder/flake.nix b/lib/nix-thin-edge-builder/flake.nix index 8ece62d3..904d0c93 100644 --- a/lib/nix-thin-edge-builder/flake.nix +++ b/lib/nix-thin-edge-builder/flake.nix @@ -1,7 +1,7 @@ { inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - crane.url = "github:ipetkov/crane"; + crane.url = "github:TheNeikos/crane/fix/use_unsafe"; crane.inputs.nixpkgs.follows = "nixpkgs"; flake-utils.url = "github:numtide/flake-utils"; }; @@ -12,76 +12,112 @@ craneLib = crane.lib.${system}; pkgs = import nixpkgs { inherit system; }; in - { - lib = rec { - officialComponents = [ - (mkThinEdgeComponent { src = ../../plugins/plugin_avg; pluginBuilderPath = "foo"; }) - ]; + { + lib = rec { + officialComponents = [ + (mkThinEdgeComponent { src = ../../plugins/plugin_avg; pluginBuilderPath = "plugin_avg::AvgPluginBuilder"; }) + ]; - mkThinEdgeComponent = attrs@{ - name ? null, - version ? null, - src, - pluginBuilderPath, - }: - let - crateInfo = craneLib.crateNameFromCargoToml { - cargoToml = "${src}/Cargo.toml"; - }; - in { - name = if builtins.isNull name - then crateInfo.pname - else name; + mkThinEdgeComponent = + attrs@{ name ? null + , version ? null + , nativeBuildInputs ? [ ] + , buildInputs ? [ ] + , src + , pluginBuilderPath + , + }: + let + crateInfo = craneLib.crateNameFromCargoToml { + cargoToml = "${src}/Cargo.toml"; + }; + in + { + name = + if builtins.isNull name + then crateInfo.pname + else name; - version = if builtins.isNull version - then crateInfo.version - else version; + version = + if builtins.isNull version + then crateInfo.version + else version; - inherit src pluginBuilderPath; - }; + inherit src pluginBuilderPath nativeBuildInputs buildInputs; + }; + + mkThinEdgeCrateSource = + attrs@{ defaultComponents ? officialComponents + , extraComponents ? [ ] + , + }: + let + allComponents = defaultComponents ++ extraComponents; + + writeCargoToml = import ./writeCargoToml.nix; + writeSrcMainRs = import ./writeSrcMainRs.nix; + writeCargoRegistry = import ./writeCargoRegistry.nix; - mkThinEdgeCrateSource = attrs@{ - defaultComponents ? officialComponents, - extraComponents ? [], - }: - let - allComponents = defaultComponents ++ extraComponents; + cargoToml = writeCargoToml { + inherit pkgs craneLib; + components = allComponents; + }; - writeCargoToml = import ./writeCargoToml.nix; - writeSrcMainRs = import ./writeSrcMainRs.nix; + srcmainrs = writeSrcMainRs { + inherit pkgs craneLib; + components = allComponents; + }; - cargoToml = writeCargoToml { - inherit craneLib; - components = allComponents; - }; + cargoConfig = writeCargoRegistry { + inherit pkgs craneLib; + }; - srcmainrs = writeSrcMainRs { - inherit pkgs craneLib; - components = allComponents; - }; + thinEdgeLockFile = pkgs.runCommand "buildThinEdgeLockFile" { } '' + mkdir -p src + mkdir -p .cargo + mkdir -p home + HOME=$(pwd)/home + cp -v "${cargoToml}" Cargo.toml + cp -v "${cargoConfig}" .cargo/config.toml + cp -v "${srcmainrs}" src/main.rs + ${pkgs.cargo}/bin/cargo -v generate-lockfile --offline + cp Cargo.lock $out + ''; - thinEdgeCrateSrc = pkgs.runCommand "buildThinEdgeCrateSrc" {} '' - mkdir -p $out/src - cp -v "${cargoToml}" $out/Cargo.toml - cp -v "${srcmainrs}" $out/src/main.rs - ''; - in thinEdgeCrateSrc; + thinEdgeCrateSrc = pkgs.runCommand "buildThinEdgeCrateSrc" { } '' + mkdir -p $out/src + mkdir -p $out/.cargo + mkdir -p $out/home + cp -v "${cargoToml}" $out/Cargo.toml + cp -v "${thinEdgeLockFile}" $out/Cargo.lock + cp -v "${srcmainrs}" $out/src/main.rs + ''; + in + thinEdgeCrateSrc; - mkThinEdge = attrs@{ - defaultComponents ? officialComponents, - extraComponents ? [], - }: - let - thinEdgeCrateSrc = mkThinEdgeCrateSource { - inherit defaultComponents extraComponents; - }; + mkThinEdge = + attrs@{ defaultComponents ? officialComponents + , extraComponents ? [ ] + }: + let + thinEdgeCrateSrc = mkThinEdgeCrateSource { + inherit defaultComponents extraComponents; + }; + collectedInputs = { + buildInputs = pkgs.lib.lists.concatMap (comp: comp.buildInputs) extraComponents; + nativeBuildInputs = pkgs.lib.lists.concatMap (comp: comp.nativeBuildInputs) extraComponents; + }; - builtPackage = craneLib.buildPackage { - pname = "thin-edge"; - version = "0.1.0"; - src = thinEdgeCrateSrc; - }; - in builtPackage; - }; - }); + + builtPackage = craneLib.buildPackage { + pname = "thin-edge"; + version = "0.1.0"; + src = thinEdgeCrateSrc; + buildInputs = collectedInputs.buildInputs; + nativeBuildInputs = collectedInputs.nativeBuildInputs; + }; + in + builtPackage; + }; + }); } diff --git a/lib/nix-thin-edge-builder/writeCargoRegistry.nix b/lib/nix-thin-edge-builder/writeCargoRegistry.nix new file mode 100644 index 00000000..d536b2ab --- /dev/null +++ b/lib/nix-thin-edge-builder/writeCargoRegistry.nix @@ -0,0 +1,20 @@ +{ craneLib, pkgs }: + +let + cratesIoRegistry = fetchGit { + url = "https://github.com/rust-lang/crates.io-index.git"; + ref = "master"; + rev = "e9936adc4a5f03f929c3d19a8d3bcc73479ccc9a"; + }; + localCrateRegistry = pkgs.linkFarm "local-crate-registry" [ + { name = "index"; path = cratesIoRegistry; } + ]; + config = { + source = { + crates-io = { + local-registry = localCrateRegistry; + }; + }; + }; +in +craneLib.writeTOML "Cargo.toml" config diff --git a/lib/nix-thin-edge-builder/writeCargoToml.nix b/lib/nix-thin-edge-builder/writeCargoToml.nix index 34a276af..91dc2288 100644 --- a/lib/nix-thin-edge-builder/writeCargoToml.nix +++ b/lib/nix-thin-edge-builder/writeCargoToml.nix @@ -1,43 +1,60 @@ -{ craneLib, components }: +{ craneLib, components, pkgs }: let edition = "2021"; + applyOverrides = replacements: pkg: pkgs.runCommand "apply-${pkg.name}" { } '' + cp -R ${pkg.src}/* . + rm Cargo.toml + cat ${pkg.src}/Cargo.toml ${pkgs.lib.concatMapStrings (r: '' | sed 's|${r.localPath}|"${r.src}"|' '') replacements} > Cargo.toml + cp -R ./ $out + ''; + apiOnlyPatchCrates = [ + { localPath = "\".*/tedge_api\""; src = ../../crates/core/tedge_api; } + ]; + allPatchCrates = apiOnlyPatchCrates ++ [ + { localPath = "\".*/tedge_lib\""; src = localCrates.tedge_lib.path; } + { localPath = "\".*/tedge_core\""; src = localCrates.tedge_core.path; } + ]; + localCrates = { + tedge_api = { path = ../../crates/core/tedge_api; }; + tedge_core = { path = applyOverrides apiOnlyPatchCrates { name = "tedge_core"; src = ../../crates/core/tedge_core; }; }; + tedge_lib = { path = applyOverrides apiOnlyPatchCrates { name = "tedge_lib"; src = ../../crates/core/tedge_lib; }; }; + tedge-cli = { + default-features = false; + path = applyOverrides allPatchCrates { name = "tedge-cli"; src = ../../tedge; }; + }; + }; defaultDependencies = { - clap = { version = "3"; features = ["derive" "cargo" "suggestions"]; }; + clap = { version = "3"; features = [ "derive" "cargo" "suggestions" ]; }; toml = "0.5.8"; - tokio = { version = "1"; features = ["fs" "macros" "rt-multi-thread" "signal"]; }; - miette = { version = "4.7"; features = ["fancy"]; }; + tokio = { version = "1"; features = [ "fs" "macros" "rt-multi-thread" "signal" ]; }; + miette = { version = "4.7"; features = [ "fancy" ]; }; cfg-if = "1"; tracing = "0.1"; - tracing-subscriber = { version = "0.3.11"; features = ["env-filter"]; }; + tracing-subscriber = { version = "0.3.11"; features = [ "env-filter" ]; }; tracing-chrome = "0.6"; tracing-tracy = "0.9"; cfg_table = "1.0.0"; nu-ansi-term = "0.45.1"; - pretty = { version = "0.11.3"; features = ["termcolor"]; }; + pretty = { version = "0.11.3"; features = [ "termcolor" ]; }; termcolor = "1.1.3"; termimad = "0.20.1"; term_size = "0.3.2"; owo-colors = "3.4.0"; textwrap = "0.15.0"; - - tedge_api = { path = "../crates/core/tedge_api"; }; - tedge_core = { path = "../crates/core/tedge_core"; }; - tedge_lib = { path = "../crates/core/tedge_lib"; }; - - env_logger = { version = "0.9"; optional = true; }; - }; + } // localCrates; componentDependencies = let mkDep = component: { name = component.name; value = { - path = component.src; + path = applyOverrides allPatchCrates component; }; }; - in builtins.listToAttrs (builtins.map mkDep components); + in + builtins.listToAttrs (builtins.map mkDep components); in craneLib.writeTOML "Cargo.toml" { diff --git a/lib/nix-thin-edge-builder/writeSrcMainRs.nix b/lib/nix-thin-edge-builder/writeSrcMainRs.nix index b95bca48..73493b38 100644 --- a/lib/nix-thin-edge-builder/writeSrcMainRs.nix +++ b/lib/nix-thin-edge-builder/writeSrcMainRs.nix @@ -5,8 +5,74 @@ let in pkgs.writeText "main.rs" '' /// This file was generated by nix +use clap::Parser; +use tracing::debug; +use tracing::info; -fn main() { - println!("HelloWorld"); +use tedge_cli::Registry; +use tedge_lib::measurement::Measurement; +use tedge_lib::notification::Notification; + +#[tokio::main] +#[tracing::instrument] +async fn main() -> miette::Result<()> { + let args = tedge_cli::cli::Cli::parse(); + let _guard = tedge_cli::logging::setup_logging( + args.logging, + args.chrome_logging.as_ref(), + args.tracy_logging, + )?; + info!("Tedge booting..."); + debug!(?args, "Tedge CLI"); + + let registry = tedge_cli::Registry::default(); + info!("Building application"); + + ${ pkgs.lib.concatMapStrings (comp: '' + let registry = { + let kind_name: &'static str = <${comp.pluginBuilderPath} as tedge_api::PluginBuilder>::kind_name(); + info!(%kind_name, "Registering plugin builder"); + let mut registry = registry; + if !registry.plugin_kinds.insert(kind_name.to_string()) { + miette::bail!("Plugin kind '{}' was already registered, cannot register!", kind_name) + } + + let kind_name_str = kind_name.to_string(); + registry.doc_printers.insert(kind_name.to_string(), Box::new(move || { + use std::io::Write; + use miette::IntoDiagnostic; + use pretty::Arena; + + let mut stdout = std::io::stdout(); + if let Some(config_desc) = <${comp.pluginBuilderPath} as tedge_api::PluginBuilder>::kind_configuration() { + let terminal_width = term_size::dimensions().map(|(w, _)| w).unwrap_or(80); + let arena = Arena::new(); + + let rendered_doc = tedge_cli::config::as_terminal_doc(&config_desc, &arena); + + let mut output = String::new(); + rendered_doc.render_fmt(terminal_width, &mut output).into_diagnostic()?; + + writeln!(stdout, " ----- Documentation for plugin '{}'", kind_name_str) + .into_diagnostic()?; + + writeln!(stdout, "{}", output).into_diagnostic()?; + } else { + let msg = format!(" Documentation for plugin '{}' is unavailable", kind_name); + writeln!(stdout, "{}", nu_ansi_term::Color::Red.bold().paint(msg)) + .into_diagnostic()?; + } + Ok(()) + })); + + tedge_cli::Registry { + app_builder: registry.app_builder.with_plugin_builder(${comp.pluginBuilderPath}), + plugin_kinds: registry.plugin_kinds, + doc_printers: registry.doc_printers, + } + }; + '') components } + + tedge_cli::run_app(args, registry).await } '' -- cgit v1.2.3