diff options
40 files changed, 154 insertions, 214 deletions
diff --git a/src/config/mod.rs b/src/config/mod.rs index fbe10ff..679745a 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,14 +1,14 @@ -use std::path::PathBuf; +use std::collections::BTreeMap; use std::fmt::Debug; use std::ops::Deref; -use std::collections::BTreeMap; +use std::path::PathBuf; -use anyhow::Result; use anyhow::Context; -use getset::Getters; +use anyhow::Result; use getset::CopyGetters; -use serde::Deserialize; +use getset::Getters; use handlebars::Handlebars; +use serde::Deserialize; use crate::phase::PhaseName; use crate::util::EnvironmentVariableName; diff --git a/src/db/connection.rs b/src/db/connection.rs index 4e338d0..20c73bc 100644 --- a/src/db/connection.rs +++ b/src/db/connection.rs @@ -1,8 +1,8 @@ -use anyhow::Result; use anyhow::Error; +use anyhow::Result; use clap_v3::ArgMatches; -use diesel::prelude::*; use diesel::pg::PgConnection; +use diesel::prelude::*; use getset::Getters; use crate::config::Configuration; diff --git a/src/db/interface.rs b/src/db/interface.rs index e855bc5..bb473b2 100644 --- a/src/db/interface.rs +++ b/src/db/interface.rs @@ -1,14 +1,15 @@ -use std::path::PathBuf; use std::fmt::Display; +use std::path::PathBuf; +use std::process::Command; use clap_v3 as clap; -use clap::ArgMatches; -use anyhow::anyhow; use anyhow::Context; use anyhow::Error; use anyhow::Result; -use itertools::Itertools; +use anyhow::anyhow; +use clap::ArgMatches; use diesel::RunQueryDsl; +use itertools::Itertools; use crate::config::Configuration; use crate::db::DbConnectionConfig; @@ -25,8 +26,6 @@ pub fn interface(db_connection_config: DbConnectionConfig, matches: &ArgMatches, } fn cli(db_connection_config: DbConnectionConfig, matches: &ArgMatches, config: &Configuration) -> Result<()> { - use std::process::Command; - trait PgCliCommand { fn run_for_uri(&self, dbcc: DbConnectionConfig) -> Result<()>; } diff --git a/src/db/models/envvar.rs b/src/db/models/envvar.rs index 24403d6..f883661 100644 --- a/src/db/models/envvar.rs +++ b/src/db/models/envvar.rs @@ -1,5 +1,3 @@ -use std::ops::Deref; - use anyhow::Error; use anyhow::Result; use diesel::PgConnection; diff --git a/src/db/models/githash.rs b/src/db/models/githash.rs index bcd14fe..df7937c 100644 --- a/src/db/models/githash.rs +++ b/src/db/models/githash.rs @@ -1,5 +1,3 @@ -use std::ops::Deref; - use anyhow::Error; use anyhow::Result; use diesel::PgConnection; diff --git a/src/db/models/image.rs b/src/db/models/image.rs index de0b2cf..225bac7 100644 --- a/src/db/models/image.rs +++ b/src/db/models/image.rs @@ -1,5 +1,3 @@ -use std::ops::Deref; - use anyhow::Error; use anyhow::Result; use diesel::PgConnection; diff --git a/src/db/models/submit.rs b/src/db/models/submit.rs index 37f5189..1f81584 100644 --- a/src/db/models/submit.rs +++ b/src/db/models/submit.rs @@ -1,20 +1,15 @@ -use std::ops::Deref; - -use anyhow::anyhow; -use anyhow::Error; use anyhow::Context; +use anyhow::Error; use anyhow::Result; +use chrono::NaiveDateTime; use diesel::PgConnection; use diesel::prelude::*; -use chrono::NaiveDateTime; -use diesel::sql_types::Uuid; -use diesel::sql_types::Jsonb; -use crate::schema::submits::*; -use crate::schema::submits; +use crate::db::models::GitHash; use crate::db::models::Image; use crate::db::models::Package; -use crate::db::models::GitHash; +use crate::schema::submits::*; +use crate::schema::submits; #[derive(Queryable)] pub struct Submit { diff --git a/src/endpoint/configuration.rs b/src/endpoint/configuration.rs index 1670c1a..753e2a3 100644 --- a/src/endpoint/configuration.rs +++ b/src/endpoint/configuration.rs @@ -1,9 +1,9 @@ +use anyhow::Result; use getset::Getters; use typed_builder::TypedBuilder; -use anyhow::Result; -use crate::util::docker::ImageName; use crate::endpoint::Endpoint; +use crate::util::docker::ImageName; #[derive(Getters, TypedBuilder)] pub struct EndpointConfiguration { diff --git a/src/endpoint/configured.rs b/src/endpoint/configured.rs index f000f1c..08fcac1 100644 --- a/src/endpoint/configured.rs +++ b/src/endpoint/configured.rs @@ -1,27 +1,25 @@ use std::fmt::{Debug, Formatter}; +use std::path::PathBuf; +use std::str::FromStr; use std::sync::Arc; use std::sync::RwLock; -use std::str::FromStr; -use std::path::PathBuf; -use anyhow::Error; use anyhow::Context; +use anyhow::Error; use anyhow::Result; use anyhow::anyhow; use getset::{Getters, CopyGetters}; -use shiplift::ContainerOptions; use shiplift::Docker; use shiplift::ExecContainerOptions; -use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::mpsc::UnboundedSender; use typed_builder::TypedBuilder; -use crate::util::docker::ImageName; use crate::endpoint::EndpointConfiguration; -use crate::job::RunnableJob; +use crate::filestore::StagingStore; use crate::job::JobResource; +use crate::job::RunnableJob; use crate::log::LogItem; -use crate::filestore::StagingStore; +use crate::util::docker::ImageName; #[derive(Getters, CopyGetters, TypedBuilder)] pub struct Endpoint { diff --git a/src/endpoint/scheduler.rs b/src/endpoint/scheduler.rs index 01033e2..82fac8c 100644 --- a/src/endpoint/scheduler.rs +++ b/src/endpoint/scheduler.rs @@ -1,20 +1,20 @@ -use std::ops::DerefMut; +use std::path::PathBuf; use std::sync::Arc; use std::sync::RwLock; -use std::path::PathBuf; -use anyhow::anyhow; use anyhow::Context; use anyhow::Result; +use anyhow::anyhow; +use futures::FutureExt; +use itertools::Itertools; use tokio::stream::StreamExt; use tokio::sync::mpsc::UnboundedSender; use crate::endpoint::Endpoint; use crate::endpoint::EndpointConfiguration; -use crate::job::JobSet; +use crate::filestore::StagingStore; use crate::job::RunnableJob; use crate::log::LogItem; -use crate::filestore::StagingStore; pub struct EndpointScheduler { endpoints: Vec<Arc<RwLock<Endpoint>>>, @@ -34,8 +34,6 @@ impl EndpointScheduler { } async fn setup_endpoints(endpoints: Vec<EndpointConfiguration>) -> Result<Vec<Arc<RwLock<Endpoint>>>> { - use futures::FutureExt; - let unordered = futures::stream::FuturesUnordered::new(); for cfg in endpoints.into_iter() { @@ -66,9 +64,6 @@ impl EndpointScheduler { } async fn select_free_endpoint(&self) -> Result<Arc<RwLock<Endpoint>>> { - use itertools::Itertools; - use futures::FutureExt; - loop { let unordered = futures::stream::FuturesUnordered::new(); for ep in self.endpoints.iter().cloned() { diff --git a/src/filestore/artifact.rs b/src/filestore/artifact.rs index 9ce752e..92efd3d 100644 --- a/src/filestore/artifact.rs +++ b/src/filestore/artifact.rs @@ -1,16 +1,15 @@ -use std::path::Path; -use std::path::PathBuf; -use std::cmp::PartialOrd; use std::cmp::Ord; use std::cmp::Ordering; +use std::cmp::PartialOrd; +use std::path::Path; +use std::path::PathBuf; use anyhow::Context; use anyhow::Error; -use anyhow::anyhow; use anyhow::Result; -use pom::*; -use pom::parser::Parser as PomParser; +use anyhow::anyhow; use getset::Getters; +use pom::parser::Parser as PomParser; use crate::package::PackageName; use crate::package::PackageVersion; diff --git a/src/filestore/merged.rs b/src/filestore/merged.rs index 5f3f755..7cdeb78 100644 --- a/src/filestore/merged.rs +++ b/src/filestore/merged.rs @@ -1,14 +1,13 @@ use std::sync::Arc; use std::sync::RwLock; -use anyhow::anyhow; use anyhow::Result; +use anyhow::anyhow; use crate::filestore::Artifact; use crate::filestore::ReleaseStore; use crate::filestore::StagingStore; use crate::package::PackageName; -use crate::package::PackageVersion; use crate::package::PackageVersionConstraint; /// A type that merges the release store and the staging store diff --git a/src/filestore/release.rs b/src/filestore/release.rs index a72aa51..fb0bac5 100644 --- a/src/filestore/release.rs +++ b/src/filestore/release.rs @@ -1,13 +1,10 @@ -use std::path::Path; -use std::path::PathBuf; use std::fmt::Debug; +use std::path::Path; use anyhow::Result; use indicatif::ProgressBar; use crate::filestore::util::FileStoreImpl; -use crate::filestore::MergedStores; -use crate::filestore::StagingStore; // The implementation of this type must be available in the merged filestore. pub struct ReleaseStore(pub (in crate::filestore) FileStoreImpl); diff --git a/src/filestore/staging.rs b/src/filestore/staging.rs index 361da53..d471d44 100644 --- a/src/filestore/staging.rs +++ b/src/filestore/staging.rs @@ -1,19 +1,19 @@ +use std::fmt::Debug; use std::path::Path; use std::path::PathBuf; -use std::fmt::Debug; -use anyhow::anyhow; -use anyhow::Result; -use anyhow::Error; use anyhow::Context; -use indicatif::ProgressBar; +use anyhow::Error; +use anyhow::Result; +use anyhow::anyhow; use futures::stream::Stream; -use resiter::Map; +use indicatif::ProgressBar; use resiter::Filter; +use resiter::Map; use tar; -use crate::filestore::util::FileStoreImpl; use crate::filestore::Artifact; +use crate::filestore::util::FileStoreImpl; // The implementation of this type must be available in the merged filestore. pub struct StagingStore(pub (in crate::filestore) FileStoreImpl); @@ -38,7 +38,6 @@ impl StagingStore { where S: Stream<Item = Result<Vec<u8>>> { use futures::stream::TryStreamExt; - use std::io::Read; let dest = &self.0.root; stream.try_concat() diff --git a/src/filestore/util.rs b/src/filestore/util.rs index 854374d..c898420 100644 --- a/src/filestore/util.rs +++ b/src/filestore/util.rs @@ -5,17 +5,16 @@ use std::collections::BTreeMap; use std::path::Path; use std::path::PathBuf; -use anyhow::anyhow; -use anyhow::Result; use anyhow::Error; -use walkdir::WalkDir; -use resiter::Map; -use resiter::AndThen; +use anyhow::Result; +use anyhow::anyhow; use indicatif::ProgressBar; +use resiter::AndThen; +use resiter::Map; +use walkdir::WalkDir; use crate::filestore::Artifact; use crate::package::PackageName; -use crate::package::PackageVersion; use crate::package::PackageVersionConstraint; /// The actual filestore implementation diff --git a/src/job/job.rs b/src/job/job.rs index 9a97c30..1046b5d 100644 --- a/src/job/job.rs +++ b/src/job/job.rs @@ -1,3 +1,4 @@ +use getset::Getters; use uuid::Uuid; use crate::job::JobResource; @@ -5,8 +6,6 @@ use crate::package::Package; use crate::phase::PhaseName; use crate::util::docker::ImageName; -use getset::Getters; - /// A prepared, but not necessarily runnable, job configuration #[derive(Debug, Getters)] pub struct Job { diff --git a/src/job/runnable.rs b/src/job/runnable.rs index 1654937..6b26d43 100644 --- a/src/job/runnable.rs +++ b/src/job/runnable.rs @@ -1,20 +1,18 @@ -use getset::Getters; -use uuid::Uuid; -use anyhow::anyhow; use anyhow::Context; use anyhow::Error; use anyhow::Result; -use resiter::Map; +use anyhow::anyhow; +use getset::Getters; +use uuid::Uuid; +use crate::filestore::MergedStores; use crate::job::Job; use crate::job::JobResource; use crate::package::Package; +use crate::package::ParseDependency; use crate::package::Script; use crate::package::ScriptBuilder; -use crate::phase::PhaseName; use crate::util::docker::ImageName; -use crate::filestore::MergedStores; -use crate::package::ParseDependency; /// A job configuration that can be run. All inputs are clear here. #[derive(Debug, Getters)] diff --git a/src/job/set.rs b/src/job/set.rs index e4ea7eb..375db08 100644 --- a/src/job/set.rs +++ b/src/job/set.rs @@ -1,12 +1,11 @@ use anyhow::Result; +use crate::filestore::MergedStores; use crate::job::Job; -use crate::package::Package; +use crate::job::RunnableJob; use crate::package::Tree; use crate::phase::PhaseName; use crate::util::docker::ImageName; -use crate::job::RunnableJob; -use crate::filestore::MergedStores; /// A set of jobs that could theoretically be run in parallel #[derive(Debug)] @@ -87,7 +86,6 @@ mod tests { use std::collections::BTreeMap; - use url::Url; use crate::package::tests::pname; use crate::package::tests::pversion; use crate::package::tests::package; diff --git a/src/log/filesink.rs b/src/log/filesink.rs index 0784ad1..39de00b 100644 --- a/src/log/filesink.rs +++ b/src/log/filesink.rs @@ -1,17 +1,15 @@ -use std::path::Path; -use std::path::PathBuf; use std::fs::File; use std::fs::OpenOptions; -use std::convert::TryInto; use std::io::Write; +use std::path::Path; +use std::path::PathBuf; -use anyhow::Result; use anyhow::Error; -use anyhow::anyhow; +use anyhow::Result; -use crate::log::LogSink; -use crate::log::LogItem; use crate::job::Job; +use crate::log::LogItem; +use crate::log::LogSink; pub struct FileSink { file: File, diff --git a/src/log/item.rs b/src/log/item.rs index 9130045..58fb466 100644 --- a/src/log/item.rs +++ b/src/log/item.rs @@ -1,7 +1,4 @@ -use std::convert::TryInto; - use anyhow::Result; -use anyhow::Error; #[derive(Debug, PartialEq, Eq, Hash)] pub enum LogItem { diff --git a/src/log/parser.rs b/src/log/parser.rs index 9a604be..ab8f116 100644 --- a/src/log/parser.rs +++ b/src/log/parser.rs @@ -1,20 +1,15 @@ use std::result::Result as RResult; use std::str::FromStr; -use std::char::{decode_utf16, REPLACEMENT_CHARACTER}; -use anyhow::Error; -use anyhow::Result; -use anyhow::anyhow; use futures::AsyncBufReadExt; use futures::Stream; use futures::StreamExt; use futures::TryStreamExt; -use pom::*; use pom::parser::Parser as PomParser; use shiplift::tty::TtyChunk; -use crate::log::util::*; use crate::log::LogItem; +use crate::log::util::*; type IoResult<T> = RResult<T, futures::io::Error>; @@ -29,7 +24,6 @@ pub fn buffer_stream_to_line_stream<S>(stream: S) -> impl Stream<Item = IoResult pub fn parser<'a>() -> PomParser<'a, u8, LogItem> { use pom::parser::*; - use pom::char_class::hex_digit; let number = one_of(b"0123456789") .repeat(1..) @@ -73,6 +67,8 @@ pub fn parser<'a>() -> PomParser<'a, u8, LogItem> { #[cfg(test)] mod tests { use super::*; + use anyhow::Error; + use anyhow::Result; // Helper function for showing log item in error message in pretty fn prettify_item(e: &LogItem) -> String { diff --git a/src/main.rs b/src/main.rs index 0775eff..445246f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,49 +1,52 @@ #[macro_use] extern crate log as logcrate; #[macro_use] extern crate diesel; -use logcrate::debug; -use std::ops::Deref; +use std::collections::BTreeMap; use std::path::Path; use std::path::PathBuf; -use std::collections::BTreeMap; +use std::sync::Arc; +use std::sync::RwLock; -use anyhow::anyhow; use anyhow::Context; -use anyhow::Result; use anyhow::Error; -use walkdir::WalkDir; -use indicatif::*; -use tokio::stream::StreamExt; +use anyhow::Result; +use anyhow::anyhow; use clap_v3::ArgMatches; use diesel::PgConnection; use diesel::prelude::*; +use indicatif::*; +use logcrate::debug; use resiter::Filter; use resiter::Map; +use tokio::stream::StreamExt; +use walkdir::WalkDir; mod cli; -mod job; +mod config; +mod db; mod endpoint; -mod util; +mod filestore; +mod job; mod log; +mod orchestrator; mod package; mod phase; -mod config; mod repository; -mod filestore; -mod ui; -mod orchestrator; mod schema; -mod db; +mod ui; +mod util; + use crate::config::*; -use crate::repository::Repository; +use crate::filestore::ReleaseStore; +use crate::filestore::StagingStore; +use crate::job::JobSet; +use crate::orchestrator::OrchestratorSetup; use crate::package::PackageName; use crate::package::PackageVersion; use crate::package::Tree; -use crate::filestore::ReleaseStore; -use crate::filestore::StagingStore; +use crate::repository::Repository; +use crate::util::docker::ImageName; use crate::util::progress::ProgressBars; -use crate::orchestrator::Orchestrator; -use crate::orchestrator::OrchestratorSetup; #[tokio::main] async fn main() -> Result<()> { @@ -125,13 +128,6 @@ async fn build<'a>(matches: &ArgMatches, Image, Submit, }; - use schema::packages; - use schema::githashes; - use schema::images; - use crate::job::JobSet; - use std::sync::Arc; - use std::sync::RwLock; - use crate::util::docker::ImageName; let now = chrono::offset::Local::now().naive_local(); let submit_id = uuid::Uuid::new_v4(); diff --git a/src/orchestrator/orchestrator.rs b/src/orchestrator/orchestrator.rs index 0132e08..e511669 100644 --- a/src/orchestrator/orchestrator.rs +++ b/src/orchestrator/orchestrator.rs @@ -1,31 +1,23 @@ use std::path::PathBuf; -use std::sync::RwLock; use std::sync::Arc; -use std::collections::BTreeMap; +use std::sync::RwLock; -use anyhow::anyhow; -use anyhow::Error; use anyhow::Context; +use anyhow::Error; use anyhow::Result; -use typed_builder::TypedBuilder; +use anyhow::anyhow; use diesel::PgConnection; -use tokio::sync::mpsc::UnboundedReceiver; -use tokio::sync::mpsc::UnboundedSender; -use resiter::AndThen; +use typed_builder::TypedBuilder; +use crate::db::models::Submit; use crate::endpoint::EndpointConfiguration; use crate::endpoint::EndpointScheduler; -use crate::job::JobSet; -use crate::job::RunnableJob; -use crate::log::LogItem; -use crate::filestore::StagingStore; +use crate::filestore::MergedStores; use crate::filestore::ReleaseStore; +use crate::filestore::StagingStore; +use crate::job::JobSet; use crate::log::FileLogSinkFactory; -use crate::log::LogSink; -use crate::db::models::Submit; -use crate::db::models::EnvVar; -use crate::job::JobResource; -use crate::filestore::MergedStores; +use crate::log::LogItem; use crate::util::progress::ProgressBars; pub struct Orchestrator { diff --git a/src/package/dependency/build.rs b/src/package/dependency/build.rs index 96e885e..c354a90 100644 --- a/src/package/dependency/build.rs +++ b/src/package/dependency/build.rs @@ -1,11 +1,11 @@ -use serde::Serialize; -use serde::Deserialize; use anyhow::Result; +use serde::Deserialize; +use serde::Serialize; -use crate::package::dependency::StringEqual; -use crate::package::dependency::ParseDependency; use crate::package::PackageName; use crate::package::PackageVersionConstraint; +use crate::package::dependency::ParseDependency; +use crate::package::dependency::StringEqual; /// A dependency that is packaged and is only required during build time #[derive(Serialize, Deserialize, Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] diff --git a/src/package/dependency/mod.rs b/src/package/dependency/mod.rs index 9bdb2fb..6d4cb81 100644 --- a/src/package/dependency/mod.rs +++ b/src/package/dependency/mod.rs @@ -1,23 +1,23 @@ -use lazy_static::lazy_static; -use anyhow::anyhow; use anyhow::Result; +use anyhow::anyhow; +use lazy_static::lazy_static; use regex::Regex; use crate::package::PackageName; use crate::package::PackageVersionConstraint; -mod system; -pub use system::*; - -mod system_runtime; -pub use system_runtime::*; - mod build; pub use build::*; mod runtime; pub use runtime::*; +mod system; +pub use system::*; + +mod system_runtime; +pub use system_runtime::*; + pub trait StringEqual { fn str_equal(&self, s: &str) -> bool; } @@ -56,9 +56,6 @@ pub(in crate::package::dependency) fn parse_package_dependency_string_into_name_ mod tests { use super::*; - use std::convert::TryInto; - - use crate::package::Package; use crate::package::PackageName; use crate::package::PackageVersion; diff --git a/src/package/dependency/runtime.rs b/src/package/dependency/runtime.rs index daf157b..d4fa920 100644 --- a/src/package/dependency/runtime.rs +++ b/src/package/dependency/runtime.rs @@ -1,12 +1,11 @@ -use serde::Serialize; -use serde::Deserialize; use anyhow::Result; +use serde::Deserialize; +use serde::Serialize; use crate::package::PackageName; use crate::package::PackageVersionConstraint; -use crate::package::dependency::StringEqual; use crate::package::dependency::ParseDependency; - +use crate::package::dependency::StringEqual; /// A dependency that is packaged and is required during runtime #[derive(Serialize, Deserialize, Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] diff --git a/src/package/dependency/system.rs b/src/package/dependency/system.rs index 42471e8..689f0d8 100644 --- a/src/package/dependency/system.rs +++ b/src/package/dependency/system.rs @@ -1,11 +1,11 @@ -use serde::Serialize; -use serde::Deserialize; use anyhow::Result; +use serde::Deserialize; +use serde::Serialize; |