path: root/src/repository/repository.rs

//
// Copyright (c) 2020-2021 science+computing ag and other contributors
//
// This program and the accompanying materials are made
// available under the terms of the Eclipse Public License 2.0
// which is available at https://www.eclipse.org/legal/epl-2.0/
//
// SPDX-License-Identifier: EPL-2.0
//

use std::collections::BTreeMap;
use std::path::Path;
use std::path::PathBuf;

use anyhow::anyhow;
use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
use log::trace;
use resiter::Map;

use crate::package::Package;
use crate::package::PackageName;
use crate::package::PackageVersion;
use crate::package::PackageVersionConstraint;
use crate::repository::fs::FileSystemAccessor;

/// A repository represents a collection of packages
pub struct Repository {
    inner: BTreeMap<(PackageName, PackageVersion), Package>,
}

#[cfg(test)]
impl From<BTreeMap<(PackageName, PackageVersion), Package>> for Repository {
    fn from(inner: BTreeMap<(PackageName, PackageVersion), Package>) -> Self {
        Repository { inner }
    }
}

impl Repository {
    pub async fn load(path: &Path, progress: &indicatif::ProgressBar) -> Result<Self> {
        use futures::StreamExt;
        use futures::FutureExt;

        async fn all_subdirs(p: PathBuf, accessor: FileSystemAccessor) -> Result<impl futures::stream::Stream<Item = Result<PathBuf>>> {
            let stream = accessor.read_dir(p)
                .await?
                .then(move |entry| {
                    async move {
                        let de = entry?;
                        let is_dir = de.file_type().await?.is_dir();
                        let is_hidden = de
                            .path()
                            .file_name()
                            .and_then(|s| s.to_str())
                            .map(|s| s.starts_with('.'))
                            .unwrap_or(false);

                        if is_dir && !is_hidden {
                            Ok(Some(de.path()))
                        } else {
                            Ok(None)
                        }
                    }
                })
                .filter_map(|t| async {
                    match t {
                        Err(e) => Some(Err(e)),
                        Ok(None) => None,
                        Ok(Some(o)) => Some(Ok(o)),
                    }
                });

            Ok(stream)
        }

        fn load_recursive<'a>(
            root: &'a Path,
            path: &'a Path,
            mut config: config::Config,
            progress: &'a indicatif::ProgressBar,
            accessor: FileSystemAccessor,
        ) -> futures::future::BoxFuture<'a, Result<Vec<Result<Package>>>> {
            async move {
                let pkg_file = path.join("pkg.toml");

                if accessor.is_file(&pkg_file).await? {
                    log::trace!("Reading: {}", pkg_file.display());
                    let buf = accessor.read_to_string(&pkg_file)
                        .await
                        .with_context(|| format!("Reading {}", pkg_file.display()))?;

                    // This function has an issue: It loads packages recursively, but if there are
                    // patches set for a package, these patches are set _relative_ to the current
                    // pkg.toml file.
                    //
                    // E.G.:
                    // (1) /pkg.toml
                    // (2) /a/pkg.toml
                    // (3) /a/1.0/pkg.toml
                    // (4) /a/2.0/pkg.toml
                    //
                    // If (2) defines a patches = ["./1.patch"], the patch exists at /a/1.patch.
                    // We can fix that by modifying the Config object after loading (2) and fixing the
                    // path of the patch to be relative to the repostory root.
                    //
                    // But if we continue loading the /a/ subdirectory recursively, this value gets
                    // overwritten by Config::refresh(), which is called by Config::merge, for example.
                    //
                    // The trick is, to get the list of patches _before_ the merge, and later
                    // re-setting them after the merge, if there were no new patches set (which itself
                    // is tricky to find out, because the `Config` object _looks like_ there is a new
                    // array set).
                    //
                    // If (3), for example, does set a new patches=[] array, the old array is
                    // invalidated and no longer relevant for that package!
                    // Thus, we can savely throw it away and continue with the new array, fixing the
                    // pathes to be relative to repo root again.
                    //
                    // If (4) does _not_ set any patches, we must ensure that the patches from the
                    // loading of (2) are used and not overwritten by the Config::refresh() call
                    // happening during Config::merge().
                    //

                    // first of all, we get the patches array.
                    // This is either the patches array from the last recursion or the newly set one,
                    // that doesn't matter here.
                    log::trace!("Fetching patches before merge");
                    let patches_before_merge = match config.get_array("patches") {
                        Ok(v)  => {
                            log::trace!("Array found");
                            v.into_iter()
                                .map(|p| {
                                    log::trace!("Patch path to string: {:?}", p);
                                    p.into_str()
                                        .map(PathBuf::from)
                                        .with_context(|| anyhow!("patches must be strings"))
                                        .map_err(Error::from)
                                })
                                .collect::<Result<Vec<_>>>()
                                .context("Collecting patches")?
                        },
                        Err(config::ConfigError::NotFound(_)) => vec![],
                        Err(e)                                => return Err(e).map_err(Error::from),
                    };
                    trace!("[{}] Patches before merging: {:?}", path.display(), patches_before_merge);

                    // Merge the new pkg.toml file over the already loaded configuration
                    config
                        .merge(config::File::from_str(&buf, config::FileFormat::Toml))
                        .with_context(|| format!("Loading contents of {}", pkg_file.display()))?;

                    let path_relative_to_root = path.strip_prefix(root)?;

                    // get the patches that are in the `config` object after the merge
                    log::trace!("Fetching patches after merge");
                    let patches = match config.get_array("patches") {
                        Ok(v) => {
                            trace!("[{}] Patches after merging: {:?}", path.display(), v);
                            v
                        },

                        // if there was none, we simply use an empty array
                        // This is cheap because Vec::with_capacity(0) does not allocate
                        Err(config::ConfigError::NotFound(_)) => Vec::with_capacity(0),
                        Err(e)                                => return Err(e).map_err(Error::from),
                    }
                    .into_iter()

                    // Map all `Value`s to String and then join them on the path that is relative to
                    // the root directory of the repository.
                    .map(|patch| patch.into_str().map_err(Error::from))
                    .map_ok(|patch| path_relative_to_root.join(patch))
                    .inspect(|patch| trace!("[{}] Patch relative to root: {:?}", path.display(), patch.as_ref().map(|p| p.display())));

                    // now, we need to "manually" iter over the iterator here, because we need to
                    // call FileSystemAccessor::exists for each patch, and the method is async.
                    //
                    // We collect the patches "manually" here into a preallocated vector of decent
                    // size
                    log::trace!("Allocating collection for found patches with size {}", patches.size_hint().0);
                    let mut collected_patches = Vec::with_capacity(patches.size_hint().0);
                    log::trace!("Collecting patches...");
                    for patch in patches {
                        match patch {
                            Ok(patch) => if accessor.exists(&patch).await? {
                                trace!("[{}] Path to patch exists: {}", path.display(), patch.display());

                                collected_patches.push(patch);
                            } else if patches_before_merge.iter().any(|pb| pb.file_name() == patch.file_name()) {
                                // We have a patch already in the array that is named equal to the patch
                                // we have in the current recursion.
                                // It seems like this patch was already in the list and we re-found it
                                // because we loaded a deeper pkg.toml file.
                                /* do nothing */
                            } else {
                                trace!("[{}] Path to patch does not exist: {}", path.display(), patch.display());
                                Err(anyhow!("Patch does not exist: {}", patch.display()))
                                    .context("Collecting patches after merge")?
                            },

                            Err(e) => Err(e).context("Collecting patches after merge")?,
                        }
                    }
                    log::trace!("Collected patches");
                    let patches = collected_patches; // rebind variable

                    // If we found any patches, use them. Otherwise use the array from before the merge
                    // (which already has the correct pathes from the previous recursion).
                    //
                    // We don't need to check whether the patches all exist, because we checked
                    // that above when collecting them
                    let patches = if !patches.is_empty() /* && patches.iter().all(|p| accessor.exists(p).await?) */ {
                        patches
                    } else {
                        patc