~matthiasbeyer/imag

f21b6e53a20c6e748292dd1804e46c0f4ea87980 — Matthias Beyer 2 years ago a2433f3 + 2268fd8
Merge branch 'libimagentrylink-split' into master
M Cargo.toml => Cargo.toml +1 -0
@@ 48,6 48,7 @@ members = [
    "lib/entry/libimagentryfilter",
    "lib/entry/libimagentrygps",
    "lib/entry/libimagentrylink",
    "lib/entry/libimagentryurl",
    "lib/entry/libimagentrymarkdown",
    "lib/entry/libimagentryref",
    "lib/entry/libimagentrytag",

M bin/core/imag-annotate/src/main.rs => bin/core/imag-annotate/src/main.rs +1 -1
@@ 66,7 66,7 @@ use libimagrt::runtime::Runtime;
use libimagrt::setup::generate_runtime_setup;
use libimagstore::store::FileLockEntry;
use libimagstore::iter::get::StoreIdGetIteratorExtension;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::linker::InternalLinker;

mod ui;


M bin/core/imag-diagnostics/src/main.rs => bin/core/imag-diagnostics/src/main.rs +1 -1
@@ 55,7 55,7 @@ use libimagerror::io::ToExitCode;
use libimagerror::exit::ExitUnwrap;
use libimagstore::store::FileLockEntry;
use libimagstore::storeid::StoreId;
use libimagentrylink::internal::*;
use libimagentrylink::linker::InternalLinker;

use toml::Value;
use toml_query::read::TomlValueReadExt;

M bin/core/imag-link/Cargo.toml => bin/core/imag-link/Cargo.toml +1 -0
@@ 31,6 31,7 @@ libimagstore     = { version = "0.10.0", path = "../../../lib/core/libimagstore"
libimagrt        = { version = "0.10.0", path = "../../../lib/core/libimagrt" }
libimagerror     = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink" }
libimagentryurl = { version = "0.10.0", path = "../../../lib/entry/libimagentryurl" }
libimagutil      = { version = "0.10.0", path = "../../../lib/etc/libimagutil" }

[dependencies.clap]

M bin/core/imag-link/src/main.rs => bin/core/imag-link/src/main.rs +7 -6
@@ 44,6 44,7 @@ extern crate failure;
#[cfg(test)] extern crate env_logger;

extern crate libimagentrylink;
extern crate libimagentryurl;
#[macro_use] extern crate libimagrt;
extern crate libimagstore;
extern crate libimagerror;


@@ 61,9 62,9 @@ use std::path::PathBuf;
use failure::Error;
use failure::err_msg;

use libimagentrylink::external::ExternalLinker;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::internal::store_check::StoreLinkConsistentExt;
use libimagentryurl::linker::UrlLinker;
use libimagentrylink::linker::InternalLinker;
use libimagentrylink::storecheck::StoreLinkConsistentExt;
use libimagerror::trace::{MapErrTrace, trace_error};
use libimagerror::exit::ExitUnwrap;
use libimagerror::io::ToExitCode;


@@ 157,7 158,7 @@ fn link_from_to<'a, I>(rt: &'a Runtime, from: &'a str, to: I)
            });

            let iter = from_entry
                .add_external_link(rt.store(), url)
                .add_url(rt.store(), url)
                .map_err_trace_exit_unwrap()
                .into_iter();



@@ 238,7 239,7 @@ fn remove_linking(rt: &Runtime) {
                        error!("Error parsing URL: {:?}", e);
                        ::std::process::exit(1);
                    });
                    from.remove_external_link(rt.store(), url).map_err_trace_exit_unwrap();
                    from.remove_url(rt.store(), url).map_err_trace_exit_unwrap();
                    info!("Ok: {}", id);
                } else {
                    warn!("Entry not found: {:?}", id);


@@ 313,7 314,7 @@ fn list_linkings(rt: &Runtime) {
                    }

                    if list_externals {
                        entry.get_external_links(rt.store())
                        entry.get_urls(rt.store())
                            .map_err_trace_exit_unwrap()
                            .enumerate()
                            .for_each(|(i, link)| {

M bin/core/imag-mv/src/main.rs => bin/core/imag-mv/src/main.rs +1 -1
@@ 56,7 56,7 @@ use libimagerror::exit::ExitUnwrap;
use libimagstore::storeid::StoreId;
use libimagstore::store::Store;
use libimagstore::store::FileLockEntry;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::linker::InternalLinker;
use libimagstore::iter::get::StoreIdGetIteratorExtension;

fn main() {

M bin/domain/imag-bookmark/src/main.rs => bin/domain/imag-bookmark/src/main.rs +1 -1
@@ 61,7 61,7 @@ use libimagerror::trace::{MapErrTrace, trace_error};
use libimagerror::io::ToExitCode;
use libimagerror::exit::ExitUnwrap;
use libimagutil::debug_result::DebugResult;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::linker::InternalLinker;


mod ui;

M bin/domain/imag-wiki/src/main.rs => bin/domain/imag-wiki/src/main.rs +1 -1
@@ 252,7 252,7 @@ fn show(rt: &Runtime, wiki_name: &str) {
}

fn delete(rt: &Runtime, wiki_name: &str) {
    use libimagentrylink::internal::InternalLinker;
    use libimagentrylink::linker::InternalLinker;

    let scmd   = rt.cli().subcommand_matches("delete").unwrap(); // safed by clap
    let name   = String::from(scmd.value_of("delete-name").unwrap()); // safe by clap

M lib/core/libimagrt/Cargo.toml => lib/core/libimagrt/Cargo.toml +5 -1
@@ 25,7 25,6 @@ toml = "0.5"
xdg-basedir = "1.0"
itertools = "0.7"
ansi_term = "0.11"
toml-query = "0.9"
atty = "0.2"
failure        = "0.1"
failure_derive = "0.1"


@@ 52,6 51,11 @@ version          = "^1.0.5"
default-features = false
features         = ["no_logging"]

[dependencies.toml-query]
version = "0.9"
default-features = false
features = [ "typed" ]

[features]
default = []


M lib/domain/libimagbookmark/Cargo.toml => lib/domain/libimagbookmark/Cargo.toml +1 -0
@@ 27,4 27,5 @@ failure = "0.1"
libimagstore     = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
libimagerror     = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink" }
libimagentryurl  = { version = "0.10.0", path = "../../../lib/entry/libimagentryurl" }


M lib/domain/libimagbookmark/src/collection.rs => lib/domain/libimagbookmark/src/collection.rs +14 -14
@@ 20,7 20,7 @@
//! BookmarkCollection module
//!
//! A BookmarkCollection is nothing more than a simple store entry. One can simply call functions
//! from the libimagentrylink::external::ExternalLinker trait on this to generate external links.
//! from the libimagentryurl::linker::UrlLinker trait on this to generate external links.
//!
//! The BookmarkCollection type offers helper functions to get all links or such things.



@@ 34,10 34,10 @@ use libimagstore::store::Store;
use libimagstore::store::Entry;
use libimagstore::store::FileLockEntry;
use libimagstore::storeid::StoreId;
use libimagentrylink::external::ExternalLinker;
use libimagentrylink::external::iter::UrlIter;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::internal::Link as StoreLink;
use libimagentryurl::linker::UrlLinker;
use libimagentryurl::iter::UrlIter;
use libimagentrylink::linker::InternalLinker;
use libimagentrylink::link::Link as StoreLink;

use crate::link::Link;



@@ 80,7 80,7 @@ impl<'a> BookmarkCollectionStore<'a> for Store {

}

pub trait BookmarkCollection : Sized + InternalLinker + ExternalLinker {
pub trait BookmarkCollection : Sized + InternalLinker + UrlLinker {
    fn links<'a>(&self, store: &'a Store)                        -> Result<UrlIter<'a>>;
    fn link_entries(&self)                                       -> Result<Vec<StoreLink>>;
    fn add_link(&mut self, store: &Store, l: Link)               -> Result<Vec<StoreId>>;


@@ 91,27 91,27 @@ pub trait BookmarkCollection : Sized + InternalLinker + ExternalLinker {
impl BookmarkCollection for Entry {

    fn links<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>> {
        self.get_external_links(store)
        self.get_urls(store)
    }

    fn link_entries(&self) -> Result<Vec<StoreLink>> {
        use libimagentrylink::external::is_external_link_storeid;
        use libimagentryurl::util::is_external_link_storeid;
        self.get_internal_links().map(|v| v.filter(|id| is_external_link_storeid(id)).collect())
    }

    fn add_link(&mut self, store: &Store, l: Link) -> Result<Vec<StoreId>> {
        use crate::link::IntoUrl;
        l.into_url().and_then(|url| self.add_external_link(store, url))
        l.into_url().and_then(|url| self.add_url(store, url))
    }

    fn get_links_matching<'a>(&self, store: &'a Store, r: Regex) -> Result<LinksMatchingRegexIter<'a>> {
        use self::iter::IntoLinksMatchingRegexIter;
        self.get_external_links(store).map(|iter| iter.matching_regex(r))
        self.get_urls(store).map(|iter| iter.matching_regex(r))
    }

    fn remove_link(&mut self, store: &Store, l: Link) -> Result<Vec<StoreId>> {
        use crate::link::IntoUrl;
        l.into_url().and_then(|url| self.remove_external_link(store, url))
        l.into_url().and_then(|url| self.remove_url(store, url))
    }

}


@@ 120,6 120,9 @@ pub mod iter {
    use crate::link::Link;
    use failure::Fallible as Result;
    use failure::Error;
    use regex::Regex;

    use libimagentryurl::iter::UrlIter;

    pub struct LinkIter<I>(I)
        where I: Iterator<Item = Link>;


@@ 144,9 147,6 @@ pub mod iter {
        }
    }

    use libimagentrylink::external::iter::UrlIter;
    use regex::Regex;

    pub struct LinksMatchingRegexIter<'a>(UrlIter<'a>, Regex);

    impl<'a> LinksMatchingRegexIter<'a> {

M lib/domain/libimagbookmark/src/lib.rs => lib/domain/libimagbookmark/src/lib.rs +1 -0
@@ 44,6 44,7 @@ extern crate regex;
#[macro_use] extern crate libimagstore;
extern crate libimagerror;
extern crate libimagentrylink;
extern crate libimagentryurl;

module_entry_path_mod!("bookmark");


M lib/domain/libimaghabit/src/habit.rs => lib/domain/libimaghabit/src/habit.rs +1 -1
@@ 33,7 33,7 @@ use crate::util::IsHabitCheck;
use crate::util::get_string_header_from_entry;
use crate::instance::IsHabitInstance;

use libimagentrylink::internal::InternalLinker;
use libimagentrylink::linker::InternalLinker;
use libimagstore::store::Store;
use libimagstore::store::FileLockEntry;
use libimagstore::store::Entry;

M lib/domain/libimagwiki/src/entry.rs => lib/domain/libimagwiki/src/entry.rs +1 -1
@@ 57,7 57,7 @@ impl WikiEntry for Entry {
        let processor = LinkProcessor::default()
            .process_internal_links(true)
            .create_internal_targets(true)
            .process_external_links(true)
            .process_urls(true)
            .process_refs(true);

        self.autolink_with_processor(store, processor)

M lib/domain/libimagwiki/src/wiki.rs => lib/domain/libimagwiki/src/wiki.rs +1 -1
@@ 22,7 22,7 @@ use std::path::PathBuf;
use libimagstore::store::Store;
use libimagstore::store::FileLockEntry;
use libimagstore::iter::Entries;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::linker::InternalLinker;

use failure::Fallible as Result;
use failure::Error;

M lib/entry/libimagentryannotation/src/annotateable.rs => lib/entry/libimagentryannotation/src/annotateable.rs +1 -1
@@ 24,7 24,7 @@ use libimagstore::store::Entry;
use libimagstore::store::FileLockEntry;
use libimagstore::store::Store;
use libimagstore::storeid::StoreIdIterator;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::linker::InternalLinker;
use libimagentryutil::isa::Is;
use libimagentryutil::isa::IsKindHeaderPathProvider;


M lib/entry/libimagentrycategory/src/category.rs => lib/entry/libimagentrycategory/src/category.rs +1 -1
@@ 22,7 22,7 @@ use libimagentryutil::isa::IsKindHeaderPathProvider;
use libimagstore::store::Entry;
use libimagstore::store::Store;
use libimagstore::storeid::StoreIdIterator;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::linker::InternalLinker;

use toml_query::read::TomlValueReadTypeExt;


M lib/entry/libimagentrycategory/src/entry.rs => lib/entry/libimagentrycategory/src/entry.rs +1 -1
@@ 23,7 23,7 @@ use toml_query::read::TomlValueReadTypeExt;
use toml::Value;

use libimagstore::store::Entry;
use libimagentrylink::internal::InternalLinker;
use libimagentrylink::linker::InternalLinker;
use libimagerror::errors::ErrorMsg as EM;

use failure::Fallible as Result;

M lib/entry/libimagentrycategory/src/store.rs => lib/entry/libimagentrycategory/src/store.rs +1 -1
@@ 84,7 84,7 @@ impl CategoryStore for Store {
    ///
    /// Automatically removes all category settings from entries which are linked to this category.
    fn delete_category(&self, name: &str) -> Result<()> {
        use libimagentrylink::internal::InternalLinker;
        use libimagentrylink::linker::InternalLinker;
        use crate::category::Category;

        trace!("Deleting category: '{}'", name);

D lib/entry/libimagentrylink/src/external.rs => lib/entry/libimagentrylink/src/external.rs +0 -476
@@ 1,476 0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

/// External linking is a complex implementation to be able to serve a clean and easy-to-use
/// interface.
///
/// Internally, there are no such things as "external links" (plural). Each Entry in the store can
/// only have _one_ external link.
///
/// This library does the following therefor: It allows you to have several external links with one
/// entry, which are internally one file in the store for each link, linked with "internal
/// linking".
///
/// This helps us greatly with deduplication of URLs.
///

use std::ops::DerefMut;
use std::fmt::Debug;

use libimagstore::store::Entry;
use libimagstore::store::Store;
use libimagstore::storeid::StoreId;
use libimagutil::debug_result::*;
use libimagerror::errors::ErrorMsg as EM;

use toml_query::read::TomlValueReadExt;
use toml_query::read::TomlValueReadTypeExt;
use toml_query::insert::TomlValueInsertExt;
use toml::map::Map;
use failure::Error;
use failure::Fallible as Result;
use failure::ResultExt;
use failure::err_msg;

use crate::internal::InternalLinker;

use self::iter::*;

use toml::Value;
use url::Url;
use sha1::{Sha1, Digest};
use hex;

pub trait Link {

    fn get_link_uri_from_filelockentry(&self) -> Result<Option<Url>>;

    fn get_url(&self) -> Result<Option<Url>>;

}

impl Link for Entry {

    fn get_link_uri_from_filelockentry(&self) -> Result<Option<Url>> {
        self.get_header()
            .read_string("links.external.content.url")
            .context(format_err!("Error reading header 'links.external.content.url' from '{}'", self.get_location()))
            .context(EM::EntryHeaderReadError)
            .map_err(Error::from)
            .and_then(|opt| match opt {
                None        => Ok(None),
                Some(ref s) => {
                    debug!("Found url, parsing: {:?}", s);
                    Url::parse(&s[..])
                        .map_err(Error::from)
                        .context(format_err!("Failed to parse URL: '{}'", s))
                        .context(err_msg("Invalid URI"))
                        .map_err(Error::from)
                        .map(Some)
                },
            })
            .context("Failed to get link URI from entry")
            .map_err(Error::from)
    }

    fn get_url(&self) -> Result<Option<Url>> {
        match self.get_header().read_string("links.external.url")? {
            None        => Ok(None),
            Some(ref s) => Url::parse(&s[..])
                .context(format_err!("Failed to parse URL: '{}'", s))
                .map(Some)
                .map_err(Error::from)
                .context(EM::EntryHeaderReadError)
                .map_err(Error::from),
        }
    }

}

pub trait ExternalLinker : InternalLinker {

    /// Get the external links from the implementor object
    fn get_external_links<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>>;

    /// Set the external links for the implementor object
    fn set_external_links(&mut self, store: &Store, links: Vec<Url>) -> Result<Vec<StoreId>>;

    /// Add an external link to the implementor object
    fn add_external_link(&mut self, store: &Store, link: Url) -> Result<Vec<StoreId>>;

    /// Remove an external link from the implementor object
    fn remove_external_link(&mut self, store: &Store, link: Url) -> Result<Vec<StoreId>>;

}

pub mod iter {
    //! Iterator helpers for external linking stuff
    //!
    //! Contains also helpers to filter iterators for external/internal links
    //!
    //!
    //! # Warning
    //!
    //! This module uses `internal::Link` as link type, so we operate on _store ids_ here.
    //!
    //! Not to confuse with `external::Link` which is a real `FileLockEntry` under the hood.
    //!

    use libimagutil::debug_result::*;
    use libimagstore::store::Store;

    use crate::internal::Link;
    use crate::internal::iter::LinkIter;
    use failure::Fallible as Result;

    use url::Url;

    /// Helper for building `OnlyExternalIter` and `NoExternalIter`
    ///
    /// The boolean value defines, how to interpret the `is_external_link_storeid()` return value
    /// (here as "pred"):
    ///
    /// ```ignore
    ///     pred | bool | xor | take?
    ///     ---- | ---- | --- | ----
    ///        0 |    0 |   0 |   1
    ///        0 |    1 |   1 |   0
    ///        1 |    0 |   1 |   0
    ///        1 |    1 |   0 |   1
    /// ```
    ///
    /// If `bool` says "take if return value is false", we take the element if the `pred` returns
    /// false... and so on.
    ///
    /// As we can see, the operator between these two operants is `!(a ^ b)`.
    pub struct ExternalFilterIter(LinkIter, bool);

    impl Iterator for ExternalFilterIter {
        type Item = Link;

        fn next(&mut self) -> Option<Self::Item> {
            use super::is_external_link_storeid;

            while let Some(elem) = self.0.next() {
                trace!("Check whether is external: {:?}", elem);
                if !(self.1 ^ is_external_link_storeid(&elem)) {
                    trace!("Is external id: {:?}", elem);
                    return Some(elem);
                }
            }
            None
        }
    }

    /// Helper trait to be implemented on `LinkIter` to select or deselect all external links
    ///
    /// # See also
    ///
    /// Also see `OnlyExternalIter` and `NoExternalIter` and the helper traits/functions
    /// `OnlyInteralLinks`/`only_internal_links()` and `OnlyExternalLinks`/`only_external_links()`.
    pub trait SelectExternal {
        fn select_external_links(self, b: bool) -> ExternalFilterIter;
    }

    impl SelectExternal for LinkIter {
        fn select_external_links(self, b: bool) -> ExternalFilterIter {
            ExternalFilterIter(self, b)
        }
    }


    pub struct OnlyExternalIter(ExternalFilterIter);

    impl OnlyExternalIter {
        pub fn new(li: LinkIter) -> OnlyExternalIter {
            OnlyExternalIter(ExternalFilterIter(li, true))
        }

        pub fn urls<'a>(self, store: &'a Store) -> UrlIter<'a> {
            UrlIter(self, store)
        }
    }

    impl Iterator for OnlyExternalIter {
        type Item = Link;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next()
        }
    }

    pub struct NoExternalIter(ExternalFilterIter);

    impl NoExternalIter {
        pub fn new(li: LinkIter) -> NoExternalIter {
            NoExternalIter(ExternalFilterIter(li, false))
        }
    }

    impl Iterator for NoExternalIter {
        type Item = Link;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next()
        }
    }

    pub trait OnlyExternalLinks : Sized {
        fn only_external_links(self) -> OnlyExternalIter ;

        fn no_internal_links(self) -> OnlyExternalIter {
            self.only_external_links()
        }
    }

    impl OnlyExternalLinks for LinkIter {
        fn only_external_links(self) -> OnlyExternalIter {
            OnlyExternalIter::new(self)
        }
    }

    pub trait OnlyInternalLinks : Sized {
        fn only_internal_links(self) -> NoExternalIter;

        fn no_external_links(self) -> NoExternalIter {
            self.only_internal_links()
        }
    }

    impl OnlyInternalLinks for LinkIter {
        fn only_internal_links(self) -> NoExternalIter {
            NoExternalIter::new(self)
        }
    }

    pub struct UrlIter<'a>(OnlyExternalIter, &'a Store);

    impl<'a> Iterator for UrlIter<'a> {
        type Item = Result<Url>;

        fn next(&mut self) -> Option<Self::Item> {
            use crate::external::Link;

            loop {
                let next = self.0
                    .next()
                    .map(|id| {
                        debug!("Retrieving entry for id: '{:?}'", id);
                        self.1
                            .retrieve(id.clone())
                            .map_dbg_err(|_| format!("Retrieving entry for id: '{:?}' failed", id))
                            .map_err(From::from)
                            .and_then(|f| {
                                debug!("Store::retrieve({:?}) succeeded", id);
                                debug!("getting external link from file now");
                                f.get_link_uri_from_filelockentry()
                                    .map_dbg_str("Error happened while getting link URI from FLE")
                                    .map_dbg_err(|e| format!("URL -> Err = {:?}", e))
                            })
                    });

                match next {
                    Some(Ok(Some(link))) => return Some(Ok(link)),
                    Some(Ok(None))       => continue,
                    Some(Err(e))         => return Some(Err(e)),
                    None                 => return None
                }
            }
        }

    }

}


/// Check whether the StoreId starts with `/link/external/`
pub fn is_external_link_storeid<A: AsRef<StoreId> + Debug>(id: A) -> bool {
    debug!("Checking whether this is a 'links/external/': '{:?}'", id);
    id.as_ref().is_in_collection(&["links", "external"])
}

/// Implement `ExternalLinker` for `Entry`, hiding the fact that there is no such thing as an external
/// link in an entry, but internal links to other entries which serve as external links, as one
/// entry in the store can only have one external link.
impl ExternalLinker for Entry {

    /// Get the external links from the implementor object
    fn get_external_links<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>> {
        // Iterate through all internal links and filter for FileLockEntries which live in
        // /link/external/<SHA> -> load these files and get the external link from their headers,
        // put them into the return vector.
        self.get_internal_links()
            .map(|iter| {
                debug!("Getting external links");
                iter.only_external_links().urls(store)
            })
    }

    /// Set the external links for the implementor object
    ///
    /// # Return Value
    ///
    /// Returns the StoreIds which were newly created for the new external links, if there are more
    /// external links than before.
    /// If there are less external links than before, an empty vec![] is returned.
    ///
    fn set_external_links(&mut self, store: &Store, links: Vec<Url>) -> Result<Vec<StoreId>> {
        // Take all the links, generate a SHA sum out of each one, filter out the already existing
        // store entries and store the other URIs in the header of one FileLockEntry each, in
        // the path /link/external/<SHA of the URL>

        debug!("Iterating {} links = {:?}", links.len(), links);
        links.into_iter().map(|link| {
            let hash = hex::encode(Sha1::digest(&link.as_str().as_bytes()));
            let file_id = crate::module_path::new_id(format!("external/{}", hash))
                .map_dbg_err(|_| {
                    format!("Failed to build StoreId for this hash '{:?}'", hash)
                })?;

            debug!("Link    = '{:?}'", link);
            debug!("Hash    = '{:?}'", hash);
            debug!("StoreId = '{:?}'", file_id);

            let link_already_exists = store.get(file_id.clone())?.is_some();

            // retrieve the file from the store, which implicitely creates the entry if it does not
            // exist
            let mut file = store
                .retrieve(file_id.clone())
                .map_dbg_err(|_| {
                    format!("Failed to create or retrieve an file for this link '{:?}'", link)
                })?;

            debug!("Generating header content!");
            {
                let hdr = file.deref_mut().get_header_mut();

                let mut table = match hdr.read("links.external.content")? {
                    Some(&Value::Table(ref table)) => table.clone(),
                    Some(_) => {
                        warn!("There is a value at 'links.external.content' which is not a table.");
                        warn!("Going to override this value");
                        Map::new()
                    },
                    None => Map::new(),
                };

                let v = Value::String(link.into_string());

                debug!("setting URL = '{:?}", v);
                table.insert(String::from("url"), v);

                let _ = hdr.insert("links.external.content", Value::Table(table))?;
                debug!("Setting URL worked");
            }

            // then add an internal link to the new file or return an error if this fails
            let _ = self.add_internal_link(file.deref_mut())?;
            debug!("Error adding internal link");

            Ok((link_already_exists, file_id))
        })
        .filter_map(|res| match res {
            Ok((exists, entry)) => if exists { Some(Ok(entry)) } else { None },
            Err(e) => Some(Err(e))
        })
        .collect()
    }

    /// Add an external link to the implementor object
    ///
    /// # Return Value
    ///
    /// (See ExternalLinker::set_external_links())
    ///
    /// Returns the StoreIds which were newly created for the new external links, if there are more
    /// external links than before.
    /// If there are less external links than before, an empty vec![] is returned.
    ///
    fn add_external_link(&mut self, store: &Store, link: Url) -> Result<Vec<StoreId>> {
        // get external links, add this one, save them
        debug!("Getting links");
        self.get_external_links(store)
            .and_then(|links| {
                let mut links = links.collect::<Result<Vec<_>>>()?;

                debug!("Adding link = '{:?}' to links = {:?}", link, links);
                links.push(link);

                debug!("Setting {} links = {:?}", links.len(), links);
                self.set_external_links(store, links)
            })
    }

    /// Remove an external link from the implementor object
    ///
    /// # Return Value
    ///
    /// (See ExternalLinker::set_external_links())
    ///
    /// Returns the StoreIds which were newly created for the new external links, if there are more
    /// external links than before.
    /// If there are less external links than before, an empty vec![] is returned.
    ///
    fn remove_external_link(&mut self, store: &Store, link: Url) -> Result<Vec<StoreId>> {
        // get external links, remove this one, save them
        self.get_external_links(store)
            .and_then(|links| {
                debug!("Removing link = '{:?}'", link);
                let links = links
                    .filter_map(Result::ok)
                    .filter(|l| l.as_str() != link.as_str())
                    .collect::<Vec<_>>();
                self.set_external_links(store, links)
            })
    }

}

#[cfg(test)]
mod tests {
    use super::*;
    use std::path::PathBuf;

    use libimagstore::store::Store;

    fn setup_logging() {
        let _ = env_logger::try_init();
    }

    pub fn get_store() -> Store {
        Store::new_inmemory(PathBuf::from("/"), &None).unwrap()
    }


    #[test]
    fn test_simple() {
        setup_logging();
        let store = get_store();
        let mut e = store.retrieve(PathBuf::from("base-test_simple")).unwrap();
        let url   = Url::parse("http://google.de").unwrap();

        assert!(e.add_external_link(&store, url.clone()).is_ok());

        assert_eq!(1, e.get_external_links(&store).unwrap().count());
        assert_eq!(url, e.get_external_links(&store).unwrap().next().unwrap().unwrap());
    }

}


D lib/entry/libimagentrylink/src/internal.rs => lib/entry/libimagentrylink/src/internal.rs +0 -855
@@ 1,855 0,0 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use libimagstore::storeid::StoreId;
use libimagstore::storeid::IntoStoreId;
use libimagstore::store::Entry;
use libimagstore::store::Store;
use libimagerror::errors::ErrorMsg as EM;

use toml_query::read::TomlValueReadExt;
use toml_query::insert::TomlValueInsertExt;
use toml::map::Map;
use failure::ResultExt;
use failure::Fallible as Result;
use failure::Error;
use failure::err_msg;

use self::iter::LinkIter;
use self::iter::IntoValues;

use toml::Value;

#[derive(Eq, PartialOrd, Ord, Hash, Debug, Clone)]
pub enum Link {
    Id          { link: StoreId },
    Annotated   { link: StoreId, annotation: String },
}

impl Link {

    pub fn exists(&self, store: &Store) -> Result<bool> {
        match *self {
            Link::Id { ref link }             => store.exists(link.clone()),
            Link::Annotated { ref link, .. }  => store.exists(link.clone()),
        }
        .map_err(From::from)
    }

    pub fn to_str(&self) -> Result<String> {
        match *self {
            Link::Id { ref link }             => link.to_str(),
            Link::Annotated { ref link, .. }  => link.to_str(),
        }
        .map_err(From::from)
    }


    fn eq_store_id(&self, id: &StoreId) -> bool {
        match self {
            &Link::Id { link: ref s }             => s.eq(id),
            &Link::Annotated { link: ref s, .. }  => s.eq(id),
        }
    }

    /// Get the StoreId inside the Link, which is always present
    pub fn get_store_id(&self) -> &StoreId {
        match self {
            &Link::Id { link: ref s }             => s,
            &Link::Annotated { link: ref s, .. }  => s,
        }
    }

    /// Helper wrapper around Link for StoreId
    fn without_base(self) -> Link {
        match self {
            Link::Id { link: s } => Link::Id { link: s },
            Link::Annotated { link: s, annotation: ann } =>
                Link::Annotated { link: s, annotation: ann },
        }
    }

    fn to_value(&self) -> Result<Value> {
        match self {
            &Link::Id { link: ref s } =>
                s.to_str()
                .map(Value::String)
                .context(EM::ConversionError)
                .map_err(Error::from),
            &Link::Annotated { ref link, annotation: ref anno } => {
                link.to_str()
                    .map(Value::String)
                    .context(EM::ConversionError)
                    .map_err(Error::from)
                    .map(|link| {
                        let mut tab = Map::new();

                        tab.insert("link".to_owned(),       link);
                        tab.insert("annotation".to_owned(), Value::String(anno.clone()));
                        Value::Table(tab)
                    })
            }
        }
    }

}

impl ::std::cmp::PartialEq for Link {
    fn eq(&self, other: &Self) -> bool {
        match (self, other) {
            (&Link::Id { link: ref a }, &Link::Id { link: ref b }) => a.eq(&b),
            (&Link::Annotated { link: ref a, annotation: ref ann1 },
             &Link::Annotated { link: ref b, annotation: ref ann2 }) =>
                (a, ann1).eq(&(b, ann2)),
            _ => false,
        }
    }
}

impl From<StoreId> for Link {

    fn from(s: StoreId) -> Link {
        Link::Id { link: s }
    }
}

impl Into<StoreId> for Link {
    fn into(self) -> StoreId {
        match self {
            Link::Id { link }            => link,
            Link::Annotated { link, .. } => link,
        }
    }
}

impl IntoStoreId for Link {
    fn into_storeid(self) -> Result<StoreId> {
        match self {
            Link::Id { link }            => Ok(link),
            Link::Annotated { link, .. } => Ok(link),
        }
    }
}

impl AsRef<StoreId> for Link {
    fn as_ref(&self) -> &StoreId {
        match self {
            &Link::Id { ref link }            => &link,
            &Link::Annotated { ref link, .. } => &link,
        }
    }
}

pub trait InternalLinker {

    /// Get the internal links from the implementor object
    fn get_internal_links(&self) -> Result<LinkIter>;

    /// Add an internal link to the implementor object
    fn add_internal_link(&mut self, link: &mut Entry) -> Result<()>;

    /// Remove an internal link from the implementor object
    fn remove_internal_link(&mut self, link: &mut Entry) -> Result<()>;

    /// Remove _all_ internal links
    fn unlink(&mut self, store: &Store) -> Result<()>;

    /// Add internal annotated link
    fn add_internal_annotated_link(&mut self, link: &mut Entry, annotation: String) -> Result<()>;
}

pub mod iter {
    use std::vec::IntoIter;
    use super::Link;

    use failure::Error;
    use failure::Fallible as Result;
    use failure::ResultExt;
    use toml::Value;
    use itertools::Itertools;

    use libimagstore::store::Store;
    use libimagstore::store::FileLockEntry;
    use libimagerror::errors::ErrorMsg as EM;

    pub struct LinkIter(IntoIter<Link>);

    impl LinkIter {

        pub fn new(v: Vec<Link>) -> LinkIter {
            LinkIter(v.into_iter())
        }

        pub fn into_getter(self, store: &Store) -> GetIter {
            GetIter(self.0, store)
        }

    }

    impl Iterator for LinkIter {
        type Item = Link;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next()
        }
    }

    pub trait IntoValues {
        fn into_values(self) -> Vec<Result<Value>>;
    }

    impl<I: Iterator<Item = Link>> IntoValues for I {
        fn into_values(self) -> Vec<Result<Value>> {
            self.map(|s| s.without_base())
                .unique()
                .sorted()
                .into_iter() // Cannot sort toml::Value, hence uglyness here
                .map(|link| link.to_value().context(EM::ConversionError).map_err(Error::from))
                .collect()
        }
    }

    /// An Iterator that `Store::get()`s the Entries from the store while consumed
    pub struct GetIter<'a>(IntoIter<Link>, &'a Store);

    impl<'a> GetIter<'a> {
        pub fn new(i: IntoIter<Link>, store: &'a Store) -> GetIter<'a> {
            GetIter(i, store)
        }

        pub fn store(&self) -> &Store {
            self.1
        }
    }

    impl<'a> Iterator for GetIter<'a> {
        type Item = Result<FileLockEntry<'a>>;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next().and_then(|id| match self.1.get(id) {
                Ok(None)    => None,
                Ok(Some(x)) => Some(Ok(x)),
                Err(e)      => Some(Err(e).map_err(From::from)),
            })
        }

    }

}

impl InternalLinker for Entry {

    fn get_internal_links(&self) -> Result<LinkIter> {
        debug!("Getting internal links");
        trace!("Getting internal links from header of '{}' = {:?}", self.get_location(), self.get_header());
        let res = self
            .get_header()
            .read("links.internal")
            .context(format_err!("Failed to read header 'links.internal' of '{}'", self.get_location()))
            .context(EM::EntryHeaderReadError)
            .context(EM::EntryHeaderError)
            .map_err(Error::from)
            .map(|r| r.cloned());
        process_rw_result(res)
    }

    fn add_internal_link(&mut self, link: &mut Entry) -> Result<()> {
        debug!("Adding internal link: {:?}", link);
        let location = link.get_location().clone().into();
        add_internal_link_with_instance(self, link, location)
    }

    fn remove_internal_link(&mut self, link: &mut Entry) -> Result<()> {
        debug!("Removing internal link: {:?}", link);

        // Cloning because of borrowing
        let own_loc   = self.get_location().clone();
        let other_loc = link.get_location().clone();

        debug!("Removing internal link from {:?} to {:?}", own_loc, other_loc);

        let links = link.get_internal_links()?;
        debug!("Rewriting own links for {:?}, without {:?}", other_loc, own_loc);

        let links = links.filter(|l| !l.eq_store_id(&own_loc));
        let _     = rewrite_links(link.get_header_mut(), links)?;

        self.get_internal_links()
            .and_then(|links| {
                debug!("Rewriting own links for {:?}, without {:?}", own_loc, other_loc);
                let links = links.filter(|l| !l.eq_store_id(&other_loc));
                rewrite_links(self.get_header_mut(), links)
            })
    }

    fn unlink(&mut self, store: &Store) -> Result<()> {
        for id in self.get_internal_links()?.map(|l| l.get_store_id().clone()) {
            match store.get(id).context("Failed to get entry")? {
                Some(mut entry) => self.remove_internal_link(&mut entry)?,
                None            => return Err(err_msg("Link target does not exist")),
            }
        }

        Ok(())
    }

    fn add_internal_annotated_link(&mut self, link: &mut Entry, annotation: String) -> Result<()> {
        let new_link = Link::Annotated {
            link: link.get_location().clone(),
            annotation: annotation,
        };

        add_internal_link_with_instance(self, link, new_link)
    }

}

fn add_internal_link_with_instance(this: &mut Entry, link: &mut Entry, instance: Link) -> Result<()> {
    debug!("Adding internal link from {:?} to {:?}", this.get_location(), instance);

    add_foreign_link(link, this.get_location().clone())
        .and_then(|_| {
            this.get_internal_links()
                .and_then(|links| {
                    let links = links.chain(LinkIter::new(vec![instance]));
                    rewrite_links(this.get_header_mut(), links)
                })
        })
}

fn rewrite_links<I: Iterator<Item = Link>>(header: &mut Value, links: I) -> Result<()> {
    let links = links.into_values()
                     .into_iter()
                     .fold(Ok(vec![]), |acc: Result<Vec<_>>, elem| {
                        acc.and_then(move |mut v| {
                            v.push(elem.context(EM::ConversionError)?);
                            Ok(v)
                        })
                     })?;

    debug!("Setting new link array: {:?}", links);
    let process = header
        .insert("links.internal", Value::Array(links))
        .context(format_err!("Failed to insert header 'links.internal'"))
        .context(EM::EntryHeaderReadError)
        .map_err(Error::from);
    process_rw_result(process).map(|_| ())
}

/// When Linking A -> B, the specification wants us to link back B -> A.
/// This is a helper function which does this.
fn add_foreign_link(target: &mut Entry, from: StoreId) -> Result<()> {
    debug!("Linking back from {:?} to {:?}", target.get_location(), from);
    target.get_internal_links()
        .and_then(|links| {
            let links = links
                             .chain(LinkIter::new(vec![from.into()]))
                             .into_values()
                             .into_iter()
                             .fold(Ok(vec![]), |acc: Result<Vec<_>>, elem| {
                                acc.and_then(move |mut v| {
                                    v.push(elem.context(EM::ConversionError)?);
                                    Ok(v)
                                })
                             })?;
            debug!("Setting links in {:?}: {:?}", target.get_location(), links);

            let res = target
                .get_header_mut()
                .insert("links.internal", Value::Array(links))
                .context(format_err!("Failed to insert header 'links.internal'"))
                .context(EM::EntryHeaderReadError)
                .map_err(Error::from);

            process_rw_result(res).map(|_| ())
        })
}

fn process_rw_result(links: Result<Option<Value>>) -> Result<LinkIter> {
    use std::path::PathBuf;

    let links = match links {
        Err(e) => {
            debug!("RW action on store failed. Generating LinkError");
            return Err(e).context(EM::EntryHeaderReadError).map_err(Error::from)
        },
        Ok(None) => {
            debug!("We got no value from the header!");
            return Ok(LinkIter::new(vec![]))
        },
        Ok(Some(Value::Array(l))) => l,
        Ok(Some(_)) => {
            debug!("We expected an Array for the links, but there was a non-Array!");
            return Err(err_msg("Link type error"));
        }
    };

    if !links.iter().all(|l| is_match!(*l, Value::String(_)) || is_match!(*l, Value::Table(_))) {
        debug!("At least one of the Values which were expected in the Array of links is not a String or a Table!");
        debug!("Generating LinkError");
        return Err(err_msg("Existing Link type error"));
    }

    let links : Vec<Link> = links.into_iter()
        .map(|link| {
            debug!("Matching the link: {:?}", link);
            match link {
                Value::String(s) => StoreId::new(PathBuf::from(s))
                    .map(|s| Link::Id { link: s })
                    .map_err(From::from)
                    ,
                Value::Table(mut tab) => {
                    debug!("Destructuring table");
                    if !tab.contains_key("link")
                    || !tab.contains_key("annotation") {
                        debug!("Things missing... returning Error instance");
                        Err(err_msg("Link parser error"))
                    } else {
                        let link = tab.remove("link")
                            .ok_or(err_msg("Link parser: field missing"))?;

                        let anno = tab.remove("annotation")
                            .ok_or(err_msg("Link parser: Field missing"))?;

                        debug!("Ok, here we go with building a Link::Annotated");
                        match (link, anno) {
                            (Value::String(link), Value::String(anno)) => {
                                StoreId::new(PathBuf::from(link))
                                    .map_err(From::from)
                                    .map(|link| {
                                        Link::Annotated {
                                            link: link,
                                            annotation: anno,
                                        }
                                    })
                            },
                            _ => Err(err_msg("Link parser: Field type error")),
                        }
                    }
                }
                _ => unreachable!(),
            }
        })
        .collect::<Result<Vec<Link>>>()?;

    debug!("Ok, the RW action was successful, returning link vector now!");
    Ok(LinkIter::new(links))
}

pub mod store_check {
    use libimagstore::store::Store;

    use failure::ResultExt;
    use failure::Fallible as Result;
    use failure::Error;
    use failure::err_msg;

    pub trait StoreLinkConsistentExt {
        fn check_link_consistency(&self) -> Result<()>;
    }

    impl StoreLinkConsistentExt for Store {
        fn check_link_consistency(&self) -> Result<()> {
            use std::collections::HashMap;

            use crate::internal::InternalLinker;

            use libimagstore::storeid::StoreId;
            use libimagutil::debug_result::DebugResult;

            // Helper data structure to collect incoming and outgoing links for each StoreId
            #[derive(Debug, Default)]
            struct Linking {
                outgoing: Vec<StoreId>,
                incoming: Vec<StoreId>,
            }

            // Helper function to aggregate the Link network
            //
            // This function aggregates a HashMap which maps each StoreId object in the store onto
            // a Linking object, which contains a list of StoreIds which this entry links to and a
            // list of StoreIds which link to the current one.
            //
            // The lambda returns an error if something fails
            let aggregate_link_network = |store: &Store| -> Result<HashMap<StoreId, Linking>> {
                store
                    .entries()?
                    .into_get_iter()
                    .fold(Ok(HashMap::new()), |map, element| {
                        map.and_then(|mut map| {
                            debug!("Checking element = {:?}", element);
                            let entry = element?.ok_or_else(|| err_msg("TODO: Not yet handled"))?;

                            debug!("Checking entry = {:?}", entry.get_location());

                            let internal_links = entry
                                .get_internal_links()?
                                .into_getter(store); // get the FLEs from the Store

                            let mut linking = Linking::default();
                            for internal_link in internal_links {
                                debug!("internal link = {:?}", internal_link);

                                linking.outgoing.push(internal_link?.get_location().clone());
                                linking.incoming.push(entry.get_location().clone());
                            }

                            map.insert(entry.get_location().clone(), linking);
                            Ok(map)
                        })
                    })
            };

            // Helper to check whethre all StoreIds in the network actually exists
            //
            // Because why not?
            let all_collected_storeids_exist = |network: &HashMap<StoreId, Linking>| -> Result<()> {
                for (id, _) in network.iter() {
                    if is_match!(self.get(id.clone()), Ok(Some(_))) {
                        debug!("Exists in store: {:?}", id);

                        if !self.exists(id.clone())? {
                            warn!("Does exist in store but not on FS: {:?}", id);
                            return Err(err_msg("Link target does not exist"))
                        }
                    } else {
                        warn!("Does not exist in store: {:?}", id);
                        return Err(err_msg("Link target does not exist"))
                    }
                }

                Ok(())
            };

            // Helper function to create a SLCECD::OneDirectionalLink error object
            let mk_one_directional_link_err = |src: StoreId, target: StoreId| -> Error {
                Error::from(format_err!("Dead link: {} -> {}",
                                        src.local_display_string(),
                                        target.local_display_string()))
            };

            // Helper lambda to check whether the _incoming_ links of each entry actually also
            // appear in the _outgoing_ list of the linked entry
            let incoming_links_exists_as_outgoing_links =
                |src: &StoreId, linking: &Linking, network: &HashMap<StoreId, Linking>| -> Result<()> {
                    for link in linking.incoming.iter() {
                        // Check whether the links which are _incoming_ on _src_ are outgoing
                        // in each of the links in the incoming list.
                        let incoming_consistent = network.get(link)
                            .map(|l| l.outgoing.contains(src))
                            .unwrap_or(false);

                        if !incoming_consistent {
                            return Err(mk_one_directional_link_err(src.clone(), link.clone()))
                        }
                    }

                    Ok(())
                };

            // Helper lambda to check whether the _outgoing links of each entry actually also
            // appear in the _incoming_ list of the linked entry
            let outgoing_links_exist_as_incoming_links =
                |src: &StoreId, linking: &Linking, network: &HashMap<StoreId, Linking>| -> Result<()> {
                    for link in linking.outgoing.iter() {
                        // Check whether the links which are _outgoing_ on _src_ are incoming
                        // in each of the links in the outgoing list.
                        let outgoing_consistent = network.get(link)
                            .map(|l| l.incoming.contains(src))
                            .unwrap_or(false);

                        if !outgoing_consistent {
                            return Err(mk_one_directional_link_err(link.clone(), src.clone()))
                        }
                    }

                    Ok(())
                };

            aggregate_link_network(&self)
                .map_dbg_str("Aggregated")
                .map_dbg(|nw| {
                    let mut s = String::new();
                    for (k, v) in nw {
                        s.push_str(&format!("{}\n in: {:?}\n out: {:?}", k, v.incoming, v.outgoing));
                    }
                    s
                })
                .and_then(|nw| {
                    all_collected_storeids_exist(&nw)
                        .map(|_| nw)
                        .context(err_msg("Link handling error"))
                        .map_err(Error::from)
                })
                .and_then(|nw| {
                    for (id, linking) in nw.iter() {
                        incoming_links_exists_as_outgoing_links(id, linking, &nw)?;
                        outgoing_links_exist_as_incoming_links(id, linking, &nw)?;
                    }
                    Ok(())
                })
                .map(|_| ())
        }
    }

}

#[cfg(test)]
mod test {
    use std::path::PathBuf;

    use libimagstore::store::Store;

    use super::InternalLinker;
    use super::Link;

    fn setup_logging() {
        let _ = ::env_logger::try_init();
    }

    pub fn get_store() -> Store {
        Store::new_inmemory(PathBuf::from("/"), &None).unwrap()
    }

    #[test]
    fn test_new_entry_no_links() {
        setup_logging();
        let store = get_store();
        let entry = store.create(PathBuf::from("test_new_entry_no_links")).unwrap();
        let links = entry.get_internal_links();
        assert!(links.is_ok());
        let links = links.unwrap();
        assert_eq!(links.collect::<Vec<_>>().len(), 0);
    }

    #[test]
    fn test_link_two_entries() {
        setup_logging();
        let store = get_store();
        let mut e1 = store.create(PathBuf::from("test_link_two_entries1")).unwrap();
        assert!(e1.get_internal_links().is_ok());

        let mut e2 = store.create(PathBuf::from("test_link_two_entries2")).unwrap();
        assert!(e2.get_internal_links().is_ok());

        {
            assert!(e1.add_internal_link(&mut e2).is_ok());

            let e1_links = e1.get_internal_links().unwrap().collect::<Vec<_>>();
            let e2_links = e2.get_internal_links().unwrap().collect::<Vec<_>>();

            debug!("1 has links: {:?}", e1_links);
            debug!("2 has links: {:?}", e2_links);

            assert_eq!(e1_links.len(), 1);
            assert_eq!(e2_links.len(), 1);

            assert!(e1_links.first().map(|l| l.clone().eq_store_id(e2.get_location())).unwrap_or(false));
            assert!(e2_links.first().map(|l| l.clone().eq_store_id(e1.get_location())).unwrap_or(false));
        }

        {
            assert!(e1.remove_internal_link(&mut e2).is_ok());

            debug!("{:?}", e2.to_str());
            let e2_links = e2.get_internal_links().unwrap().collect::<Vec<_>>();
            assert_eq!(e2_links.len(), 0, "Expected [], got: {:?}", e2_links);

            debug!("{:?}", e1.to_str());
            let e1_links = e1.get_internal_links().unwrap().collect::<Vec<_>>();
            assert_eq!(e1_links.len(), 0, "Expected [], got: {:?}", e1_links);

        }
    }

    #[test]
    fn test_multiple_links() {
        setup_logging();
        let store = get_store();

        let mut e1 = store.retrieve(PathBuf::from("1")).unwrap();
        let mut e2 = store.retrieve(PathBuf::from("2")).unwrap();
        let mut e3 = store.retrieve(PathBuf::from("3")).unwrap();
        let mut e4 = store.retrieve(PathBuf::from("4")).unwrap();
        let mut e5 = store.retrieve(PathBuf::from("5")).unwrap();

        assert!(e1.add_internal_link(&mut e2).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e3).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e4).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 3);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e5).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 4);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);

        assert!(e5.remove_internal_link(&mut e1).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 3);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e4.remove_internal_link(&mut e1).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e3.remove_internal_link(&mut e1).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e2.remove_internal_link(&mut e1).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

    }

    #[test]
    fn test_link_deleting() {
        setup_logging();
        let store = get_store();

        let mut e1 = store.retrieve(PathBuf::from("1")).unwrap();
        let mut e2 = store.retrieve(PathBuf::from("2")).unwrap();

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e2).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);

        assert!(e1.remove_internal_link(&mut e2).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
    }

    #[test]
    fn test_link_deleting_multiple_links() {
        setup_logging();
        let store = get_store();

        let mut e1 = store.retrieve(PathBuf::from("1")).unwrap();
        let mut e2 = store.retrieve(PathBuf::from("2")).unwrap();
        let mut e3 = store.retrieve(PathBuf::from("3")).unwrap();

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e2).is_ok()); // 1-2
        assert!(e1.add_internal_link(&mut e3).is_ok()); // 1-2, 1-3

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);

        assert!(e2.add_internal_link(&mut e3).is_ok()); // 1-2, 1-3, 2-3

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);

        assert!(e1.remove_internal_link(&mut e2).is_ok()); // 1-3, 2-3

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);

        assert!(e1.remove_internal_link(&mut e3).is_ok()); // 2-3

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);

        assert!(e2.remove_internal_link(&mut e3).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
    }

    #[test]
    fn test_link_annotating() {
        setup_logging();
        let store      = get_store();
        let mut entry1 = store.create(PathBuf::from("test_link_annotating-1")).unwrap();
        let mut entry2 = store.create(PathBuf::from("test_link_annotating-2")).unwrap();

        let res = entry1.add_internal_annotated_link(&mut entry2, String::from("annotation"));
        assert!(res.is_ok());

        {
            for link in entry1.get_internal_links().unwrap() {
                match link  {
                    Link::Annotated {annotation, ..} => assert_eq!(annotation, "annotation"),
                    _ => assert!(false, "Non-annotated link found"),
                }
            }
        }

        {
            for link in entry2.get_internal_links().unwrap() {
                match link  {
                    Link::Id {..}        => {},
                    Link::Annotated {..} => assert!(false, "Annotated link found"),
                }
            }
        }
    }

}


A lib/entry/libimagentrylink/src/iter.rs => lib/entry/libimagentrylink/src/iter.rs +97 -0
@@ 0,0 1,97 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//


use std::vec::IntoIter;

use failure::Error;
use failure::ResultExt;
use failure::Fallible as Result;
use toml::Value;
use itertools::Itertools;

use libimagstore::store::Store;
use libimagstore::store::FileLockEntry;
use libimagerror::errors::ErrorMsg as EM;

use crate::link::Link;

pub struct LinkIter(IntoIter<Link>);

impl LinkIter {

    pub fn new(v: Vec<Link>) -> LinkIter {
        LinkIter(v.into_iter())
    }

    pub fn into_getter(self, store: &Store) -> GetIter {
        GetIter(self.0, store)
    }

}

impl Iterator for LinkIter {
    type Item = Link;

    fn next(&mut self) -> Option<Self::Item> {
        self.0.next()
    }
}

pub trait IntoValues {
    fn into_values(self) -> Vec<Result<Value>>;
}

impl<I: Iterator<Item = Link>> IntoValues for I {
    fn into_values(self) -> Vec<Result<Value>> {
        self.map(|s| s.without_base())
            .unique()
            .sorted()
            .into_iter() // Cannot sort toml::Value, hence uglyness here
            .map(|link| link.to_value().context(EM::ConversionError).map_err(Error::from))
            .collect()
    }
}

/// An Iterator that `Store::get()`s the Entries from the store while consumed
pub struct GetIter<'a>(IntoIter<Link>, &'a Store);

impl<'a> GetIter<'a> {
    pub fn new(i: IntoIter<Link>, store: &'a Store) -> GetIter<'a> {
        GetIter(i, store)
    }

    pub fn store(&self) -> &Store {
        self.1
    }
}

impl<'a> Iterator for GetIter<'a> {
    type Item = Result<FileLockEntry<'a>>;

    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().and_then(|id| match self.1.get(id) {
            Ok(None)    => None,
            Ok(Some(x)) => Some(Ok(x)),
            Err(e)      => Some(Err(e).map_err(From::from)),
        })
    }

}


M lib/entry/libimagentrylink/src/lib.rs => lib/entry/libimagentrylink/src/lib.rs +4 -2
@@ 56,6 56,8 @@ extern crate libimagutil;

module_entry_path_mod!("links");

pub mod external;
pub mod internal;
pub mod iter;
pub mod linker;
pub mod link;
pub mod storecheck;


A lib/entry/libimagentrylink/src/link.rs => lib/entry/libimagentrylink/src/link.rs +150 -0
@@ 0,0 1,150 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use libimagstore::storeid::StoreId;
use libimagstore::storeid::IntoStoreId;
use libimagstore::store::Store;
use libimagerror::errors::ErrorMsg as EM;

use toml::Value;
use toml::map::Map;
use failure::ResultExt;
use failure::Fallible as Result;
use failure::Error;

#[derive(Eq, PartialOrd, Ord, Hash, Debug, Clone)]
pub enum Link {
    Id          { link: StoreId },
    Annotated   { link: StoreId, annotation: String },
}

impl Link {

    pub fn exists(&self, store: &Store) -> Result<bool> {
        match *self {
            Link::Id { ref link }             => store.exists(link.clone()),
            Link::Annotated { ref link, .. }  => store.exists(link.clone()),
        }
        .map_err(From::from)
    }

    pub fn to_str(&self) -> Result<String> {
        match *self {
            Link::Id { ref link }             => link.to_str(),
            Link::Annotated { ref link, .. }  => link.to_str(),
        }
        .map_err(From::from)
    }


    pub(crate) fn eq_store_id(&self, id: &StoreId) -> bool {
        match self {
            &Link::Id { link: ref s }             => s.eq(id),
            &Link::Annotated { link: ref s, .. }  => s.eq(id),
        }
    }

    /// Get the StoreId inside the Link, which is always present
    pub fn get_store_id(&self) -> &StoreId {
        match self {
            &Link::Id { link: ref s }             => s,
            &Link::Annotated { link: ref s, .. }  => s,
        }
    }

    /// Helper wrapper around Link for StoreId
    pub(crate) fn without_base(self) -> Link {
        match self {
            Link::Id { link: s } => Link::Id { link: s },
            Link::Annotated { link: s, annotation: ann } =>
                Link::Annotated { link: s, annotation: ann },
        }
    }

    pub(crate) fn to_value(&self) -> Result<Value> {
        match self {
            &Link::Id { link: ref s } =>
                s.to_str()
                .map(Value::String)
                .context(EM::ConversionError)
                .map_err(Error::from),
            &Link::Annotated { ref link, annotation: ref anno } => {
                link.to_str()
                    .map(Value::String)
                    .context(EM::ConversionError)
                    .map_err(Error::from)
                    .map(|link| {
                        let mut tab = Map::new();

                        tab.insert("link".to_owned(),       link);
                        tab.insert("annotation".to_owned(), Value::String(anno.clone()));
                        Value::Table(tab)
                    })
            }
        }
    }

}

impl ::std::cmp::PartialEq for Link {
    fn eq(&self, other: &Self) -> bool {
        match (self, other) {
            (&Link::Id { link: ref a }, &Link::Id { link: ref b }) => a.eq(&b),
            (&Link::Annotated { link: ref a, annotation: ref ann1 },
             &Link::Annotated { link: ref b, annotation: ref ann2 }) =>
                (a, ann1).eq(&(b, ann2)),
            _ => false,
        }
    }
}

impl From<StoreId> for Link {

    fn from(s: StoreId) -> Link {
        Link::Id { link: s }
    }
}

impl Into<StoreId> for Link {
    fn into(self) -> StoreId {
        match self {
            Link::Id { link }            => link,
            Link::Annotated { link, .. } => link,
        }
    }
}

impl IntoStoreId for Link {
    fn into_storeid(self) -> Result<StoreId> {
        match self {
            Link::Id { link }            => Ok(link),
            Link::Annotated { link, .. } => Ok(link),
        }
    }
}

impl AsRef<StoreId> for Link {
    fn as_ref(&self) -> &StoreId {
        match self {
            &Link::Id { ref link }            => &link,
            &Link::Annotated { ref link, .. } => &link,
        }
    }
}


A lib/entry/libimagentrylink/src/linker.rs => lib/entry/libimagentrylink/src/linker.rs +496 -0
@@ 0,0 1,496 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use libimagstore::storeid::StoreId;
use libimagstore::store::Entry;
use libimagstore::store::Store;
use libimagerror::errors::ErrorMsg as EM;

use toml_query::read::TomlValueReadExt;
use toml_query::insert::TomlValueInsertExt;
use failure::ResultExt;
use failure::Fallible as Result;
use failure::Error;
use failure::err_msg;

use crate::iter::LinkIter;
use crate::iter::IntoValues;
use crate::link::Link;

use toml::Value;

pub trait InternalLinker {

    /// Get the internal links from the implementor object
    fn get_internal_links(&self) -> Result<LinkIter>;

    /// Add an internal link to the implementor object
    fn add_internal_link(&mut self, link: &mut Entry) -> Result<()>;

    /// Remove an internal link from the implementor object
    fn remove_internal_link(&mut self, link: &mut Entry) -> Result<()>;

    /// Remove _all_ internal links
    fn unlink(&mut self, store: &Store) -> Result<()>;

    /// Add internal annotated link
    fn add_internal_annotated_link(&mut self, link: &mut Entry, annotation: String) -> Result<()>;
}

impl InternalLinker for Entry {

    fn get_internal_links(&self) -> Result<LinkIter> {
        debug!("Getting internal links");
        trace!("Getting internal links from header of '{}' = {:?}", self.get_location(), self.get_header());
        let res = self
            .get_header()
            .read("links.internal")
            .context(format_err!("Failed to read header 'links.internal' of '{}'", self.get_location()))
            .context(EM::EntryHeaderReadError)
            .context(EM::EntryHeaderError)
            .map_err(Error::from)
            .map(|r| r.cloned());
        process_rw_result(res)
    }

    fn add_internal_link(&mut self, link: &mut Entry) -> Result<()> {
        debug!("Adding internal link: {:?}", link);
        let location = link.get_location().clone().into();
        add_internal_link_with_instance(self, link, location)
    }

    fn remove_internal_link(&mut self, link: &mut Entry) -> Result<()> {
        debug!("Removing internal link: {:?}", link);

        // Cloning because of borrowing
        let own_loc   = self.get_location().clone();
        let other_loc = link.get_location().clone();

        debug!("Removing internal link from {:?} to {:?}", own_loc, other_loc);

        let links = link.get_internal_links()?;
        debug!("Rewriting own links for {:?}, without {:?}", other_loc, own_loc);

        let links = links.filter(|l| !l.eq_store_id(&own_loc));
        let _     = rewrite_links(link.get_header_mut(), links)?;

        self.get_internal_links()
            .and_then(|links| {
                debug!("Rewriting own links for {:?}, without {:?}", own_loc, other_loc);
                let links = links.filter(|l| !l.eq_store_id(&other_loc));
                rewrite_links(self.get_header_mut(), links)
            })
    }

    fn unlink(&mut self, store: &Store) -> Result<()> {
        for id in self.get_internal_links()?.map(|l| l.get_store_id().clone()) {
            match store.get(id).context("Failed to get entry")? {
                Some(mut entry) => self.remove_internal_link(&mut entry)?,
                None            => return Err(err_msg("Link target does not exist")),
            }
        }

        Ok(())
    }

    fn add_internal_annotated_link(&mut self, link: &mut Entry, annotation: String) -> Result<()> {
        let new_link = Link::Annotated {
            link: link.get_location().clone(),
            annotation: annotation,
        };

        add_internal_link_with_instance(self, link, new_link)
    }

}

fn add_internal_link_with_instance(this: &mut Entry, link: &mut Entry, instance: Link) -> Result<()> {
    debug!("Adding internal link from {:?} to {:?}", this.get_location(), instance);

    add_foreign_link(link, this.get_location().clone())
        .and_then(|_| {
            this.get_internal_links()
                .and_then(|links| {
                    let links = links.chain(LinkIter::new(vec![instance]));
                    rewrite_links(this.get_header_mut(), links)
                })
        })
}

fn rewrite_links<I: Iterator<Item = Link>>(header: &mut Value, links: I) -> Result<()> {
    let links = links.into_values()
                     .into_iter()
                     .fold(Ok(vec![]), |acc: Result<Vec<_>>, elem| {
                        acc.and_then(move |mut v| {
                            v.push(elem.context(EM::ConversionError)?);
                            Ok(v)
                        })
                     })?;

    debug!("Setting new link array: {:?}", links);
    let process = header
        .insert("links.internal", Value::Array(links))
        .context(format_err!("Failed to insert header 'links.internal'"))
        .context(EM::EntryHeaderReadError)
        .map_err(Error::from);
    process_rw_result(process).map(|_| ())
}

/// When Linking A -> B, the specification wants us to link back B -> A.
/// This is a helper function which does this.
fn add_foreign_link(target: &mut Entry, from: StoreId) -> Result<()> {
    debug!("Linking back from {:?} to {:?}", target.get_location(), from);
    target.get_internal_links()
        .and_then(|links| {
            let links = links
                             .chain(LinkIter::new(vec![from.into()]))
                             .into_values()
                             .into_iter()
                             .fold(Ok(vec![]), |acc: Result<Vec<_>>, elem| {
                                acc.and_then(move |mut v| {
                                    v.push(elem.context(EM::ConversionError)?);
                                    Ok(v)
                                })
                             })?;
            debug!("Setting links in {:?}: {:?}", target.get_location(), links);

            let res = target
                .get_header_mut()
                .insert("links.internal", Value::Array(links))
                .context(format_err!("Failed to insert header 'links.internal'"))
                .context(EM::EntryHeaderReadError)
                .map_err(Error::from);

            process_rw_result(res).map(|_| ())
        })
}

fn process_rw_result(links: Result<Option<Value>>) -> Result<LinkIter> {
    use std::path::PathBuf;

    let links = match links {
        Err(e) => {
            debug!("RW action on store failed. Generating LinkError");
            return Err(e).context(EM::EntryHeaderReadError).map_err(Error::from)
        },
        Ok(None) => {
            debug!("We got no value from the header!");
            return Ok(LinkIter::new(vec![]))
        },
        Ok(Some(Value::Array(l))) => l,
        Ok(Some(_)) => {
            debug!("We expected an Array for the links, but there was a non-Array!");
            return Err(err_msg("Link type error"));
        }
    };

    if !links.iter().all(|l| is_match!(*l, Value::String(_)) || is_match!(*l, Value::Table(_))) {
        debug!("At least one of the Values which were expected in the Array of links is not a String or a Table!");
        debug!("Generating LinkError");
        return Err(err_msg("Existing Link type error"));
    }

    let links : Vec<Link> = links.into_iter()
        .map(|link| {
            debug!("Matching the link: {:?}", link);
            match link {
                Value::String(s) => StoreId::new(PathBuf::from(s))
                    .map(|s| Link::Id { link: s })
                    .map_err(From::from)
                    ,
                Value::Table(mut tab) => {
                    debug!("Destructuring table");
                    if !tab.contains_key("link")
                    || !tab.contains_key("annotation") {
                        debug!("Things missing... returning Error instance");
                        Err(err_msg("Link parser error"))
                    } else {
                        let link = tab.remove("link")
                            .ok_or(err_msg("Link parser: field missing"))?;

                        let anno = tab.remove("annotation")
                            .ok_or(err_msg("Link parser: Field missing"))?;

                        debug!("Ok, here we go with building a Link::Annotated");
                        match (link, anno) {
                            (Value::String(link), Value::String(anno)) => {
                                StoreId::new(PathBuf::from(link))
                                    .map_err(From::from)
                                    .map(|link| {
                                        Link::Annotated {
                                            link: link,
                                            annotation: anno,
                                        }
                                    })
                            },
                            _ => Err(err_msg("Link parser: Field type error")),
                        }
                    }
                }
                _ => unreachable!(),
            }
        })
        .collect::<Result<Vec<Link>>>()?;

    debug!("Ok, the RW action was successful, returning link vector now!");
    Ok(LinkIter::new(links))
}

#[cfg(test)]
mod test {
    use std::path::PathBuf;

    use libimagstore::store::Store;

    use super::InternalLinker;
    use super::Link;

    fn setup_logging() {
        let _ = ::env_logger::try_init();
    }

    pub fn get_store() -> Store {
        Store::new_inmemory(PathBuf::from("/"), &None).unwrap()
    }

    #[test]
    fn test_new_entry_no_links() {
        setup_logging();
        let store = get_store();
        let entry = store.create(PathBuf::from("test_new_entry_no_links")).unwrap();
        let links = entry.get_internal_links();
        assert!(links.is_ok());
        let links = links.unwrap();
        assert_eq!(links.collect::<Vec<_>>().len(), 0);
    }

    #[test]
    fn test_link_two_entries() {
        setup_logging();
        let store = get_store();
        let mut e1 = store.create(PathBuf::from("test_link_two_entries1")).unwrap();
        assert!(e1.get_internal_links().is_ok());

        let mut e2 = store.create(PathBuf::from("test_link_two_entries2")).unwrap();
        assert!(e2.get_internal_links().is_ok());

        {
            assert!(e1.add_internal_link(&mut e2).is_ok());

            let e1_links = e1.get_internal_links().unwrap().collect::<Vec<_>>();
            let e2_links = e2.get_internal_links().unwrap().collect::<Vec<_>>();

            debug!("1 has links: {:?}", e1_links);
            debug!("2 has links: {:?}", e2_links);

            assert_eq!(e1_links.len(), 1);
            assert_eq!(e2_links.len(), 1);

            assert!(e1_links.first().map(|l| l.clone().eq_store_id(e2.get_location())).unwrap_or(false));
            assert!(e2_links.first().map(|l| l.clone().eq_store_id(e1.get_location())).unwrap_or(false));
        }

        {
            assert!(e1.remove_internal_link(&mut e2).is_ok());

            debug!("{:?}", e2.to_str());
            let e2_links = e2.get_internal_links().unwrap().collect::<Vec<_>>();
            assert_eq!(e2_links.len(), 0, "Expected [], got: {:?}", e2_links);

            debug!("{:?}", e1.to_str());
            let e1_links = e1.get_internal_links().unwrap().collect::<Vec<_>>();
            assert_eq!(e1_links.len(), 0, "Expected [], got: {:?}", e1_links);

        }
    }

    #[test]
    fn test_multiple_links() {
        setup_logging();
        let store = get_store();

        let mut e1 = store.retrieve(PathBuf::from("1")).unwrap();
        let mut e2 = store.retrieve(PathBuf::from("2")).unwrap();
        let mut e3 = store.retrieve(PathBuf::from("3")).unwrap();
        let mut e4 = store.retrieve(PathBuf::from("4")).unwrap();
        let mut e5 = store.retrieve(PathBuf::from("5")).unwrap();

        assert!(e1.add_internal_link(&mut e2).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e3).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e4).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 3);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e5).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 4);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);

        assert!(e5.remove_internal_link(&mut e1).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 3);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e4.remove_internal_link(&mut e1).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e3.remove_internal_link(&mut e1).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e2.remove_internal_link(&mut e1).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e4.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e5.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

    }

    #[test]
    fn test_link_deleting() {
        setup_logging();
        let store = get_store();

        let mut e1 = store.retrieve(PathBuf::from("1")).unwrap();
        let mut e2 = store.retrieve(PathBuf::from("2")).unwrap();

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e2).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);

        assert!(e1.remove_internal_link(&mut e2).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
    }

    #[test]
    fn test_link_deleting_multiple_links() {
        setup_logging();
        let store = get_store();

        let mut e1 = store.retrieve(PathBuf::from("1")).unwrap();
        let mut e2 = store.retrieve(PathBuf::from("2")).unwrap();
        let mut e3 = store.retrieve(PathBuf::from("3")).unwrap();

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);

        assert!(e1.add_internal_link(&mut e2).is_ok()); // 1-2
        assert!(e1.add_internal_link(&mut e3).is_ok()); // 1-2, 1-3

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);

        assert!(e2.add_internal_link(&mut e3).is_ok()); // 1-2, 1-3, 2-3

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);

        assert!(e1.remove_internal_link(&mut e2).is_ok()); // 1-3, 2-3

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 2);

        assert!(e1.remove_internal_link(&mut e3).is_ok()); // 2-3

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 1);

        assert!(e2.remove_internal_link(&mut e3).is_ok());

        assert_eq!(e1.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e2.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
        assert_eq!(e3.get_internal_links().unwrap().collect::<Vec<_>>().len(), 0);
    }

    #[test]
    fn test_link_annotating() {
        setup_logging();
        let store      = get_store();
        let mut entry1 = store.create(PathBuf::from("test_link_annotating-1")).unwrap();
        let mut entry2 = store.create(PathBuf::from("test_link_annotating-2")).unwrap();

        let res = entry1.add_internal_annotated_link(&mut entry2, String::from("annotation"));
        assert!(res.is_ok());

        {
            for link in entry1.get_internal_links().unwrap() {
                match link  {
                    Link::Annotated {annotation, ..} => assert_eq!(annotation, "annotation"),
                    _ => assert!(false, "Non-annotated link found"),
                }
            }
        }

        {
            for link in entry2.get_internal_links().unwrap() {
                match link  {
                    Link::Id {..}        => {},
                    Link::Annotated {..} => assert!(false, "Annotated link found"),
                }
            }
        }
    }

}

A lib/entry/libimagentrylink/src/storecheck.rs => lib/entry/libimagentrylink/src/storecheck.rs +173 -0
@@ 0,0 1,173 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use std::collections::HashMap;

use libimagstore::store::Store;
use libimagstore::storeid::StoreId;
use libimagutil::debug_result::DebugResult;

use failure::ResultExt;
use failure::Fallible as Result;
use failure::Error;
use failure::err_msg;

use crate::linker::*;

pub trait StoreLinkConsistentExt {
    fn check_link_consistency(&self) -> Result<()>;
}

impl StoreLinkConsistentExt for Store {
    fn check_link_consistency(&self) -> Result<()> {
        // Helper data structure to collect incoming and outgoing links for each StoreId
        #[derive(Debug, Default)]
        struct Linking {
            outgoing: Vec<StoreId>,
            incoming: Vec<StoreId>,
        }

        // Helper function to aggregate the Link network
        //
        // This function aggregates a HashMap which maps each StoreId object in the store onto
        // a Linking object, which contains a list of StoreIds which this entry links to and a
        // list of StoreIds which link to the current one.
        //
        // The lambda returns an error if something fails
        let aggregate_link_network = |store: &Store| -> Result<HashMap<StoreId, Linking>> {
            store
                .entries()?
                .into_get_iter()
                .fold(Ok(HashMap::new()), |map, element| {
                    map.and_then(|mut map| {
                        debug!("Checking element = {:?}", element);
                        let entry = element?.ok_or_else(|| err_msg("TODO: Not yet handled"))?;

                        debug!("Checking entry = {:?}", entry.get_location());

                        let internal_links = entry
                            .get_internal_links()?
                            .into_getter(store); // get the FLEs from the Store

                        let mut linking = Linking::default();
                        for internal_link in internal_links {
                            debug!("internal link = {:?}", internal_link);

                            linking.outgoing.push(internal_link?.get_location().clone());
                            linking.incoming.push(entry.get_location().clone());
                        }

                        map.insert(entry.get_location().clone(), linking);
                        Ok(map)
                    })
                })
        };

        // Helper to check whethre all StoreIds in the network actually exists
        //
        // Because why not?
        let all_collected_storeids_exist = |network: &HashMap<StoreId, Linking>| -> Result<()> {
            for (id, _) in network.iter() {
                if is_match!(self.get(id.clone()), Ok(Some(_))) {
                    debug!("Exists in store: {:?}", id);

                    if !self.exists(id.clone())? {
                        warn!("Does exist in store but not on FS: {:?}", id);
                        return Err(err_msg("Link target does not exist"))
                    }
                } else {
                    warn!("Does not exist in store: {:?}", id);
                    return Err(err_msg("Link target does not exist"))
                }
            }

            Ok(())
        };

        // Helper function to create a SLCECD::OneDirectionalLink error object
        let mk_one_directional_link_err = |src: StoreId, target: StoreId| -> Error {
            Error::from(format_err!("Dead link: {} -> {}",
                                    src.local_display_string(),
                                    target.local_display_string()))
        };

        // Helper lambda to check whether the _incoming_ links of each entry actually also
        // appear in the _outgoing_ list of the linked entry
        let incoming_links_exists_as_outgoing_links =
            |src: &StoreId, linking: &Linking, network: &HashMap<StoreId, Linking>| -> Result<()> {
                for link in linking.incoming.iter() {
                    // Check whether the links which are _incoming_ on _src_ are outgoing
                    // in each of the links in the incoming list.
                    let incoming_consistent = network.get(link)
                        .map(|l| l.outgoing.contains(src))
                        .unwrap_or(false);

                    if !incoming_consistent {
                        return Err(mk_one_directional_link_err(src.clone(), link.clone()))
                    }
                }

                Ok(())
            };

        // Helper lambda to check whether the _outgoing links of each entry actually also
        // appear in the _incoming_ list of the linked entry
        let outgoing_links_exist_as_incoming_links =
            |src: &StoreId, linking: &Linking, network: &HashMap<StoreId, Linking>| -> Result<()> {
                for link in linking.outgoing.iter() {
                    // Check whether the links which are _outgoing_ on _src_ are incoming
                    // in each of the links in the outgoing list.
                    let outgoing_consistent = network.get(link)
                        .map(|l| l.incoming.contains(src))
                        .unwrap_or(false);

                    if !outgoing_consistent {
                        return Err(mk_one_directional_link_err(link.clone(), src.clone()))
                    }
                }

                Ok(())
            };

        aggregate_link_network(&self)
            .map_dbg_str("Aggregated")
            .map_dbg(|nw| {
                let mut s = String::new();
                for (k, v) in nw {
                    s.push_str(&format!("{}\n in: {:?}\n out: {:?}", k, v.incoming, v.outgoing));
                }
                s
            })
            .and_then(|nw| {
                all_collected_storeids_exist(&nw)
                    .map(|_| nw)
                    .context(err_msg("Link handling error"))
                    .map_err(Error::from)
            })
            .and_then(|nw| {
                for (id, linking) in nw.iter() {
                    incoming_links_exists_as_outgoing_links(id, linking, &nw)?;
                    outgoing_links_exist_as_incoming_links(id, linking, &nw)?;
                }
                Ok(())
            })
            .map(|_| ())
    }
}


M lib/entry/libimagentrymarkdown/Cargo.toml => lib/entry/libimagentrymarkdown/Cargo.toml +1 -0
@@ 30,6 30,7 @@ sha-1   = "0.8"
libimagstore     = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
libimagerror     = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink/" }
libimagentryurl  = { version = "0.10.0", path = "../../../lib/entry/libimagentryurl/" }
libimagentryref  = { version = "0.10.0", path = "../../../lib/entry/libimagentryref/" }
libimagutil      = { version = "0.10.0", path = "../../../lib/etc/libimagutil/" }


M lib/entry/libimagentrymarkdown/src/lib.rs => lib/entry/libimagentrymarkdown/src/lib.rs +1 -0
@@ 42,6 42,7 @@ extern crate url;
extern crate libimagstore;
extern crate libimagerror;
extern crate libimagentrylink;
extern crate libimagentryurl;
extern crate libimagentryref;
extern crate libimagutil;
#[macro_use] extern crate failure;

M lib/entry/libimagentrymarkdown/src/processor.rs => lib/entry/libimagentrymarkdown/src/processor.rs +20 -20
@@ 24,8 24,8 @@ use failure::ResultExt;
use failure::Error;
use crate::link::extract_links;

use libimagentrylink::external::ExternalLinker;
use libimagentrylink::internal::InternalLinker;
use libimagentryurl::linker::UrlLinker;
use libimagentrylink::linker::InternalLinker;
use libimagentryref::reference::MutRef;
use libimagentryref::reference::RefFassade;
use libimagentryref::hasher::sha1::Sha1Hasher;


@@ 58,7 58,7 @@ use url::Url;
pub struct LinkProcessor {
    process_internal_links: bool,
    create_internal_targets: bool,
    process_external_links: bool,
    process_urls: bool,
    process_refs: bool
}



@@ 86,8 86,8 @@ impl LinkProcessor {
    /// Switch external link processing on/off
    ///
    /// An external link must start with `https://` or `http://`.
    pub fn process_external_links(mut self, b: bool) -> Self {
        self.process_external_links = b;
    pub fn process_urls(mut self, b: bool) -> Self {
        self.process_urls = b;
        self
    }



@@ 154,11 154,11 @@ impl LinkProcessor {
                    let _ = entry.add_internal_link(&mut target)?;
                },
                LinkQualification::ExternalLink(url) => {
                    if !self.process_external_links {
                    if !self.process_urls {
                        continue
                    }

                    entry.add_external_link(store, url)?;
                    entry.add_url(store, url)?;
                },
                LinkQualification::RefLink(url) => {
                    use sha1::{Sha1, Digest};


@@ 273,7 273,7 @@ impl Default for LinkProcessor {
        LinkProcessor {
            process_internal_links: true,
            create_internal_targets: false,
            process_external_links: true,
            process_urls: true,
            process_refs: false
        }
    }


@@ 286,7 286,7 @@ mod tests {
    use std::path::PathBuf;

    use libimagstore::store::Store;
    use libimagentrylink::internal::InternalLinker;
    use libimagentrylink::linker::InternalLinker;

    fn setup_logging() {
        let _ = ::env_logger::try_init();


@@ 330,7 330,7 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(true)
            .create_internal_targets(false)
            .process_external_links(false)
            .process_urls(false)
            .process_refs(false);

        let result = processor.process(&mut base, &store);


@@ 370,7 370,7 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(true)
            .create_internal_targets(false)
            .process_external_links(false)
            .process_urls(false)
            .process_refs(false);

        let result = processor.process(&mut base, &store);


@@ 391,7 391,7 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(true)
            .create_internal_targets(true)
            .process_external_links(false)
            .process_urls(false)
            .process_refs(false);

        let result = processor.process(&mut base, &store);


@@ 418,7 418,7 @@ mod tests {
    }

    #[test]
    fn test_process_one_external_link() {
    fn test_process_one_url() {
        setup_logging();
        let store = get_store();



@@ 431,14 431,14 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(true)
            .create_internal_targets(true)
            .process_external_links(true)
            .process_urls(true)
            .process_refs(false);

        let result = processor.process(&mut base, &store);
        assert!(result.is_ok(), "Should be Ok(()): {:?}", result);

        // The hash of "http://example.com" processed in the `libimagentrylink` way.
        let expected_link = "links/external/9c17e047f58f9220a7008d4f18152fee4d111d14";
        let expected_link = "url/external/9c17e047f58f9220a7008d4f18152fee4d111d14";
        {
            let base_links = base.get_internal_links();
            assert!(base_links.is_ok());


@@ 481,7 481,7 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(false)
            .create_internal_targets(false)
            .process_external_links(false)
            .process_urls(false)
            .process_refs(true);

        let result = processor.process(&mut base, &store);


@@ 514,7 514,7 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(false)
            .create_internal_targets(false)
            .process_external_links(false)
            .process_urls(false)
            .process_refs(true);

        let result = processor.process(&mut base, &store);


@@ 547,7 547,7 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(false)
            .create_internal_targets(false)
            .process_external_links(false)
            .process_urls(false)
            .process_refs(false);

        let result = processor.process(&mut base, &store);


@@ 575,7 575,7 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(true)
            .create_internal_targets(true)
            .process_external_links(false)
            .process_urls(false)
            .process_refs(false);

        let result = processor.process(&mut base, &store);


@@ 605,7 605,7 @@ mod tests {
        let processor = LinkProcessor::default()
            .process_internal_links(false)
            .create_internal_targets(false)
            .process_external_links(false)
            .process_urls(false)
            .process_refs(false);

        let result = processor.process(&mut base, &store);

A lib/entry/libimagentryurl/Cargo.toml => lib/entry/libimagentryurl/Cargo.toml +41 -0
@@ 0,0 1,41 @@
[package]
name = "libimagentryurl"
version = "0.10.0"
authors = ["Matthias Beyer <mail@beyermatthias.de>"]

description = "Library for the imag core distribution"

keywords    = ["imag", "PIM", "personal", "information", "management"]
readme      = "../../../README.md"
license     = "LGPL-2.1"

documentation = "https://imag-pim.org/doc/"
repository    = "https://github.com/matthiasbeyer/imag"
homepage      = "http://imag-pim.org"

[badges]
travis-ci                         = { repository = "matthiasbeyer/imag" }
is-it-maintained-issue-resolution = { repository = "matthiasbeyer/imag" }
is-it-maintained-open-issues      = { repository = "matthiasbeyer/imag" }
maintenance                       = { status     = "actively-developed" }

[dependencies]
itertools = "0.7"
log = "0.4.0"
toml = "0.5"
url = "1.5"
sha-1 = "0.7"
hex = "0.3"
is-match = "0.1"
toml-query = "0.9"
failure        = "0.1"
failure_derive = "0.1"

libimagstore     = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
libimagerror     = { version = "0.10.0", path = "../../../lib/core/libimagerror" }
libimagutil      = { version = "0.10.0", path = "../../../lib/etc/libimagutil" }
libimagentrylink = { version = "0.10.0", path = "../../../lib/entry/libimagentrylink" }

[dev-dependencies]
env_logger = "0.5"


A lib/entry/libimagentryurl/src/iter.rs => lib/entry/libimagentryurl/src/iter.rs +195 -0
@@ 0,0 1,195 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

//! Iterator helpers for external linking stuff
//!
//! Contains also helpers to filter iterators for external/internal links
//!
//!
//! # Warning
//!
//! This module uses `internal::Link` as link type, so we operate on _store ids_ here.
//!
//! Not to confuse with `external::Link` which is a real `FileLockEntry` under the hood.
//!

use libimagentrylink::link::Link;
use libimagentrylink::iter::LinkIter;
use libimagstore::store::Store;
use libimagutil::debug_result::DebugResult;

use failure::Fallible as Result;
use url::Url;

/// Helper for building `OnlyExternalIter` and `NoExternalIter`
///
/// The boolean value defines, how to interpret the `is_external_link_storeid()` return value
/// (here as "pred"):
///
/// ```ignore
///     pred | bool | xor | take?
///     ---- | ---- | --- | ----
///        0 |    0 |   0 |   1
///        0 |    1 |   1 |   0
///        1 |    0 |   1 |   0
///        1 |    1 |   0 |   1
/// ```
///
/// If `bool` says "take if return value is false", we take the element if the `pred` returns
/// false... and so on.
///
/// As we can see, the operator between these two operants is `!(a ^ b)`.
pub struct ExternalFilterIter(LinkIter, bool);

impl Iterator for ExternalFilterIter {
    type Item = Link;

    fn next(&mut self) -> Option<Self::Item> {
        use crate::util::is_external_link_storeid;

        while let Some(elem) = self.0.next() {
            trace!("Check whether is external: {:?}", elem);
            if !(self.1 ^ is_external_link_storeid(&elem)) {
                trace!("Is external id: {:?}", elem);
                return Some(elem);
            }
        }
        None
    }
}

/// Helper trait to be implemented on `LinkIter` to select or deselect all external links
///
/// # See also
///
/// Also see `OnlyExternalIter` and `NoExternalIter` and the helper traits/functions
/// `OnlyInteralLinks`/`only_internal_links()` and `OnlyExternalLinks`/`only_urls()`.
pub trait SelectExternal {
    fn select_urls(self, b: bool) -> ExternalFilterIter;
}

impl SelectExternal for LinkIter {
    fn select_urls(self, b: bool) -> ExternalFilterIter {
        ExternalFilterIter(self, b)
    }
}


pub struct OnlyExternalIter(ExternalFilterIter);

impl OnlyExternalIter {
    pub fn new(li: LinkIter) -> OnlyExternalIter {
        OnlyExternalIter(ExternalFilterIter(li, true))
    }

    pub fn urls<'a>(self, store: &'a Store) -> UrlIter<'a> {
        UrlIter(self, store)
    }
}

impl Iterator for OnlyExternalIter {
    type Item = Link;

    fn next(&mut self) -> Option<Self::Item> {
        self.0.next()
    }
}

pub struct NoExternalIter(ExternalFilterIter);

impl NoExternalIter {
    pub fn new(li: LinkIter) -> NoExternalIter {
        NoExternalIter(ExternalFilterIter(li, false))
    }
}

impl Iterator for NoExternalIter {
    type Item = Link;

    fn next(&mut self) -> Option<Self::Item> {
        self.0.next()
    }
}

pub trait OnlyExternalLinks : Sized {
    fn only_urls(self) -> OnlyExternalIter ;

    fn no_internal_links(self) -> OnlyExternalIter {
        self.only_urls()
    }
}

impl OnlyExternalLinks for LinkIter {
    fn only_urls(self) -> OnlyExternalIter {
        OnlyExternalIter::new(self)
    }
}

pub trait OnlyInternalLinks : Sized {
    fn only_internal_links(self) -> NoExternalIter;

    fn no_urls(self) -> NoExternalIter {
        self.only_internal_links()
    }
}

impl OnlyInternalLinks for LinkIter {
    fn only_internal_links(self) -> NoExternalIter {
        NoExternalIter::new(self)
    }
}

pub struct UrlIter<'a>(OnlyExternalIter, &'a Store);

impl<'a> Iterator for UrlIter<'a> {
    type Item = Result<Url>;

    fn next(&mut self) -> Option<Self::Item> {
        use crate::link::Link;

        loop {
            let next = self.0
                .next()
                .map(|id| {
                    debug!("Retrieving entry for id: '{:?}'", id);
                    self.1
                        .retrieve(id.clone())
                        .map_dbg_err(|_| format!("Retrieving entry for id: '{:?}' failed", id))
                        .map_err(From::from)
                        .and_then(|f| {
                            debug!("Store::retrieve({:?}) succeeded", id);
                            debug!("getting external link from file now");
                            f.get_link_uri_from_filelockentry()
                                .map_dbg_str("Error happened while getting link URI from FLE")
                                .map_dbg_err(|e| format!("URL -> Err = {:?}", e))
                        })
                });

            match next {
                Some(Ok(Some(link))) => return Some(Ok(link)),
                Some(Ok(None))       => continue,
                Some(Err(e))         => return Some(Err(e)),
                None                 => return None
            }
        }
    }

}



A lib/entry/libimagentryurl/src/lib.rs => lib/entry/libimagentryurl/src/lib.rs +74 -0
@@ 0,0 1,74 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

#![forbid(unsafe_code)]

#![deny(
    dead_code,
    non_camel_case_types,
    non_snake_case,
    path_statements,
    trivial_numeric_casts,
    unstable_features,
    unused_allocation,
    unused_import_braces,
    unused_imports,
    unused_must_use,
    unused_mut,
    unused_qualifications,
    while_true,
)]

//! External linking is a complex implementation to be able to serve a clean and easy-to-use
//! interface.
//!
//! Internally, there are no such things as "external links" (plural). Each Entry in the store can
//! only have _one_ external link.
//!
//! This library does the following therefor: It allows you to have several external links with one
//! entry, which are internally one file in the store for each link, linked with "internal
//! linking".
//!
//! This helps us greatly with deduplication of URLs.
//!

extern crate itertools;
#[macro_use] extern crate log;
extern crate toml;
extern crate toml_query;
extern crate url;
extern crate sha1;
extern crate hex;
#[macro_use] extern crate failure;

#[cfg(test)]
extern crate env_logger;

#[macro_use] extern crate libimagstore;
extern crate libimagerror;
extern crate libimagutil;
extern crate libimagentrylink;

module_entry_path_mod!("url");

pub mod iter;
pub mod link;
pub mod linker;
pub mod util;


A lib/entry/libimagentryurl/src/link.rs => lib/entry/libimagentryurl/src/link.rs +76 -0
@@ 0,0 1,76 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use failure::Error;
use failure::ResultExt;
use failure::Fallible as Result;
use failure::err_msg;
use url::Url;

use libimagstore::store::Entry;
use libimagerror::errors::ErrorMsg as EM;

use toml_query::read::TomlValueReadTypeExt;

pub trait Link {

    fn get_link_uri_from_filelockentry(&self) -> Result<Option<Url>>;

    fn get_url(&self) -> Result<Option<Url>>;

}

impl Link for Entry {

    fn get_link_uri_from_filelockentry(&self) -> Result<Option<Url>> {
        self.get_header()
            .read_string("links.external.content.url")
            .context(format_err!("Error reading header 'links.external.content.url' from '{}'", self.get_location()))
            .context(EM::EntryHeaderReadError)
            .map_err(Error::from)
            .and_then(|opt| match opt {
                None        => Ok(None),
                Some(ref s) => {
                    debug!("Found url, parsing: {:?}", s);
                    Url::parse(&s[..])
                        .map_err(Error::from)
                        .context(format_err!("Failed to parse URL: '{}'", s))
                        .context(err_msg("Invalid URI"))
                        .map_err(Error::from)
                        .map(Some)
                },
            })
            .context("Failed to get link URI from entry")
            .map_err(Error::from)
    }

    fn get_url(&self) -> Result<Option<Url>> {
        match self.get_header().read_string("links.external.url")? {
            None        => Ok(None),
            Some(ref s) => Url::parse(&s[..])
                .context(format_err!("Failed to parse URL: '{}'", s))
                .map(Some)
                .map_err(Error::from)
                .context(EM::EntryHeaderReadError)
                .map_err(Error::from),
        }
    }

}


A lib/entry/libimagentryurl/src/linker.rs => lib/entry/libimagentryurl/src/linker.rs +224 -0
@@ 0,0 1,224 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use std::ops::DerefMut;

use libimagstore::storeid::StoreId;
use libimagstore::store::Store;
use libimagstore::store::Entry;
use libimagutil::debug_result::DebugResult;
use libimagentrylink::linker::InternalLinker;

use failure::Fallible as Result;
use toml::Value;
use toml::map::Map;
use toml_query::read::TomlValueReadExt;
use toml_query::insert::TomlValueInsertExt;
use url::Url;
use sha1::{Sha1, Digest};
use hex;

use crate::iter::UrlIter;

pub trait UrlLinker : InternalLinker {

    /// Get the external links from the implementor object
    fn get_urls<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>>;

    /// Set the external links for the implementor object
    fn set_urls(&mut self, store: &Store, links: Vec<Url>) -> Result<Vec<StoreId>>;

    /// Add an external link to the implementor object
    fn add_url(&mut self, store: &Store, link: Url) -> Result<Vec<StoreId>>;

    /// Remove an external link from the implementor object
    fn remove_url(&mut self, store: &Store, link: Url) -> Result<Vec<StoreId>>;

}

/// Implement `ExternalLinker` for `Entry`, hiding the fact that there is no such thing as an external
/// link in an entry, but internal links to other entries which serve as external links, as one
/// entry in the store can only have one external link.
impl UrlLinker for Entry {

    /// Get the external links from the implementor object
    fn get_urls<'a>(&self, store: &'a Store) -> Result<UrlIter<'a>> {
        use crate::iter::OnlyExternalLinks;

        // Iterate through all internal links and filter for FileLockEntries which live in
        // /link/external/<SHA> -> load these files and get the external link from their headers,
        // put them into the return vector.
        self.get_internal_links()
            .map(|iter| {
                debug!("Getting external links");
                iter.only_urls().urls(store)
            })
    }

    /// Set the external links for the implementor object
    ///
    /// # Return Value
    ///
    /// Returns the StoreIds which were newly created for the new external links, if there are more
    /// external links than before.
    /// If there are less external links than before, an empty vec![] is returned.
    ///
    fn set_urls(&mut self, store: &Store, links: Vec<Url>) -> Result<Vec<StoreId>> {
        // Take all the links, generate a SHA sum out of each one, filter out the already existing
        // store entries and store the other URIs in the header of one FileLockEntry each, in
        // the path /link/external/<SHA of the URL>

        debug!("Iterating {} links = {:?}", links.len(), links);
        links.into_iter().map(|link| {
            let hash = hex::encode(Sha1::digest(&link.as_str().as_bytes()));
            let file_id = crate::module_path::new_id(format!("external/{}", hash))
                .map_dbg_err(|_| {
                    format!("Failed to build StoreId for this hash '{:?}'", hash)
                })?;

            debug!("Link    = '{:?}'", link);
            debug!("Hash    = '{:?}'", hash);
            debug!("StoreId = '{:?}'", file_id);

            let link_already_exists = store.get(file_id.clone())?.is_some();

            // retrieve the file from the store, which implicitely creates the entry if it does not
            // exist
            let mut file = store
                .retrieve(file_id.clone())
                .map_dbg_err(|_| {
                    format!("Failed to create or retrieve an file for this link '{:?}'", link)
                })?;

            debug!("Generating header content!");
            {
                let hdr = file.deref_mut().get_header_mut();

                let mut table = match hdr.read("links.external.content")? {
                    Some(&Value::Table(ref table)) => table.clone(),
                    Some(_) => {
                        warn!("There is a value at 'links.external.content' which is not a table.");
                        warn!("Going to override this value");
                        Map::new()
                    },
                    None => Map::new(),
                };

                let v = Value::String(link.into_string());

                debug!("setting URL = '{:?}", v);
                table.insert(String::from("url"), v);

                let _ = hdr.insert("links.external.content", Value::Table(table))?;
                debug!("Setting URL worked");
            }

            // then add an internal link to the new file or return an error if this fails
            let _ = self.add_internal_link(file.deref_mut())?;
            debug!("Added internal link");

            Ok((link_already_exists, file_id))
        })
        .filter_map(|res| match res {
            Ok((exists, entry)) => if exists { Some(Ok(entry)) } else { None },
            Err(e) => Some(Err(e))
        })
        .collect()
    }

    /// Add an external link to the implementor object
    ///
    /// # Return Value
    ///
    /// (See ExternalLinker::set_urls())
    ///
    /// Returns the StoreIds which were newly created for the new external links, if there are more
    /// external links than before.
    /// If there are less external links than before, an empty vec![] is returned.
    ///
    fn add_url(&mut self, store: &Store, link: Url) -> Result<Vec<StoreId>> {
        // get external links, add this one, save them
        debug!("Getting links");
        self.get_urls(store)
            .and_then(|links| {
                let mut links = links.collect::<Result<Vec<_>>>()?;

                debug!("Adding link = '{:?}' to links = {:?}", link, links);
                links.push(link);

                debug!("Setting {} links = {:?}", links.len(), links);
                self.set_urls(store, links)
            })
    }

    /// Remove an external link from the implementor object
    ///
    /// # Return Value
    ///
    /// (See ExternalLinker::set_urls())
    ///
    /// Returns the StoreIds which were newly created for the new external links, if there are more
    /// external links than before.
    /// If there are less external links than before, an empty vec![] is returned.
    ///
    fn remove_url(&mut self, store: &Store, link: Url) -> Result<Vec<StoreId>> {
        // get external links, remove this one, save them
        self.get_urls(store)
            .and_then(|links| {
                debug!("Removing link = '{:?}'", link);
                let links = links
                    .filter_map(Result::ok)
                    .filter(|l| l.as_str() != link.as_str())
                    .collect::<Vec<_>>();
                self.set_urls(store, links)
            })
    }

}

#[cfg(test)]
mod tests {
    use super::*;
    use std::path::PathBuf;

    use libimagstore::store::Store;

    fn setup_logging() {
        let _ = env_logger::try_init();
    }

    pub fn get_store() -> Store {
        Store::new_inmemory(PathBuf::from("/"), &None).unwrap()
    }


    #[test]
    fn test_simple() {
        setup_logging();
        let store = get_store();
        let mut e = store.retrieve(PathBuf::from("base-test_simple")).unwrap();
        let url   = Url::parse("http://google.de").unwrap();

        assert!(e.add_url(&store, url.clone()).is_ok());

        assert_eq!(1, e.get_urls(&store).unwrap().count());
        assert_eq!(url, e.get_urls(&store).unwrap().next().unwrap().unwrap());
    }

}

A lib/entry/libimagentryurl/src/util.rs => lib/entry/libimagentryurl/src/util.rs +28 -0
@@ 0,0 1,28 @@
//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015-2019 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use std::fmt::Debug;

use libimagstore::storeid::StoreId;

/// Check whether the StoreId starts with `/link/external/`
pub fn is_external_link_storeid<A: AsRef<StoreId> + Debug>(id: A) -> bool {
    debug!("Checking whether this is a 'url/external/': '{:?}'", id);
    id.as_ref().is_in_collection(&["url", "external"])
}

M scripts/release.sh => scripts/release.sh +1 -0
@@ 16,6 16,7 @@ CRATES=(
    ./lib/etc/libimaginteraction
    ./lib/core/libimagrt
    ./lib/entry/libimagentrylink
    ./lib/entry/libimagentryurl
    ./lib/entry/libimagentrytag
    ./lib/entry/libimagentryfilter
    ./lib/entry/libimagentrygps