~matthiasbeyer/imag

23499d18eb7b7d5e6565c42270f0123101328434 — Matthias Beyer 1 year, 10 months ago 17fc7e7 + 51fd196
Merge branch 'imag-mail/scan' into master

Signed-off-by: Matthias Beyer <mail@beyermatthias.de>
M bin/domain/imag-mail/Cargo.toml => bin/domain/imag-mail/Cargo.toml +2 -0
@@ 24,6 24,8 @@ log = "0.4.6"
failure = "0.1.5"
resiter = "0.4"
handlebars = "2"
walkdir = "2"
rayon = "1"

libimagrt       = { version = "0.10.0", path = "../../../lib/core/libimagrt" }
libimagstore    = { version = "0.10.0", path = "../../../lib/core/libimagstore" }

M bin/domain/imag-mail/src/lib.rs => bin/domain/imag-mail/src/lib.rs +65 -0
@@ 40,6 40,8 @@ extern crate clap;
extern crate toml_query;
extern crate resiter;
extern crate handlebars;
extern crate walkdir;
extern crate rayon;

extern crate libimagrt;
extern crate libimagmail;


@@ 58,10 60,14 @@ use failure::Error;
use toml_query::read::TomlValueReadTypeExt;
use clap::App;
use resiter::AndThen;
use resiter::Filter;
use resiter::IterInnerOkOrElse;
use resiter::Map;
use rayon::prelude::*;

use libimagmail::store::MailStore;
use libimagentryref::util::get_ref_config;
use libimagentryref::reference::Config as RefConfig;
use libimagrt::runtime::Runtime;
use libimagrt::application::ImagApplication;
use libimagutil::info_result::*;


@@ 79,6 85,7 @@ pub enum ImagMail {}
impl ImagApplication for ImagMail {
    fn run(rt: Runtime) -> Result<()> {
        match rt.cli().subcommand_name().ok_or_else(|| err_msg("No subcommand called"))? {
            "scan"        => scan(&rt),
            "import-mail" => import_mail(&rt),
            "list"        => list(&rt),
            "mail-store"  => mail_store(&rt),


@@ 110,6 117,64 @@ impl ImagApplication for ImagMail {
    }
}

fn scan(rt: &Runtime) -> Result<()> {
    let collection_name = get_ref_collection_name(rt)?;
    let refconfig       = get_ref_config(rt, "imag-mail")?;
    let scmd            = rt.cli().subcommand_matches("scan").unwrap();

    /// Helper function to get an Iterator for all files from one PathBuf
    fn walk_pathes(path: PathBuf) -> impl Iterator<Item = Result<PathBuf>> {
        walkdir::WalkDir::new(path)
            .follow_links(false)
            .into_iter()
            .filter_ok(|entry| entry.file_type().is_file())
            .map_err(Error::from)
            .map_ok(|entry| entry.into_path())
            .inspect(|e| trace!("Processing = {:?}", e))
    }

    /// Helper function to process an iterator over Result<PathBuf> and create store entries for
    /// each path in the iterator
    fn process_iter(i: &mut dyn Iterator<Item = Result<PathBuf>>, rt: &Runtime, collection_name: &str, refconfig: &RefConfig) -> Result<()> {
        let scmd = rt.cli().subcommand_matches("scan").unwrap();
        i.and_then_ok(|path| {
                if scmd.is_present("ignore-existing-ids") {
                    rt.store().retrieve_mail_from_path(path, collection_name, refconfig, true)
                } else {
                    rt.store().create_mail_from_path(path, collection_name, refconfig)
                }
            })
            .and_then_ok(|e| rt.report_touched(e.get_location()).map_err(Error::from))
            .collect::<Result<Vec<_>>>()
            .map(|_| ())
    }

    let pathes = scmd.values_of("path")
        .unwrap() // enforced by clap
        .map(PathBuf::from)
        .collect::<Vec<_>>();

    if scmd.is_present("scan-parallel") {
        debug!("Fetching pathes in parallel");
        let mut i = pathes.into_par_iter()
            .map(|path| walk_pathes(path).collect::<Result<_>>())
            .collect::<Result<Vec<PathBuf>>>()?
            .into_iter()
            .map(Ok);

        debug!("Processing pathes");
        process_iter(&mut i, rt, &collection_name, &refconfig)
    } else {
        debug!("Fetching pathes not in parallel");
        let mut i = pathes
            .into_iter()
            .map(walk_pathes)
            .flatten();

        debug!("Processing pathes");
        process_iter(&mut i, rt, &collection_name, &refconfig)
    }
}

fn import_mail(rt: &Runtime) -> Result<()> {
    let collection_name = get_ref_collection_name(rt)?;

M bin/domain/imag-mail/src/ui.rs => bin/domain/imag-mail/src/ui.rs +27 -0
@@ 47,6 47,33 @@ pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> {
                         .value_name("PATH"))
                    )

        .subcommand(SubCommand::with_name("scan")
                    .about("Scan a directory for mails")
                    .version("0.1")
                    .arg(Arg::with_name("ignore-existing-ids")
                         .long("ignore-existing")
                         .short("I")
                         .takes_value(false)
                         .required(false)
                         .help("Ignore errors that might occur when store entries exist already"))

                    .arg(Arg::with_name("scan-parallel")
                         .long("parallel")
                         .takes_value(false)
                         .required(false)
                         .multiple(false)
                         .help("Scan with multiple threads. Might be faster, but might slow down other tasks"))

                    .arg(Arg::with_name("path")
                         .index(1)
                         .takes_value(true)
                         .multiple(true)
                         .required(true)
                         .validator(libimagutil::cli_validators::is_directory)
                         .value_name("DIR")
                         .help("Path to the directory containing mails"))
                    )

        .subcommand(SubCommand::with_name("list")
                    .about("List all stored references to mails")
                    .version("0.1")