summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatthias Beyer <mail@beyermatthias.de>2018-02-19 14:18:17 +0100
committerGitHub <noreply@github.com>2018-02-19 14:18:17 +0100
commit2c0c8347e973dcd0c4113700eba37b67959561ed (patch)
treed26d9d8449e18906e1c519bfe968e48ff7192f84
parent7653a040da152385f29b1f709e45ca718dc0da6d (diff)
parent2dc23c8101f4e371d12a5207e3de00e5ac4d19a4 (diff)
downloadimag-2c0c8347e973dcd0c4113700eba37b67959561ed.zip
imag-2c0c8347e973dcd0c4113700eba37b67959561ed.tar.gz
Merge pull request #1285 from matthiasbeyer/libimagentryref/refactor
libimagentryref: Rewrite
-rw-r--r--bin/core/imag-ref/Cargo.toml1
-rw-r--r--bin/core/imag-ref/src/main.rs107
-rw-r--r--bin/core/imag-ref/src/ui.rs60
-rw-r--r--bin/domain/imag-contact/Cargo.toml2
-rw-r--r--bin/domain/imag-contact/src/create.rs8
-rw-r--r--bin/domain/imag-contact/src/main.rs5
-rw-r--r--doc/src/05100-lib-entryref.md68
-rw-r--r--lib/domain/libimagcontact/Cargo.toml8
-rw-r--r--lib/domain/libimagcontact/src/contact.rs2
-rw-r--r--lib/domain/libimagcontact/src/error.rs1
-rw-r--r--lib/domain/libimagcontact/src/lib.rs1
-rw-r--r--lib/domain/libimagcontact/src/store.rs45
-rw-r--r--lib/domain/libimagmail/src/error.rs4
-rw-r--r--lib/domain/libimagmail/src/hasher.rs81
-rw-r--r--lib/domain/libimagmail/src/lib.rs1
-rw-r--r--lib/domain/libimagmail/src/mail.rs75
-rw-r--r--lib/entry/libimagentrymarkdown/Cargo.toml7
-rw-r--r--lib/entry/libimagentrymarkdown/src/processor.rs30
-rw-r--r--lib/entry/libimagentryref/Cargo.toml17
-rw-r--r--lib/entry/libimagentryref/src/error.rs74
-rw-r--r--lib/entry/libimagentryref/src/flags.rs100
-rw-r--r--lib/entry/libimagentryref/src/generators/base.rs (renamed from lib/entry/libimagentryref/src/hashers/mod.rs)30
-rw-r--r--lib/entry/libimagentryref/src/generators/mod.rs274
-rw-r--r--lib/entry/libimagentryref/src/hasher.rs64
-rw-r--r--lib/entry/libimagentryref/src/hashers/nbytes.rs66
-rw-r--r--lib/entry/libimagentryref/src/lib.rs21
-rw-r--r--lib/entry/libimagentryref/src/lister.rs161
-rw-r--r--lib/entry/libimagentryref/src/reference.rs318
-rw-r--r--lib/entry/libimagentryref/src/refstore.rs337
-rw-r--r--lib/entry/libimagentryref/src/util.rs52
30 files changed, 704 insertions, 1316 deletions
diff --git a/bin/core/imag-ref/Cargo.toml b/bin/core/imag-ref/Cargo.toml
index b0fb814..9a0b2da 100644
--- a/bin/core/imag-ref/Cargo.toml
+++ b/bin/core/imag-ref/Cargo.toml
@@ -24,6 +24,7 @@ maintenance = { status = "actively-developed" }
[dependencies]
log = "0.4.0"
+libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
libimagrt = { version = "0.7.0", path = "../../../lib/core/libimagrt" }
libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" }
libimagentryref = { version = "0.7.0", path = "../../../lib/entry/libimagentryref" }
diff --git a/bin/core/imag-ref/src/main.rs b/bin/core/imag-ref/src/main.rs
index 3d1339e..ce24b2f 100644
--- a/bin/core/imag-ref/src/main.rs
+++ b/bin/core/imag-ref/src/main.rs
@@ -35,6 +35,7 @@
#[macro_use] extern crate log;
extern crate clap;
+extern crate libimagstore;
#[macro_use] extern crate libimagrt;
extern crate libimagentryref;
extern crate libimagerror;
@@ -48,12 +49,11 @@ use ui::build_ui;
use std::path::PathBuf;
use std::process::exit;
-use libimagentryref::refstore::RefStore;
-use libimagentryref::flags::RefFlags;
-use libimagerror::trace::trace_error;
use libimagerror::trace::MapErrTrace;
use libimagrt::setup::generate_runtime_setup;
use libimagrt::runtime::Runtime;
+use libimagstore::storeid::IntoStoreId;
+use libimagentryref::reference::Ref;
fn main() {
let version = make_imag_version!();
@@ -66,9 +66,8 @@ fn main() {
.map(|name| {
debug!("Call: {}", name);
match name {
- "add" => add(&rt),
+ "deref" => deref(&rt),
"remove" => remove(&rt),
- "list" => list(&rt),
_ => {
debug!("Unknown command"); // More error handling
},
@@ -76,83 +75,57 @@ fn main() {
});
}
-fn add(rt: &Runtime) {
- let cmd = rt.cli().subcommand_matches("add").unwrap();
- let path = cmd.value_of("path").map(PathBuf::from).unwrap(); // saved by clap
-
- let flags = RefFlags::default()
- .with_content_hashing(cmd.is_present("track-content"))
- .with_permission_tracking(cmd.is_present("track-permissions"));
-
- match RefStore::create(rt.store(), path, flags) {
- Ok(r) => {
- debug!("Reference created: {:?}", r);
- },
- Err(e) => {
- trace_error(&e);
- warn!("Failed to create reference");
+fn deref(rt: &Runtime) {
+ let cmd = rt.cli().subcommand_matches("deref").unwrap();
+ let id = cmd.value_of("ID")
+ .map(String::from)
+ .map(PathBuf::from)
+ .unwrap() // saved by clap
+ .into_storeid()
+ .map_err_trace_exit_unwrap(1);
+
+ match rt.store().get(id.clone()).map_err_trace_exit_unwrap(1) {
+ Some(entry) => entry
+ .get_path()
+ .map_err_trace_exit_unwrap(1)
+ .to_str()
+ .ok_or_else(|| {
+ error!("Could not transform path into string!");
+ exit(1)
+ })
+ .map(|s| info!("{}", s))
+ .ok(), // safe here because we exited already in the error case
+ None => {
+ error!("No entry for id '{}' found", id);
+ exit(1)
},
- }
+ };
}
fn remove(rt: &Runtime) {
use libimaginteraction::ask::ask_bool;
let cmd = rt.cli().subcommand_matches("remove").unwrap();
- let hash = cmd.value_of("hash").map(String::from).unwrap(); // saved by clap
let yes = cmd.is_present("yes");
-
- match rt.store().find_storeid_by_partial_hash(&hash).map_err_trace_exit_unwrap(1) {
- Some(sid) => {
- if yes || ask_bool(&format!("Delete Ref with hash '{}'", hash)[..], None) {
- debug!("Found for hash '{}' -> {:?}", hash, sid);
- rt.store().delete(sid).map_err_trace_exit_unwrap(1)
+ let id = cmd.value_of("ID")
+ .map(String::from)
+ .map(PathBuf::from)
+ .unwrap() // saved by clap
+ .into_storeid()
+ .map_err_trace_exit_unwrap(1);
+
+ match rt.store().get(id.clone()).map_err_trace_exit_unwrap(1) {
+ Some(mut entry) => {
+ if yes || ask_bool(&format!("Delete ref from entry '{}'", id), None) {
+ let _ = entry.remove_ref().map_err_trace_exit_unwrap(1);
} else {
info!("Aborted");
}
},
None => {
- error!("Not id for hash '{}' found", hash);
+ error!("No entry for id '{}' found", id);
exit(1)
},
};
-
-}
-
-fn list(rt: &Runtime) {
- use std::process::exit;
-
- use libimagentrylist::lister::Lister;
- use libimagentryref::lister::RefLister;
-
- let cmd = rt.cli().subcommand_matches("list").unwrap();
- let do_check_dead = cmd.is_present("check-dead");
- let do_check_changed = cmd.is_present("check-changed");
- let do_check_changed_content = cmd.is_present("check-changed-content");
- let do_check_changed_permiss = cmd.is_present("check-changed-permissions");
-
- let iter = match rt.store().retrieve_for_module("ref") {
- Ok(iter) => iter.filter_map(|id| {
- match rt.store().get(id) {
- Ok(r) => Some(r),
- Err(e) => {
- trace_error(&e);
- None
- },
- }
- }),
- Err(e) => {
- trace_error(&e);
- exit(1);
- }
- };
-
- RefLister::new()
- .check_dead(do_check_dead)
- .check_changed(do_check_changed)
- .check_changed_content(do_check_changed_content)
- .check_changed_permiss(do_check_changed_permiss)
- .list(iter.filter_map(Into::into))
- .ok();
}
diff --git a/bin/core/imag-ref/src/ui.rs b/bin/core/imag-ref/src/ui.rs
index 295b6a2..14d7991 100644
--- a/bin/core/imag-ref/src/ui.rs
+++ b/bin/core/imag-ref/src/ui.rs
@@ -19,73 +19,33 @@
use clap::{Arg, App, SubCommand};
-use libimagutil::cli_validators::is_existing_path;
-
pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> {
app
- .subcommand(SubCommand::with_name("add")
- .about("Add a reference to a file outside of the store")
+ .subcommand(SubCommand::with_name("deref")
+ .about("'Dereference' a ref. This prints the Path of the referenced file")
.version("0.1")
- .arg(Arg::with_name("path")
+ .arg(Arg::with_name("ID")
.index(1)
.takes_value(true)
.required(true)
- .help("The path of the file")
- .validator(is_existing_path)
- .value_name("PATH"))
- .arg(Arg::with_name("track-content")
- .long("content-hash")
- .short("C")
- .takes_value(false)
- .required(false)
- .help("Hash the content for the reference"))
- .arg(Arg::with_name("track-permissions")
- .long("permission-tracking")
- .short("P")
- .takes_value(false)
- .required(false)
- .help("Rememeber the permissions of the referenced file"))
+ .help("The id of the store entry to dereference")
+ .value_name("ID"))
)
.subcommand(SubCommand::with_name("remove")
- .about("Remove a reference")
+ .about("Remove a reference from an entry")
.version("0.1")
- .arg(Arg::with_name("hash")
+ .arg(Arg::with_name("ID")
.index(1)
.takes_value(true)
.required(true)
- .help("Remove the reference with this hash")
- .value_name("HASH"))
+ .multiple(true)
+ .help("Remove the reference from this store entry")
+ .value_name("ENTRIES"))
.arg(Arg::with_name("yes")
.long("yes")
.short("y")
.help("Don't ask whether this really should be done"))
)
-
- .subcommand(SubCommand::with_name("list")
- .about("List references in the store")
- .version("0.1")
-
- .arg(Arg::with_name("check-dead")
- .long("check-dead")
- .short("d")
- .help("Check each reference whether it is dead"))
-
- .arg(Arg::with_name("check-changed")
- .long("check-changed")
- .short("c")
- .help("Check whether a reference had changed (content or permissions)"))
-
- .arg(Arg::with_name("check-changed-content")
- .long("check-changed-content")
- .short("C")
- .help("Check whether the content of the referenced file changed"))
-
- .arg(Arg::with_name("check-changed-permissions")
- .long("check-changed-perms")
- .short("P")
- .help("Check whether the permissions of the referenced file changed"))
-
- )
}
diff --git a/bin/domain/imag-contact/Cargo.toml b/bin/domain/imag-contact/Cargo.toml
index 71a4b1c..b44fc22 100644
--- a/bin/domain/imag-contact/Cargo.toml
+++ b/bin/domain/imag-contact/Cargo.toml
@@ -28,7 +28,7 @@ toml-query = "0.6"
handlebars = "0.29"
vobject = "0.4"
walkdir = "1"
-uuid = { version = "0.5", features = ["v4"] }
+uuid = { version = "0.6", features = ["v4"] }
libimagrt = { version = "0.7.0", path = "../../../lib/core/libimagrt" }
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
diff --git a/bin/domain/imag-contact/src/create.rs b/bin/domain/imag-contact/src/create.rs
index 51278c2..9b44b70 100644
--- a/bin/domain/imag-contact/src/create.rs
+++ b/bin/domain/imag-contact/src/create.rs
@@ -31,13 +31,13 @@ use toml::Value;
use uuid::Uuid;
use libimagcontact::error::ContactError as CE;
+use libimagcontact::store::UniqueContactPathGenerator;
use libimagrt::runtime::Runtime;
use libimagerror::str::ErrFromStr;
use libimagerror::trace::MapErrTrace;
use libimagerror::trace::trace_error;
use libimagutil::warn_result::WarnResult;
use libimagentryref::refstore::RefStore;
-use libimagentryref::flags::RefFlags;
const TEMPLATE : &'static str = include_str!("../static/new-contact-template.toml");
@@ -144,11 +144,7 @@ pub fn create(rt: &Runtime) {
if let Some(location) = location {
if !scmd.is_present("dont-track") {
- let flags = RefFlags::default()
- .with_content_hashing(true)
- .with_permission_tracking(false);
-
- RefStore::create(rt.store(), location, flags)
+ RefStore::create_ref::<UniqueContactPathGenerator, _>(rt.store(), location)
.map_err_trace_exit_unwrap(1);
info!("Created entry in store");
diff --git a/bin/domain/imag-contact/src/main.rs b/bin/domain/imag-contact/src/main.rs
index c278590..35ad1e0 100644
--- a/bin/domain/imag-contact/src/main.rs
+++ b/bin/domain/imag-contact/src/main.rs
@@ -67,6 +67,7 @@ use libimagerror::trace::MapErrTrace;
use libimagerror::io::ToExitCode;
use libimagerror::exit::ExitUnwrap;
use libimagcontact::store::ContactStore;
+use libimagcontact::store::UniqueContactPathGenerator;
use libimagcontact::error::ContactError as CE;
use libimagcontact::contact::Contact;
use libimagstore::iter::get::StoreIdGetIteratorExtension;
@@ -130,7 +131,7 @@ fn list(rt: &Runtime) {
})
.enumerate()
.map(|(i, (fle, vcard))| {
- let hash = fle.get_path_hash().map_err_trace_exit_unwrap(1);
+ let hash = String::from(fle.get_hash().map_err_trace_exit_unwrap(1));
let vcard = vcard.unwrap_or_else(|e| {
error!("Element is not a VCARD object: {:?}", e);
exit(1)
@@ -190,7 +191,7 @@ fn show(rt: &Runtime) {
let hash = scmd.value_of("hash").map(String::from).unwrap(); // safed by clap
let contact_data = rt.store()
- .get_by_hash(hash.clone())
+ .get_ref::<UniqueContactPathGenerator, _>(hash.clone())
.map_err_trace_exit_unwrap(1)
.ok_or(CE::from(format!("No entry for hash {}", hash)))
.map_err_trace_exit_unwrap(1)
diff --git a/doc/src/05100-lib-entryref.md b/doc/src/05100-lib-entryref.md
index 459735f..37dd876 100644
--- a/doc/src/05100-lib-entryref.md
+++ b/doc/src/05100-lib-entryref.md
@@ -3,27 +3,24 @@
This library crate contains functionality to generate _references_ within the
imag store.
-It can be used to create references to other files on the filesystem (reachable
-via a filesystem path). It differs from `libimagentrylink`/external linking as
+A reference is a "pointer" to a file or directory on the filesystem and outside
+the store.
+It differs from `libimagentrylink`/external linking as
it is designed exclusively for filesystem references, not for URLs.
-A reference can have several properties, for example can a reference track the
-content of a filesystem path by hashing the content with a hashsum (SHA1) and
-one can check whether a file was changed by that.
-As files can get big (think of `debian.iso`) _partial hashing_ is supported
-(think of "hash the first 2048 bytes of a file).
+A reference is created with a unique identifier, like a hash. The implementation
+how this hash is calculated can be defined by the user of `libimagentryref`.
-The library contains functionality to re-find a moved file automatically by
-checking the content hash which was stored before.
+So this library helps to resemble something like a _symlink_.
-Permission changes can be tracked as well.
+### Usage
-So this library helps to resemble something like a _symlink_.
+Users have to implement the `UniqueRefPathGenerator` trait which should
+implement a hashing functionality for pathes.
### Limits
-Please understand that this is _not_ intended to be a version control system or
-something like that.
+This is _not_ intended to be a version control system or something like that.
We also can not use _real symlinks_ as we need imag-store-objects to be able to
link stuff.
@@ -31,39 +28,22 @@ link stuff.
This library offers functionality to refer to content outside of the store.
It can be used to refer to _nearly static stuff_ pretty easily - think of a
-Maildir - you add new mails by fetching them, but you mostly do not remove mails
-and if you do you end up with a "null pointer" in the store, which can then be
-handled properly.
-
-As this library supports custom hashes (you don't have to hash the full file,
-you can also parse the file and hash only _some_ content) this is pretty
-flexible.
-For example if you want to implement a imag module which tracks a certain kind
-of files which constantly change... but the first 5 lines do never change
-after the file is created - you can write a custom hasher that only uses the
-first 5 lines for the hash.
-
-### Internals
-
-Internally, in the store, the file gets created under
-`/ref/<hash of the path to the file to refer to>`.
-If the content of the file is hashed, we can still re-find the file via the
-content hash (which is stored in the header of the store entry).
-
-The reference object can, after the path was re-found, be updated.
+Maildir - you add new mails by fetching them, but you mostly do not remove
+mails.
+If mails get moved, they can be re-found via their hash, because Maildir objects
+hardly change. Or because the hash implementation which is used to refer to them
+hashes only the `Message-Id` and that does not change.
### Long-term TODO
-Things which have to be done here or are not yet properly tested:
-
-- [ ] Testing of different Hashers
-- [ ] Testing of re-finding of objects, including:
- - [ ] Can a moved file automatically be found by content hash?
- - [ ] Does a store-reference get updated automatically if it was moved,
- including links (as in `libimaglink`)?
- - [ ] If the content of a file changes, does the content hash get updated
- automatically?
+Not implemented yet:
-("automatically" is a strechable term here, as these things have to be triggered
-by the user anyways)
+- [ ] Re-finding of files via their hash.
+ This must be implemented with several things in mind
+ * The user of the library should be able to provide a way how the
+ filesystem is searched. Basically a Functor which yields pathes to
+ check based on the original path of the missing file.
+ This enables implementations which do only search a certain subset
+ of pathes, or does depth-first-search rather than
+ breadth-first-search.
diff --git a/lib/domain/libimagcontact/Cargo.toml b/lib/domain/libimagcontact/Cargo.toml
index 23608c9..cc1d3cb 100644
--- a/lib/domain/libimagcontact/Cargo.toml
+++ b/lib/domain/libimagcontact/Cargo.toml
@@ -25,9 +25,15 @@ log = "0.3"
toml = "0.4"
toml-query = "0.4"
vobject = "0.4"
+uuid = { version = "0.6", features = ["v4"] }
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" }
-libimagentryref = { version = "0.7.0", path = "../../../lib/entry/libimagentryref/" }
libimagentryutil = { version = "0.7.0", path = "../../../lib/entry/libimagentryutil/" }
+[dependencies.libimagentryref]
+version = "0.7.0"
+path = "../../../lib/entry/libimagentryref/"
+default-features = false
+features = ["generators", "generators-sha1"]
+
diff --git a/lib/domain/libimagcontact/src/contact.rs b/lib/domain/libimagcontact/src/contact.rs
index f94b7ae..375eeed 100644
--- a/lib/domain/libimagcontact/src/contact.rs
+++ b/lib/domain/libimagcontact/src/contact.rs
@@ -54,7 +54,7 @@ impl Contact for Entry {
fn get_contact_data(&self) -> Result<ContactData> {
let component = self
- .fs_file()
+ .get_path()
.map_err(From::from)
.and_then(util::read_to_string)
.and_then(util::parse)?;
diff --git a/lib/domain/libimagcontact/src/error.rs b/lib/domain/libimagcontact/src/error.rs
index 2074938..45535bd 100644
--- a/lib/domain/libimagcontact/src/error.rs
+++ b/lib/domain/libimagcontact/src/error.rs
@@ -34,6 +34,7 @@ error_chain! {
foreign_links {
Io(::std::io::Error);
TomlQueryError(::toml_query::error::Error);
+ UuidError(::uuid::ParseError);
}
errors {
diff --git a/lib/domain/libimagcontact/src/lib.rs b/lib/domain/libimagcontact/src/lib.rs
index 54dae12..31d0ac6 100644
--- a/lib/domain/libimagcontact/src/lib.rs
+++ b/lib/domain/libimagcontact/src/lib.rs
@@ -38,6 +38,7 @@
extern crate vobject;
extern crate toml;
extern crate toml_query;
+extern crate uuid;
#[macro_use] extern crate libimagstore;
extern crate libimagerror;
diff --git a/lib/domain/libimagcontact/src/store.rs b/lib/domain/libimagcontact/src/store.rs
index 9cb8cd0..fdfc838 100644
--- a/lib/domain/libimagcontact/src/store.rs
+++ b/lib/domain/libimagcontact/src/store.rs
@@ -17,22 +17,54 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
+use std::path::Path;
use std::path::PathBuf;
+use std::result::Result as RResult;
+use std::ffi::OsStr;
use vobject::parse_component;
+use uuid::Uuid;
use libimagstore::store::Store;
use libimagstore::store::FileLockEntry;
use libimagstore::storeid::StoreIdIterator;
use libimagentryref::refstore::RefStore;
-use libimagentryref::flags::RefFlags;
+use libimagentryref::refstore::UniqueRefPathGenerator;
+use libimagentryref::generators::sha1::Sha1;
use libimagentryutil::isa::Is;
use contact::IsContact;
+use error::ContactError as CE;
use error::Result;
use util;
-pub trait ContactStore<'a> : RefStore {
+pub struct UniqueContactPathGenerator;
+impl UniqueRefPathGenerator for UniqueContactPathGenerator {
+ type Error = CE;
+
+ /// The collection the `StoreId` should be created for
+ fn collection() -> &'static str {
+ "contact"
+ }
+
+ /// A function which should generate a unique string for a Path
+ fn unique_hash<A: AsRef<Path>>(path: A) -> RResult<String, Self::Error> {
+ debug!("Generating unique hash for path: {:?}", path.as_ref());
+
+ if let Some(p) = path.as_ref().file_name().and_then(OsStr::to_str).map(String::from) {
+ debug!("Found UUID string: '{}'", p);
+ Uuid::parse_str(&p)
+ .map_err(CE::from)
+ .map(|u| format!("{}", u.hyphenated())) // FIXME I don't know how to do in not-ugly
+ } else { // else, we sha1 the (complete) content
+ debug!("Couldn't find UUID string, using SHA1 of contents");
+ Sha1::unique_hash(path).map_err(CE::from)
+ }
+ }
+
+}
+
+pub trait ContactStore<'a> : RefStore<'a> {
// creating
@@ -42,7 +74,7 @@ pub trait ContactStore<'a> : RefStore {
///
/// Needs the `p` argument as we're finally creating a reference by path, the buffer is only for
/// collecting metadata.
- fn create_from_buf(&'a self, p: &PathBuf, buf: &String) -> Result<FileLockEntry<'a>>;
+ fn create_from_buf<P: AsRef<Path>>(&'a self, p: P, buf: &String) -> Result<FileLockEntry<'a>>;
// getting
@@ -63,12 +95,11 @@ impl<'a> ContactStore<'a> for Store {
}
/// Create contact ref from buffer
- fn create_from_buf(&'a self, p: &PathBuf, buf: &String) -> Result<FileLockEntry<'a>> {
+ fn create_from_buf<P: AsRef<Path>>(&'a self, p: P, buf: &String) -> Result<FileLockEntry<'a>> {
let component = parse_component(&buf)?;
debug!("Parsed: {:?}", component);
- let flags = RefFlags::default().with_content_hashing(true).with_permission_tracking(false);
- RefStore::create(self, p.clone(), flags)
+ RefStore::create_ref::<UniqueContactPathGenerator, P>(self, p)
.map_err(From::from)
.and_then(|mut entry| {
entry.set_isflag::<IsContact>()
@@ -78,7 +109,7 @@ impl<'a> ContactStore<'a> for Store {
}
fn all_contacts(&'a self) -> Result<StoreIdIterator> {
- self.all_references().map_err(From::from)
+ unimplemented!()
}
}
diff --git a/lib/domain/libimagmail/src/error.rs b/lib/domain/libimagmail/src/error.rs
index 870954f..519eb72 100644
--- a/lib/domain/libimagmail/src/error.rs
+++ b/lib/domain/libimagmail/src/error.rs
@@ -26,6 +26,10 @@ error_chain! {
RefError(::libimagentryref::error::RefError, ::libimagentryref::error::RefErrorKind);
}
+ foreign_links {
+ IoError(::std::io::Error);
+ }
+
errors {
RefCreationError {
diff --git a/lib/domain/libimagmail/src/hasher.rs b/lib/domain/libimagmail/src/hasher.rs
deleted file mode 100644
index 6c68c82..0000000
--- a/lib/domain/libimagmail/src/hasher.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-//
-// imag - the personal information management suite for the commandline
-// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; version
-// 2.1 of the License.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public
-// License along with this library; if not, write to the Free Software
-// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-//
-
-use std::io::Read;
-use std::path::PathBuf;
-
-use email::MimeMessage;
-
-use libimagentryref::hasher::Hasher;
-use libimagentryref::hasher::DefaultHasher;
-use libimagentryref::error::RefErrorKind as REK;
-use libimagentryref::error::ResultExt;
-use libimagentryref::error::Result as RResult;
-
-pub struct MailHasher {
- defaulthasher: DefaultHasher,
-}
-
-impl MailHasher {
-
- pub fn new() -> MailHasher {
- MailHasher { defaulthasher: DefaultHasher::new() }
- }
-
-}
-
-impl Hasher for MailHasher {
-
- fn hash_name(&self) -> &'static str {
- "default_mail_hasher"
- }
-
- fn create_hash<R: Read>(&mut self, pb: &PathBuf, c: &mut R) -> RResult<String> {
- use filters::filter::Filter;
- use email::Header;
-
- let mut s = String::new();
- c.read_to_string(&mut s)?;
-
- MimeMessage::parse(&s)
- .chain_err(|| REK::RefHashingError)
- .and_then(|mail| {
- let has_key = |hdr: &Header, exp: &str| hdr.name == exp;
-
- let subject_filter = |hdr: &Header| has_key(hdr, "Subject");
- let from_filter = |hdr: &Header| has_key(hdr, "From");
- let to_filter = |hdr: &Header| has_key(hdr, "To");
-
- let filter = subject_filter.or(from_filter).or(to_filter);
-
- let mut v : Vec<String> = vec![];
- for hdr in mail.headers.iter().filter(|item| filter.filter(item)) {
- let s = hdr
- .get_value()
- .chain_err(|| REK::RefHashingError)?;
-
- v.push(s);
- }
- let s : String = v.join("");
-
- self.defaulthasher.create_hash(pb, &mut s.as_bytes())
- })
- }
-
-}
diff --git a/lib/domain/libimagmail/src/lib.rs b/lib/domain/libimagmail/src/lib.rs
index da0460e..1439f9b 100644
--- a/lib/domain/libimagmail/src/lib.rs
+++ b/lib/domain/libimagmail/src/lib.rs
@@ -45,7 +45,6 @@ extern crate libimagstore;
extern crate libimagentryref;
pub mod error;
-pub mod hasher;
pub mod iter;
pub mod mail;
diff --git a/lib/domain/libimagmail/src/mail.rs b/lib/domain/libimagmail/src/mail.rs
index 7536b11..bf16c4a 100644
--- a/lib/domain/libimagmail/src/mail.rs
+++ b/lib/domain/libimagmail/src/mail.rs
@@ -18,22 +18,75 @@
//
use std::path::Path;
-use std::path::PathBuf;
use std::fs::File;
use std::io::Read;
+use std::fs::OpenOptions;
+use std::result::Result as RResult;
use libimagstore::store::Store;
+use libimagstore::storeid::StoreId;
use libimagstore::store::FileLockEntry;
use libimagentryref::reference::Ref;
-use libimagentryref::flags::RefFlags;
use libimagentryref::refstore::RefStore;
+use libimagentryref::refstore::UniqueRefPathGenerator;
use email::MimeMessage;
use email::results::ParsingResult as EmailParsingResult;
-use hasher::MailHasher;
use error::Result;
-use error::{ResultExt, MailErrorKind as MEK};
+use error::{ResultExt, MailError as ME, MailErrorKind as MEK};
+
+struct UniqueMailRefGenerator;
+impl UniqueRefPathGenerator for UniqueMailRefGenerator {
+ type Error = ME;
+
+ /// The collection the `StoreId` should be created for
+ fn collection() -> &'static str {
+ "mail"
+ }
+
+ /// A function which should generate a unique string for a Path
+ fn unique_hash<A: AsRef<Path>>(path: A) -> RResult<String, Self::Error> {
+ use filters::filter::Filter;
+ use email::Header;
+
+ let mut s = String::new();
+ let _ = OpenOptions::new()
+ .read(true)
+ .write(false)
+ .create(false)
+ .open(path)?
+ .read_to_string(&mut s)?;
+
+ MimeMessage::parse(&s)
+ .chain_err(|| MEK::RefCreationError)
+ .and_then(|mail| {
+ let has_key = |hdr: &Header, exp: &str| hdr.name == exp;
+
+ let subject_filter = |hdr: &Header| has_key(hdr, "Subject");
+ let from_filter = |hdr: &Header| has_key(hdr, "From");
+ let to_filter = |hdr: &Header| has_key(hdr, "To");
+
+ let filter = subject_filter.or(from_filter).or(to_filter);
+
+ let mut v : Vec<String> = vec![];
+ for hdr in mail.headers.iter().filter(|item| filter.filter(item)) {
+ let s = hdr
+ .get_value()
+ .chain_err(|| MEK::RefCreationError)?;
+
+ v.push(s);
+ }
+ let s : String = v.join("");
+ Ok(s)
+ })
+ }
+
+ /// Postprocess the generated `StoreId` object
+ fn postprocess_storeid(sid: StoreId) -> RResult<StoreId, Self::Error> {
+ Ok(sid)
+ }
+}
struct Buffer(String);
@@ -56,15 +109,10 @@ impl<'a> Mail<'a> {
/// Imports a mail from the Path passed
pub fn import_from_path<P: AsRef<Path>>(store: &Store, p: P) -> Result<Mail> {
debug!("Importing Mail from path");
- let h = MailHasher::new();
- let f = RefFlags::default().with_content_hashing(true).with_permission_tracking(false);
- let p = PathBuf::from(p.as_ref());
-
- store.create_with_hasher(p, f, h)
- .chain_err(|| MEK::RefCreationError)
+ store.retrieve_ref::<UniqueMailRefGenerator, P>(p)
.and_then(|reference| {
debug!("Build reference file: {:?}", reference);
- reference.fs_file()
+ reference.get_path()
.chain_err(|| MEK::RefHandlingError)
.and_then(|path| File::open(path).chain_err(|| MEK::IOError))
.and_then(|mut file| {
@@ -81,19 +129,18 @@ impl<'a> Mail<'a> {
/// Opens a mail by the passed hash
pub fn open<S: AsRef<str>>(store: &Store, hash: S) -> Result<Option<Mail>> {
debug!("Opening Mail by Hash");
- store.get_by_hash(String::from(hash.as_ref()))
+ store.get_ref::<UniqueMailRefGenerator, S>(hash)
.chain_err(|| MEK::FetchByHashError)
.chain_err(|| MEK::FetchError)
.and_then(|o| match o {
Some(r) => Mail::from_fle(r).map(Some),
None => Ok(None),
})
-
}
/// Implement me as TryFrom as soon as it is stable
pub fn from_fle(fle: FileLockEntry<'a>) -> Result<Mail<'a>> {
- fle.fs_file()
+ fle.get_path()
.chain_err(|| MEK::RefHandlingError)
.and_then(|path| File::open(path).chain_err(|| MEK::IOError))
.and_then(|mut file| {
diff --git a/lib/entry/libimagentrymarkdown/Cargo.toml b/lib/entry/libimagentrymarkdown/Cargo.toml
index d7981b5..7ed53b7 100644
--- a/lib/entry/libimagentrymarkdown/Cargo.toml
+++ b/lib/entry/libimagentrymarkdown/Cargo.toml
@@ -29,6 +29,11 @@ env_logger = "0.5"
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" }
libimagentrylink = { version = "0.7.0", path = "../../../lib/entry/libimagentrylink/" }
-libimagentryref = { version = "0.7.0", path = "../../../lib/entry/libimagentryref/" }
libimagutil = { version = "0.7.0", path = "../../../lib/etc/libimagutil/" }
+[dependencies.libimagentryref]
+version = "0.7.0"
+path = "../../../lib/entry/libimagentryref/"
+default-features = false
+features = [ "generators", "generators-sha512" ]
+
diff --git a/lib/entry/libimagentrymarkdown/src/processor.rs b/lib/entry/libimagentrymarkdown/src/processor.rs
index 15d60d4..7203d9a 100644
--- a/lib/entry/libimagentrymarkdown/src/processor.rs
+++ b/lib/entry/libimagentrymarkdown/src/processor.rs
@@ -17,6 +17,9 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
+use std::path::Path;
+use std::result::Result as RResult;
+
use error::MarkdownError as ME;
use error::MarkdownErrorKind as MEK;
use error::*;
@@ -25,7 +28,8 @@ use link::extract_links;
use libimagentrylink::external::ExternalLinker;
use libimagentrylink::internal::InternalLinker;
use libimagentryref::refstore::RefStore;
-use libimagentryref::flags::RefFlags;
+use libimagentryref::refstore::UniqueRefPathGenerator;
+use libimagentryref::generators::sha512::Sha512;
use libimagstore::store::Entry;
use libimagstore::store::Store;
use libimagstore::storeid::StoreId;
@@ -34,6 +38,25 @@ use std::path::PathBuf;
use url::Url;
+
+pub struct UniqueMarkdownRefGenerator;
+
+impl UniqueRefPathGenerator for UniqueMarkdownRefGenerator {
+ type Error = ME;
+
+ fn collection() -> &'static str {
+ "ref" // we can only use this collection, as we don't know about context
+ }
+
+ fn unique_hash<A: AsRef<Path>>(path: A) -> RResult<String, Self::Error> {
+ Sha512::unique_hash(path).map_err(ME::from)
+ }
+
+ fn postprocess_storeid(sid: StoreId) -> RResult<StoreId, Self::Error> {
+ Ok(sid) // don't do anything
+ }
+}
+
/// A link Processor which collects the links from a Markdown and passes them on to
/// `libimagentrylink` functionality
///
@@ -136,15 +159,12 @@ impl LinkProcessor {
continue
}
- let flags = RefFlags::default()
- .with_content_hashing(false)
- .with_permission_tracking(false);
trace!("URL = {:?}", url);
trace!("URL.path() = {:?}", url.path());
trace!("URL.host_str() = {:?}", url.host_str());
let path = url.host_str().unwrap_or_else(|| url.path());
let path = PathBuf::from(path);
- let mut target = RefStore::create(store, path, flags)?;
+ let mut target = store.create_ref::<UniqueMarkdownRefGenerator, PathBuf>(path)?;
entry.add_internal_link(&mut target)?;
},
diff --git a/lib/entry/libimagentryref/Cargo.toml b/lib/entry/libimagentryref/Cargo.toml
index e46d210..46178af 100644
--- a/lib/entry/libimagentryref/Cargo.toml
+++ b/lib/entry/libimagentryref/Cargo.toml
@@ -22,13 +22,26 @@ maintenance = { status = "actively-developed" }
[dependencies]
itertools = "0.7"
log = "0.4.0"
-rust-crypto = "0.2"
toml = "0.4"
toml-query = "0.6"
error-chain = "0.11"
-walkdir = "1"
libimagstore = { version = "0.7.0", path = "../../../lib/core/libimagstore" }
libimagerror = { version = "0.7.0", path = "../../../lib/core/libimagerror" }
libimagentrylist = { version = "0.7.0", path = "../../../lib/entry/libimagentrylist" }
libimagentryutil = { version = "0.7.0", path = "../../../lib/entry/libimagentryutil" }
+
+[dependencies.rust-crypto]
+version = "0.2"
+optional = true
+
+[features]
+default = []
+generators = []
+generators-sha1 = ["rust-crypto"]
+generators-sha224 = ["rust-crypto"]
+generators-sha256 = ["rust-crypto"]
+generators-sha384 = ["rust-crypto"]
+generators-sha512 = ["rust-crypto"]
+generators-sha3 = ["rust-crypto"]
+
diff --git a/lib/entry/libimagentryref/src/error.rs b/lib/entry/libimagentryref/src/error.rs
index 3c75c20..98fd735 100644
--- a/lib/entry/libimagentryref/src/error.rs
+++ b/lib/entry/libimagentryref/src/error.rs
@@ -23,7 +23,6 @@ error_chain! {
}
links {
- ListError(::libimagentrylist::error::ListError, ::libimagentrylist::error::ListErrorKind);
StoreError(::libimagstore::error::StoreError, ::libimagstore::error::StoreErrorKind);
TomlQueryError(::toml_query::error::Error, ::toml_query::error::ErrorKind);
EntryUtilError(::libimagentryutil::error::EntryUtilError, ::libimagentryutil::error::EntryUtilErrorKind);
@@ -34,31 +33,25 @@ error_chain! {
Utf8Error(::std::string::FromUtf8Error);
TomlDeError(::toml::de::Error);
TomlSerError(::toml::ser::Error);
- WalkDirError(::walkdir::Error);
}
errors {
- UTF8Error {
- description("UTF8 Error")
- display("UTF8 Error")
- }
-
- HeaderTypeError {
+ HeaderTypeError(field: &'static str, expectedtype: &'static str) {
description("Header type error")
- display("Header type error")
+ display("Header type error: '{}' should be {}", field, expectedtype)
}
- HeaderFieldMissingError {
+ HeaderFieldMissingError(field: &'static str) {
description("Header field missing error")
- display("Header field missing error")
+ display("Header field missing: {}", field)
}
- HeaderFieldWriteError {
+ HeaderFieldWriteError {
description("Header field cannot be written")
display("Header field cannot be written")
}
- HeaderFieldReadError {
+ HeaderFieldReadError {
description("Header field cannot be read")
display("Header field cannot be read")
}
@@ -73,61 +66,6 @@ error_chain! {
display("Path cannot be converted because of UTF8 Error")
}
- PathHashingError {
- description("Path cannot be hashed")
- display("Path cannot be hashed")
- }
-
- PathCanonicalizationError {
- description("Path cannot be canonicalized")
- display("Path cannot be canonicalized")
- }
-
- TypeConversionError {
- description("Couldn't convert types")
- display("Couldn't convert types")
- }
-
- RefToDisplayError {
- description("Cannot convert Ref to string to show it to user")
- display("Cannot convert Ref to string to show it to user")
- }
-
- RefNotInStore {
- description("Ref/StoreId does not exist in store")
- display("Ref/StoreId does not exist in store")
- }
-
- RefTargetDoesNotExist {
- description("Ref Target does not exist")
- display("Ref Target does not exist")
- }
-
- RefTargetPermissionError {
- description("Ref Target permissions insufficient for referencing")
- display("Ref Target permissions insufficient for referencing")
- }
-
- RefTargetCannotBeHashed {
- description("Ref Target cannot be hashed (is it a directory?)")
- display("Ref Target cannot be hashed (is it a directory?)")
- }
-
- RefTargetFileCannotBeOpened {
- description("Ref Target File cannot be open()ed")
- display("Ref Target File cannot be open()ed")
- }
-
- RefTargetCannotReadPermissions {
- description("Ref Target: Cannot read permissions")
- display("Ref Target: Cannot read permissions")
- }
-
- RefHashingError {
- description("Error while hashing")
- display("Error while hashing")
- }
-
}
}
diff --git a/lib/entry/libimagentryref/src/flags.rs b/lib/entry/libimagentryref/src/flags.rs
deleted file mode 100644
index ae5149a..0000000
--- a/lib/entry/libimagentryref/src/flags.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-//
-// imag - the personal information management suite for the commandline
-// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; version
-// 2.1 of the License.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public
-// License along with this library; if not, write to the Free Software
-// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-//
-
-use std::collections::BTreeMap;
-
-use toml::Value;
-
-use error::RefError as RE;
-use error::RefErrorKind as REK;
-use error::Result;
-
-pub struct RefFlags {
- content_hashing: bool,
- permission_tracking: bool,
-}
-
-impl RefFlags {
-
- /// Read the RefFlags from a TOML document
- ///
- /// Assumes that the whole TOML tree is passed. So this looks up `ref.flags` to get the flags.
- /// It assumes that this is a Map with Key = <name of the setting> and Value = boolean.
- pub fn read(v: &Value) -> Result<RefFlags> {
- fn get_field(v: &Value, key: &str) -> Result<bool> {
- use toml_query::read::TomlValueReadTypeExt;
- v.read_bool(key)?.ok_or(RE::from_kind(REK::HeaderFieldMissingError))
- }
-
- Ok(RefFlags {
- content_hashing: get_field(v, "ref.flags.content_hashing")?,
- permission_tracking: get_field(v, "ref.flags.permission_tracking")?,
- })
- }
-
- /// Alias for `RefFlags::content_hashing()`
- pub fn is_often_moving(self, b: bool) -> RefFlags {
- self.with_content_hashing(b)
- }
-
- pub fn with_content_hashing(mut self, b: bool) -> RefFlags {
- self.content_hashing = b;
- self
- }
-
- pub fn with_permission_tracking(mut self, b: bool) -> RefFlags {
- self.permission_tracking = b;
- self
- }
-
-
- pub fn get_content_hashing(&self) -> bool {
- self.content_hashing
- }
-
- pub fn get_permission_tracking(&self) -> bool {
- self.permission_tracking
- }
-
-}
-
-impl Into<Value> for RefFlags {
-
- /// Build a TOML::Value from this RefFlags object.
- ///
- /// Returns a Map which should be set in `ref.flags` in the header.
- fn into(self) -> Value {
- let mut btm = BTreeMap::new();
- btm.insert(String::from("content_hashing"), Value::Boolean(self.content_hashing));
- btm.insert(String::from("permission_tracking"), Value::Boolean(self.permission_tracking));
- return Value::Table(btm)
- }
-
-}
-
-impl Default for RefFlags {
-
- fn default() -> RefFlags {
- RefFlags {
- content_hashing: false,
- permission_tracking: false,
- }
- }
-}
-
diff --git a/lib/entry/libimagentryref/src/hashers/mod.rs b/lib/entry/libimagentryref/src/generators/base.rs
index 9e55e80..aea60b2 100644
--- a/lib/entry/libimagentryref/src/hashers/mod.rs
+++ b/lib/entry/libimagentryref/src/generators/base.rs
@@ -17,4 +17,32 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
-pub mod nbytes;
+use std::path::Path;
+
+use libimagstore::storeid::StoreId;
+
+use error::RefError;
+use refstore::UniqueRefPathGenerator;
+
+/// A base UniqueRefPathGenerator which must be overridden by the actual UniqueRefPathGenerator
+/// which is provided by this crate
+#[allow(dead_code)]
+pub struct Base;
+
+impl UniqueRefPathGenerator for Base {
+ type Error = RefError;
+
+ fn collection() -> &'static str {
+ "ref"
+ }
+
+ fn unique_hash<A: AsRef<Path>>(_path: A) -> Result<String, Self::Error> {
+ // This has to be overridden
+ panic!("Not overridden base functionality. This is a BUG!")
+ }
+
+ fn postprocess_storeid(sid: StoreId) -> Result<StoreId, Self::Error> {
+ Ok(sid) // default implementation
+ }
+}
+
diff --git a/lib/entry/libimagentryref/src/generators/mod.rs b/lib/entry/libimagentryref/src/generators/mod.rs
new file mode 100644
index 0000000..21b31d7
--- /dev/null
+++ b/lib/entry/libimagentryref/src/generators/mod.rs
@@ -0,0 +1,274 @@
+//
+// imag - the personal information management suite for the commandline
+// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; version
+// 2.1 of the License.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+//
+
+//! Default generators
+//!
+//! This module provides a number of default `UniqueRefPathGenerator`s
+//! which can be used for generating references.
+//!
+//! These generators are _NOT_ domain specific. So there won't be a "UniqueMailRefPathGenerator" in
+//! here, for example.
+//!
+//! All these generators use "ref" as collection name.
+//! They can be overridden using the `make_unique_ref_path_generator!()` convenience macro.
+//!
+//! # Note
+//!
+//! You must enable the appropriate crate feature to use any of the provided generators. With the
+//! `generators` feature, you only get the convenience macro `make_unique_ref_path_generator!()`.
+//!
+
+/// A convenience macro for wrapping a generator in a new one, reusing the functionality from the
+/// underlying generator
+///
+/// The UniqueRefPathGenerator must be in scope.
+///
+/// The macro creates a new struct `$name` over `$underlying` and changes the collection name to
+/// `$collectionname`.
+/// If passed, the new implementation is used (defaults to the implementation from the underlying
+/// generator).
+/// If passed, the new postprocessing is used (defaults to not changing the StoreId)
+///
+#[macro_export]
+macro_rules! make_unique_ref_path_generator {
+ (
+ $name:ident
+ over $underlying:ty
+ => with error $errtype:ty
+ => with collection name $collectionname:expr
+ ) => {
+ struct $name;
+
+ impl $crate::refstore::UniqueRefPathGenerator for $name {
+ type Error = $errtype;
+
+ fn collection() -> &'static str {
+ $collectionname
+ }
+
+ fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String, Self::Error> {
+ $underlying::unique_hash(path)
+ }
+
+ fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
+ -> Result<::libimagstore::storeid::StoreId, Self::Error>
+ {
+ Ok(sid)
+ }
+ }
+ };
+
+ (
+ $name:ident
+ over $underlying:ty
+ => with error $errtype:ty
+ => with collection name $collectionname:expr
+ => $impl:expr
+ ) => {
+ struct $name;
+
+ impl $crate::refstore::UniqueRefPathGenerator for $name {
+ type Error = $errtype;
+
+ fn collection() -> &'static str {
+ $collectionname
+ }
+
+ fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String, Self::Error> {
+ $impl(path)
+ }
+
+ fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
+ -> Result<::libimagstore::storeid::StoreId, Self::Error>
+ {
+ Ok(sid)
+ }
+ }
+ };
+
+ (
+ pub $name:ident
+ over $underlying:ty
+ => with error $errtype:ty
+ => with collection name $collectionname:expr
+ => $impl:expr
+ ) => {
+ make_unique_ref_path_generator!(
+ pub $name
+ over $underlying
+ => with error $errtype
+ => with collection name $collectionname
+ => $impl => |sid| { Ok(sid) }
+ );
+ };
+
+ (
+ pub $name:ident
+ over $underlying:ty
+ => with error $errtype:ty
+ => with collection name $collectionname:expr
+ => $impl:expr
+ => $postproc:expr
+ ) => {
+ pub struct $name;
+
+ impl $crate::refstore::UniqueRefPathGenerator for $name {
+ type Error = $errtype;
+
+ fn collection() -> &'static str {
+ $collectionname
+ }
+
+ fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String, Self::Error> {
+ $impl(path)
+ }
+
+ fn postprocess_storeid(sid: ::libimagstore::storeid::StoreId)
+ -> Result<::libimagstore::storeid::StoreId, Self::Error>
+ {
+ $postproc(sid)
+ }
+ }
+ };
+}
+
+
+#[cfg(any(
+ feature = "generators-sha1",
+ feature = "generators-sha224",
+ feature = "generators-sha256",
+ feature = "generators-sha384",
+ feature = "generators-sha512",
+ feature = "generators-sha3",
+ ))]
+mod base;
+
+/// Helper macro for generating implementations for the various Sha algorithms
+macro_rules! make_sha_mod {
+ {
+ $modname:ident,
+ $hashname:ident,
+ $hashingimpl:expr
+ } => {
+ pub mod $modname {
+ use std::path::Path;
+ use std::fs::OpenOptions;
+ use std::io::Read;
+
+ use error::RefError as RE;
+
+ use crypto::digest::Digest;
+ make_unique_ref_path_generator! (
+ pub $hashname
+ over generators::base::Base
+ => with error RE
+ => with collection name "ref"
+ => |path| {
+ OpenOptions::new()
+ .read(true)
+ .write(false)
+ .create(false)
+ .open(path)
+ .map_err(RE::from)
+ .and_then(|mut file| {
+ let mut buffer = String::new();
+ let _ = file.read_to_string(&mut buffer)?;
+ $hashingimpl(buffer)
+ })
+ }
+ );
+
+ impl $hashname {
+
+ /// Function which can be used by a wrapping UniqueRefPathGenerator to hash only N bytes.
+ pub fn hash_n_bytes<A: AsRef<Path>>(path: A, n: usize) -> Result<String, RE> {
+ OpenOptions::new()
+ .read(true)
+ .write(false)
+ .create(false)
+ .open(path)
+ .map_err(RE::from)
+ .and_then(|mut file| {
+ let mut buffer = Vec::with_capacity(n);
+ let _ = file.read_exact(&mut buffer)?;
+ let buffer = String::from_utf8(buffer)?;
+ $hashingimpl(buffer)
+ })
+ }
+
+ }
+
+ }
+ }
+}
+
+#[cfg(feature = "generators-sha1")]
+make_sha_mod! {
+ sha1, Sha1, |buffer: String| {
+ let mut hasher = ::crypto::sha1::Sha1::new();
+ hasher.input_str(&buffer);
+ Ok(String::from(hasher.result_str()))
+ }
+}
+
+#[cfg(feature = "generators-sha224")]
+make_sha_mod! {
+ sha224, Sha224, |buffer: String| {
+ let mut hasher = ::crypto::sha2::Sha224::new();
+ hasher.input_str(&buffer);
+ Ok(String::from(hasher.result_str()))
+ }
+}
+
+#[cfg(feature = "generators-sha256")]
+make_sha_mod! {
+ sha256, Sha256, |buffer: String| {
+ let mut hasher = ::crypto::sha2::Sha256::new();
+ hasher.input_str(&buffer);
+ Ok(String::from(hasher.result_str()))
+ }
+}
+
+#[cfg(feature = "generators-sha384")]
+make_sha_mod! {
+ sha384, Sha384, |buffer: String| {
+ let mut hasher = ::crypto::sha2::Sha384::new();
+ hasher.input_str(&buffer);
+ Ok(String::from(hasher.result_str()))
+ }
+}
+
+#[cfg(feature = "generators-sha512")]
+make_sha_mod! {
+ sha512, Sha512, |buffer: String| {
+ let mut hasher = ::crypto::sha2::Sha512::new();
+ hasher.input_str(&buffer);
+ Ok(String::from(hasher.result_str()))
+ }
+}
+
+#[cfg(feature = "generators-sha3")]
+make_sha_mod! {
+ sha3, Sha3, |buffer: String| {
+ let mut hasher = ::crypto::sha3::Sha3::sha3_256();
+ hasher.input_str(&buffer);
+ Ok(String::from(hasher.result_str()))
+ }
+}
+
diff --git a/lib/entry/libimagentryref/src/hasher.rs b/lib/entry/libimagentryref/src/hasher.rs
deleted file mode 100644
index 02cf8e8..0000000
--- a/lib/entry/libimagentryref/src/hasher.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-//
-// imag - the personal information management suite for the commandline
-// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; version
-// 2.1 of the License.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public
-// License along with this library; if not, write to the Free Software
-// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-//
-
-use std::path::PathBuf;
-use std::io::Read;
-
-use crypto::sha1::Sha1;
-use crypto::digest::Digest;
-
-use error::Result;
-
-/// The Hasher trait is used to implement custom hashing functions for the ref library.
-/// This means that one can define how the hash of a reference is constructed from the content of
-/// the file to ref to.
-pub trait Hasher {
-
- fn hash_name(&self) -> &'static str;
- fn create_hash<R: Read>(&mut self, pb: &PathBuf, contents: &mut R) -> Result<String>;
-
-}
-
-pub struct DefaultHasher {
- hasher: Sha1,
-}
-
-impl DefaultHasher {
-
- pub fn new() -> DefaultHasher {
- DefaultHasher { hasher: Sha1::new() }
- }
-
-}
-
-impl Hasher for DefaultHasher {
-
- fn hash_name(&self) -> &'static str {
- "default"
- }
-
- fn create_hash<R: Read>(&mut self, _: &PathBuf, c: &mut R) -> Result<String> {
- let mut s = String::new();
- c.read_to_string(&mut s)?;
- self.hasher.input_str(&s[..]);
- Ok(self.hasher.result_str())
- }
-
-}
-
diff --git a/lib/entry/libimagentryref/src/hashers/nbytes.rs b/lib/entry/libimagentryref/src/hashers/nbytes.rs
deleted file mode 100644
index a55dd70..0000000
--- a/lib/entry/libimagentryref/src/hashers/nbytes.rs
+++ /dev/null
@@ -1,66 +0,0 @@
-//
-// imag - the personal information management suite for the commandline
-// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; version
-// 2.1 of the License.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public
-// License along with this library; if not, write to the Free Software
-// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-//
-
-use std::io::Read;
-use std::path::PathBuf;
-use std::result::Result as RResult;
-
-use crypto::sha1::Sha1;
-use crypto::digest::Digest;
-
-use hasher::Hasher;
-use error::Result;
-use error::RefError as RE;
-
-pub struct NBytesHasher {
- hasher: Sha1,
- n: usize,
-}
-
-impl NBytesHasher {
-
- pub fn new(n: usize) -> NBytesHasher {
- NBytesHasher {
- hasher: Sha1::new(),
- n: n,
- }
- }
-
-}
-
-impl Hasher for NBytesHasher {
-
- fn hash_name(&self) -> &'static str {
- "n-bytes-hasher"
- }
-
- fn create_hash<R: Read>(&mut self, _: &PathBuf, contents: &mut R) -> Result<String> {
- let s : String = contents
- .bytes()
- .take(self.n)
- .collect::<RResult<Vec<u8>, _>>()
- .map_err(From::from)
- .and_then(|v| String::from_utf8(v).map_err(RE::from))?;
-
- self.hasher.input_str(&s[..]);
- Ok(self.hasher.result_str())
- }
-
-}
-
diff --git a/lib/entry/libimagentryref/src/lib.rs b/lib/entry/libimagentryref/src/lib.rs
index e32a9e8..39f3c54 100644
--- a/lib/entry/libimagentryref/src/lib.rs
+++ b/lib/entry/libimagentryref/src/lib.rs
@@ -36,11 +36,9 @@
)]
#[macro_use] extern crate log;
-extern crate crypto;
extern crate itertools;
extern crate toml;
extern crate toml_query;
-extern crate walkdir;
#[macro_use] extern crate libimagstore;
extern crate libimagerror;
@@ -51,10 +49,19 @@ extern crate libimagentrylist;
module_entry_path_mod!("ref");
pub mod error;
-pub mod flags;
-pub mod hasher;
-pub mod hashers;
-pub mod lister;
pub mod reference;
pub mod refstore;
-mod util;
+
+#[cfg(any(
+ feature = "generators-sha1",
+ feature = "generators-sha224",
+ feature = "generators-sha256",
+ feature = "generators-sha384",
+ feature = "generators-sha512",
+ feature = "generators-sha3",
+))]
+extern crate crypto;
+
+#[cfg(feature = "generators")]
+pub mod generators;
+
diff --git a/lib/entry/libimagentryref/src/lister.rs b/lib/entry/libimagentryref/src/lister.rs
deleted file mode 100644
index a3219b9..0000000
--- a/lib/entry/libimagentryref/src/lister.rs
+++ /dev/null
@@ -1,161 +0,0 @@
-//
-// imag - the personal information management suite for the commandline
-// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; version
-// 2.1 of the License.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public
-// License along with this library; if not, write to the Free Software
-// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-//
-
-use std::default::Default;
-use std::io::stdout;
-use std::io::Write;
-use std::ops::Deref;
-
-use libimagentrylist::lister::Lister;
-use libimagentrylist::error::Result;
-use libimagerror::trace::trace_error;
-use libimagstore::store::FileLockEntry;
-use libimagentrylist::error::ListErrorKind as LEK;
-use libimagentrylist::error as lerror;
-
-use reference::Ref;
-
-pub struct RefLister {
- check_dead: bool,
- check_changed: bool,
- check_changed_content: bool,
- check_changed_permiss: bool,
-}
-
-impl RefLister {
-
- pub fn new() -> RefLister {
- RefLister::default()
- }
-
- pub fn check_dead(mut self, b: bool) -> RefLister {
- self.check_dead = b;
- self
- }
-
- pub fn check_changed(mut self, b: bool) -> RefLister {
- self.check_changed = b;
- self
- }
-
- pub fn check_changed_content(mut self, b: bool) -> RefLister {
- self.check_changed_content = b;
- self
- }
-
- pub fn check_changed_permiss(mut self, b: bool) -> RefLister {
- self.check_changed_permiss = b;
- self
- }
-
-}
-
-impl Default for RefLister {
-
- fn default() -> RefLister {
- RefLister {
- check_dead: false,
- check_changed: false,
- check_changed_content: false,
- check_changed_permiss: false,
- }
- }
-}
-
-impl Lister for RefLister {
-
- fn list<'b, I: Iterator<Item = FileLockEntry<'b>>>(&self, entries: I) -> Result<()> {
-
- debug!("Called list()");
- let (r, n) = entries.fold((Ok(()), 0), |(accu, i), entry| {
- debug!("fold({:?}, {:?})", accu, entry);
- let r = accu.and_then(|_| {
- debug!("Listing Entry: {:?}", entry);
- {
- let is_dead = if self.check_dead {
- if lerror::ResultExt::chain_err(entry.fs_link_exists(), || LEK::FormatError)? {
- "dead"
- } else {
- "alive"
- }
- } else {
- "not checked"
- };
-
- let is_changed = if self.check_changed {
- if check_changed(entry.deref()) { "changed" } else { "unchanged" }
- } else {
- "not checked"
- };
-
- let is_changed_content = if self.check_changed_content {
- if check_changed_content(entry.deref()) { "changed" } else { "unchanged" }
- } else {
- "not checked"
- };
-
- let is_changed_permiss = if self.check_changed_permiss {
- if check_changed_permiss(entry.deref()) { "changed" } else { "unchanged" }
- } else {
- "not checked"
- };
-
- Ok(format!("{} | {} | {} | {} | {} | {}",
- is_dead,
- is_changed,
- is_changed_content,
- is_changed_permiss,
- entry.get_path_hash().unwrap_or_else(|_| String::from("Cannot get hash")),
- entry.get_location()))
- }
- .and_then(|s| {
- lerror::ResultExt::chain_err(write!(stdout(), "{}\n", s), || LEK::FormatError)
- })
- })
- .map(|_| ());
- (r, i + 1)
- });
- debug!("Iterated over {} entries", n);
- r
- }
-
-}
-
-fn check_changed<R: Ref>(r: &R) -> bool {
- check_changed_content(r) && check_changed_permiss(r)
-}
-
-fn check_changed_content<R: Ref>(r: &R) -> bool {
- r.get_current_hash()
- .and_then(|hash| r.get_stored_hash().map(|stored| (hash, stored)))
- .map(|(hash, stored)| hash == stored)
- .unwrap_or_else(|e| {
- warn!("Could not check whether the ref changed on the FS");
- trace_error(&e);
-
- // We continue here and tell the callee that this reference is unchanged
- false
- })
-}
-
-fn check_changed_permiss<R: Ref>(_: &R) -> bool {
- warn!("Permission changes tracking not supported yet.");
- false
-}
-
diff --git a/lib/entry/libimagentryref/src/reference.rs b/lib/entry/libimagentryref/src/reference.rs
index dcfc9a1..e2b081d 100644
--- a/lib/entry/libimagentryref/src/reference.rs
+++ b/lib/entry/libimagentryref/src/reference.rs
@@ -21,316 +21,98 @@
//! files outside of the imag store.
use std::path::PathBuf;
-use std::fs::File;
-use std::fs::Permissions;
+use std::result::Result as RResult;
-use libimagstore::store::Entry;
use libimagentryutil::isa::Is;
use libimagentryutil::isa::IsKindHeaderPathProvider;
+use libimagstore::store::Entry;
-use toml::Value;
-use toml_query::read::TomlValueReadTypeExt;
-use toml_query::set::TomlValueSetExt;
+use toml_query::read::TomlValueReadExt;
+use toml_query::delete::TomlValueDeleteExt;
-use error::RefErrorKind as REK;
-use error::RefError as RE;
-use error::ResultExt;
+use refstore::UniqueRefPathGenerator;
use error::Result;
-use hasher::*;
+use error::RefError as RE;
+use error::RefErrorKind as REK;
pub trait Ref {
/// Check whether the underlying object is actually a ref
fn is_ref(&self) -> Result<bool>;
- /// Get the hash from the path of the ref
- fn get_path_hash(&self) -> Result<String>;
-
- /// Get the hash of the link target which is stored in the ref object
- fn get_stored_hash(&self) -> Result<String>;
-
- /// Get the hahs of the link target which is stored in the ref object, which is hashed with a
- /// custom Hasher instance.
- fn get_stored_hash_with_hasher<H: Hasher>(&self, h: &H) -> Result<String>;
-
- /// Get the hash of the link target by reading the link target and hashing the contents
- fn get_current_hash(&self) -> Result<String>;
-
- /// Get the hash of the link target by reading the link target and hashing the contents with the
- /// custom hasher
- fn get_current_hash_with_hasher<H: Hasher>(&self, h: H) -> Result<String>;
-
- /// check whether the pointer the Ref represents still points to a file which exists
- fn fs_link_exists(&self) -> Result<bool>;
-
- /// Alias for `r.fs_link_exists() && r.deref().is_file()`
- fn is_ref_to_file(&self) -> Result<bool>;
-
- /// Alias for `r.fs_link_exists() && r.deref().is_dir()`
- fn is_ref_to_dir(&self) -> Result<bool>;
-
- /// Alias for `!Ref::fs_link_exists()`
- fn is_dangling(&self) -> Result<bool>;
-
- /// check whether the pointer the Ref represents is valid
- /// This includes:
- /// - Hashsum of the file is still the same as stored in the Ref
- /// - file permissions are still valid
- fn fs_link_valid(&self) -> Result<bool>;
-
- /// Check whether the file permissions of the referenced file are equal to the stored
- /// permissions
- fn fs_link_valid_permissions(&self) -> Result<bool>;
-
- /// Check whether the Hashsum of the referenced file is equal to the stored hashsum
- fn fs_link_valid_hash(&self) -> Result<bool>;
-
- /// Update the Ref by re-checking the file from FS
- /// This errors if the file is not present or cannot be read()
- fn update_ref(&mut self) -> Result<()>;
-
- /// Update the Ref by re-checking the file from FS using the passed Hasher instance
- /// This errors if the file is not present or cannot be read()
- fn update_ref_with_hasher<H: Hasher>(&mut self, h: &H) -> Result<()>;
-
- /// Get the path of the file which is reffered to by this Ref
- fn fs_file(&self) -> Result<PathBuf>;
-
- /// Re-find a referenced file
+ /// Get the stored hash.
///
- /// This function tries to re-find a ref by searching all directories in `search_roots` recursively
- /// for a file which matches the hash of the Ref.
- ///
- /// If `search_roots` is `None`, it starts at the filesystem root `/`.
- ///
- /// If the target cannot be found, this yields a RefTargetDoesNotExist error kind.
- ///
- /// # Warning
- ///
- /// This option causes heavy I/O as it recursively searches the Filesystem.
- fn refind(&self, search_roots: Option<Vec<PathBuf>>) -> Result<PathBuf>;
-
- /// See documentation of `Ref::refind()`
- fn refind_with_hasher<H: Hasher>(&self, search_roots: Option<Vec<PathBuf>>, h: H)
- -> Result<PathBuf>;
+ /// Does not need a `UniqueRefPathGenerator` as it reads the hash stored in the header
+ fn get_hash(&self) -> Result<&str>;
- /// Get the permissions of the file which are present
- fn get_current_permissions(&self) -> Result<Permissions>;
-}
-
-provide_kindflag_path!(pub IsRef, "ref.is_ref");
-
-impl Ref for Entry {
-
- /// Check whether the underlying object is actually a ref
- fn is_ref(&self) -> Result<bool> {
- self.is::<IsRef>().map_err(From::from)
- }
-
- /// Get the hash from the path of the ref
- fn get_path_hash(&self) -> Result<String> {
- self.get_location()
- .clone()
- .into_pathbuf()
- .map_err(From::from)
- .and_then(|pb| {
- pb.file_name()
- .and_then(|osstr| osstr.to_str())
- .and_then(|s| s.split("~").next())
- .map(String::from)
- .ok_or(String::from("String splitting error"))
- .map_err(From::from)
- })
- }
-
- /// Get the hash of the link target which is stored in the ref object
- fn get_stored_hash(&self) -> Result<String> {
- self.get_stored_hash_with_hasher(&DefaultHasher::new())
- }
-
- /// Get the hahs of the link target which is stored in the ref object, which is hashed with a
- /// custom Hasher instance.
- fn get_stored_hash_with_hasher<H: Hasher>(&self, h: &H) -> Result<String> {
- self.get_header()
- .read_string(&format!("ref.content_hash.{}", h.hash_name())[..])?
- .ok_or(RE::from_kind(REK::HeaderFieldMissingError))
- }
-
- /// Get the hash of the link target by reading the link target and hashing the contents
- fn get_current_hash(&self) -> Result<String> {
- self.get_current_hash_with_hasher(DefaultHasher::new())
- }
+ /// Get the referenced path.
+ ///
+ /// Does not need a `UniqueRefPathGenerator` as it reads the path stored in the header.
+ fn get_path(&self) -> Result<PathBuf>;
- /// Get the hash of the link target by reading the link target and hashing the contents with the
- /// custom hasher
- fn get_current_hash_with_hasher<H: Hasher>(&self, mut h: H) -> Result<String> {
- self.fs_file()
- .and_then(|pb| File::open(pb.clone()).map(|f| (pb, f)).map_err(From::from))
- .and_then(|(path, mut file)| h.create_hash(&path, &mut file))
- }
+ /// Check whether the referenced file still matches its hash
+ fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> RResult<bool, RPG::Error>;
- /// check whether the pointer the Ref represents still points to a file which exists
- fn fs_link_exists(&self) -> Result<bool> {
- self.fs_file().map(|pathbuf| pathbuf.exists())
- }
+ fn remove_ref(&mut self) -> Result<()>;
/// Alias for `r.fs_link_exists() && r.deref().is_file()`
fn is_ref_to_file(&self) -> Result<bool> {
- self.fs_file().map(|pathbuf| pathbuf.is_file())
+ self.get_path().map(|p| p.is_file())
}
/// Alias for `r.fs_link_exists() && r.deref().is_dir()`
fn is_ref_to_dir(&self) -> Result<bool> {
- self.fs_file().map(|pathbuf| pathbuf.is_dir())
+ self.get_path().map(|p| p.is_dir())
}
/// Alias for `!Ref::fs_link_exists()`
fn is_dangling(&self) -> Result<bool> {
- self.fs_link_exists().map(|b| !b)
+ self.get_path().map(|p| !p.exists())
}
- /// check whether the pointer the Ref represents is valid
- /// This includes:
- /// - Hashsum of the file is still the same as stored in the Ref
- /// - file permissions are still valid
- fn fs_link_valid(&self) -> Result<bool> {
- match (self.fs_link_valid_permissions(), self.fs_link_valid_hash()) {
- (Ok(true) , Ok(true)) => Ok(true),
- (Ok(_) , Ok(_)) => Ok(false),
- (Err(e) , _) => Err(e),
- (_ , Err(e)) => Err(e),
- }
- }
+}
- /// Check whether the file permissions of the referenced file are equal to the stored
- /// permissions
- fn fs_link_valid_permissions(&self) -> Result<bool> {
- self
- .get_header()
- .read_bool("ref.permissions.ro")
- .chain_err(|| REK::HeaderFieldReadError)?
- .ok_or(RE::from_kind(REK::HeaderFieldMissingError))
- .and_then(|ro| self.get_current_permissions().map(|perm| ro == perm.readonly()))
- .chain_err(|| REK::RefTargetCannotReadPermissions)
- }
+provide_kindflag_path!(pub IsRef, "ref.is_ref");
- /// Check whether the Hashsum of the referenced file is equal to the stored hashsum
- fn fs_link_valid_hash(&self) -> Result<bool> {
- let stored_hash = self.get_stored_hash()?;
- let current_hash = self.get_current_hash()?;
- Ok(stored_hash == current_hash)
- }
+impl Ref for Entry {
- /// Update the Ref by re-checking the file from FS
- /// This errors if the file is not present or cannot be read()
- fn update_ref(&mut self) -> Result<()> {
- self.update_ref_with_hasher(&DefaultHasher::new())
+ /// Check whether the underlying object is actually a ref
+ fn is_ref(&self) -> Result<bool> {
+ self.is::<IsRef>().map_err(From::from)
}
- /// Update the Ref by re-checking the file from FS using the passed Hasher instance
- /// This errors if the file is not present or cannot be read()
- fn update_ref_with_hasher<H: Hasher>(&mut self, h: &H) -> Result<()> {
- let current_hash = self.get_current_hash()?; // uses the default hasher
- let current_perm = self.get_current_permissions()?;
-
- self
- .get_header_mut()
- .set("ref.permissions.ro", Value::Boolean(current_perm.readonly()))
- ?;
-
- self
- .get_header_mut()
- .set(&format!("ref.content_hash.{}", h.hash_name())[..], Value::String(current_hash))
- ?;
-
- Ok(())
+ fn get_hash(&self) -> Result<&str> {
+ self.get_header()
+ .read("ref.hash")
+ .map_err(RE::from)?
+ .ok_or_else(|| REK::HeaderFieldMissingError("ref.hash").into())
+ .and_then(|v| v.as_str().ok_or_else(|| REK::HeaderTypeError("ref.hash", "string").into()))
}
- /// Get the path of the file which is reffered to by this Ref
- fn fs_file(&self) -> Result<PathBuf> {
+ fn get_path(&self) -> Result<PathBuf> {
self.get_header()
- .read_string("ref.path")?
- .ok_or(RE::from_kind(REK::HeaderFieldMissingError))
+ .read("ref.path")
+ .map_err(RE::from)?
+ .ok_or_else(|| REK::HeaderFieldMissingError("ref.path").into())
+ .and_then(|v| v.as_str().ok_or_else(|| REK::HeaderTypeError("ref.path", "string").into()))
.map(PathBuf::from)
}
- /// Re-find a referenced file
- ///
- /// This function tries to re-find a ref by searching all directories in `search_roots` recursively
- /// for a file which matches the hash of the Ref.
- ///
- /// If `search_roots` is `None`, it starts at the filesystem root `/`.
- ///
- /// If the target cannot be found, this yields a RefTargetDoesNotExist error kind.
- ///
- /// # Warning
- ///
- /// This option causes heavy I/O as it recursively searches the Filesystem.
- fn refind(&self, search_roots: Option<Vec<PathBuf>>) -> Result<PathBuf> {
- self.refind_with_hasher(search_roots, DefaultHasher::new())
- }
-
- /// See documentation of `Ref::refind()`
- fn refind_with_hasher<H: Hasher>(&self, search_roots: Option<Vec<PathBuf>>, mut h: H)
- -> Result<PathBuf>
- {
- use itertools::Itertools;
- use walkdir::WalkDir;
-
- self.get_stored_hash()
- .and_then(|stored_hash| {
- search_roots
- .unwrap_or(vec![PathBuf::from("/")])
- .into_iter()
- .map(|root| {
- WalkDir::new(root)
- .follow_links(false)
- .into_iter()
- .map(|entry| {
- entry
- .map_err(From::from)
- .and_then(|entry| {
- let pb = PathBuf::from(entry.path());
- File::open(entry.path())
- .map(|f| (pb, f))
- .map_err(From::from)
- })
- .and_then(|(p, mut f)| {
- h.create_hash(&p, &mut f)
- .map(|h| (p, h))
- .map_err(From::from)
- })
- .map(|(path, hash)| {
- if hash == stored_hash {
- Some(path)
- } else {
- None
- }
- })
- })
- .filter_map(Result::ok)
- .filter_map(|e| e)
- .next()
- })
- .flatten()
- .next()
- .ok_or(RE::from_kind(REK::RefTargetDoesNotExist))
- })
+ fn hash_valid<RPG: UniqueRefPathGenerator>(&self) -> RResult<bool, RPG::Error> {
+ self.get_path()
+ .map(PathBuf::from)
+ .map_err(RE::from)
+ .map_err(RPG::Error::from)
+ .and_then(|pb| RPG::unique_hash(pb))
+ .and_then(|h| Ok(h == self.get_hash()?))
}
- /// Get the permissions of the file which are present
- fn get_current_permissions(&self) -> Result<Permissions> {
- self.fs_file()
- .and_then(|pb| {
- File::open(pb)
- .chain_err(|| REK::HeaderFieldReadError)
- })
- .and_then(|file| {
- file
- .metadata()
- .map(|md| md.permissions())
- .chain_err(|| REK::RefTargetCannotReadPermissions)
- })
+ fn remove_ref(&mut self) -> Result<()> {
+ let hdr = self.get_header_mut();
+ let _ = hdr.delete("ref.hash")?;
+ let _ = hdr.delete("ref.path")?;
+ let _ = hdr.delete("ref")?;
+ Ok(())
}
}
diff --git a/lib/entry/libimagentryref/src/refstore.rs b/lib/entry/libimagentryref/src/refstore.rs
index 7248bbe..c428efb 100644
--- a/lib/entry/libimagentryref/src/refstore.rs
+++ b/lib/entry/libimagentryref/src/refstore.rs
@@ -17,280 +17,125 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
+use std::path::Path;
use std::path::PathBuf;
-use std::collections::BTreeMap;
-use std::fs::File;
use libimagstore::store::FileLockEntry;
-use libimagstore::storeid::IntoStoreId;
-use libimagstore::storeid::StoreId;
-use libimagstore::storeid::StoreIdIterator;
use libimagstore::store::Store;
+use libimagstore::storeid::StoreId;
use libimagentryutil::isa::Is;
+use toml_query::insert::TomlValueInsertExt;
use toml::Value;
-use error::RefErrorKind as REK;
use error::RefError as RE;
-use error::ResultExt;
-use error::Result;
-use flags::RefFlags;
+use error::RefErrorKind as REK;
use reference::IsRef;
-use hasher::*;
-use module_path::ModuleEntryPath;
-use util::*;
-
-pub trait RefStore {
-
- /// Check whether there is a reference to the file at `pb`
- fn exists(&self, pb: PathBuf) -> Result<bool>;
-
- /// Get a Ref object from the store by hash.
- ///
- /// Returns None if the hash cannot be found.
- fn get_by_hash<'a>(&'a self, hash: String) -> Result<Option<FileLockEntry<'a>>>;
- /// Find a store id by partial ref (also see documentation for
- /// `RefStore::get_by_partitial_hash()`.
- fn find_storeid_by_partial_hash(&self, hash: &String) -> Result<Option<StoreId>>;
+/// A UniqueRefPathGenerator generates unique Pathes
+///
+/// It is basically a functor which generates a StoreId from a &Path.
+/// For more information have a look at the documentation of RefStore.
+pub trait UniqueRefPathGenerator {
+ type Error: From<RE>;
- /// Get a Ref object from the store by (eventually partial) hash.
- ///
- /// If the hash is complete, `RefStore::get_by_hash()` should be used as it is cheaper.
- /// If the hash comes from user input and thus might be abbreviated, this function can be used.
- fn get_by_partitial_hash<'a>(&'a self, hash: &String) -> Result<Option<FileLockEntry<'a>>>;
-
- /// Delete a ref by hash
- ///
- /// If the returned Result contains an error, the ref might not be deleted.
- fn delete_by_hash(&self, hash: String) -> Result<()>;
+ /// The collection the `StoreId` should be created for
+ fn collection() -> &'static str {
+ "ref"
+ }
- /// Create a Ref object which refers to `pb`
- fn create<'a>(&'a self, pb: PathBuf, flags: RefFlags) -> Result<FileLockEntry<'a>>;
+ /// A function which should generate a unique string for a Path
+ fn unique_hash<A: AsRef<Path>>(path: A) -> Result<String, Self::Error>;
- fn create_with_hasher<'a, H: Hasher>(&'a self, pb: PathBuf, flags: RefFlags, h: H)
- -> Result<FileLockEntry<'a>>;
+ /// Postprocess the generated `StoreId` object
+ fn postprocess_storeid(sid: StoreId) -> Result<StoreId, Self::Error> {
+ Ok(sid)
+ }
+}
- /// Get all reference objects
- fn all_references(&self) -> Result<StoreIdIterator>;
+/// A extensions for the `Store` to handle `Ref` objects
+///
+/// The RefStore handles refs using a `UniqueRefPathGenerator`. The `UniqueRefPathGenerator`, as it
+/// name suggests, generates unique `StoreId`s for a `&Path`. It is a functor `&Path -> StoreId`.
+///
+/// It provides three functions which are called in the following sequence:
+///
+/// * The `UniqueRefPathGenerator::collection()` function is used for get the collection a `StoreId`
+/// should be in (The first element of the `StoreId` path)
+/// * The `UniqueRefPathGenerator::unique_hash()` gets the `&Path` which it then should generate a
+/// unique String for. How this is done does not matter. It can hash the Path itself, read the
+/// file and hash that or something else. It should be reproduceable, though.
+/// * These two parts are joined and put into a `StoreId` which the
+/// `UniqueRefPathGenerator::postprocess_storeid()` function is then allowed to postprocess (for
+/// example add more parts to the StoreId). The default implementation does nothing.
+///
+/// The StoreId which is generated is then used to carry out the actual action (reading, creating
+/// ...).
+/// If a entry is created, header information is set (that it is a ref, the hash which was just
+/// generated and the path of the referenced file)
+///
+/// # Details
+///
+/// The `UniqueRefPathGenerator` is passed as type parameter to enforce some semantics:
+///
+/// * The used `UniqueRefPathGenerator` is defined by the implementation rather than by the runtime
+/// of the program or some environment. Of course this is only a small hurdle to enforce this, but
+/// a hint.
+/// * The `UniqueRefPathGenerator` is a functor which does not carry state.
+///
+pub trait RefStore<'a> {
+
+ fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H) -> Result<Option<FileLockEntry<'a>>, RPG::Error>;
+ fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>, RPG::Error>;
+ fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A) -> Result<FileLockEntry<'a>, RPG::Error>;
}
-impl RefStore for Store {
-
- /// Check whether there is a reference to the file at `pb`
- fn exists(&self, pb: PathBuf) -> Result<bool> {
- pb.canonicalize()
- .chain_err(|| REK::PathCanonicalizationError)
- .and_then(|c| hash_path(&c))
- .chain_err(|| REK::PathHashingError)
- .and_then(|hash| {
- self.retrieve_for_module("ref").map(|iter| (hash, iter)).map_err(From::from)
- })
- .and_then(|(hash, possible_refs)| {
- // This is kind of a manual Iterator::filter() call what we do here, but with the
- // actual ::filter method we cannot return the error in a nice way, so we do it
- // manually here. If you can come up with a better version of this, feel free to
- // take this note as a todo.
- for r in possible_refs {
- let contains_hash = r.to_str()
- .chain_err(|| REK::TypeConversionError)
- .map(|s| s.contains(&hash[..]))
- ?;
+impl<'a> RefStore<'a> for Store {
- if !contains_hash {
- continue;
- }
+ fn get_ref<RPG: UniqueRefPathGenerator, H: AsRef<str>>(&'a self, hash: H)
+ -> Result<Option<FileLockEntry<'a>>, RPG::Error>
+ {
+ let sid = StoreId::new_baseless(PathBuf::from(format!("{}/{}", RPG::collection(), hash.as_ref())))
+ .map_err(RE::from)?;
- match self.get(r.clone())? {
- Some(fle) => {
- if read_reference(&fle).map(|path| path == pb).unwrap_or(false) {
- return Ok(true)
- }
- },
+ debug!("Getting: {:?}", sid);
+ self.get(sid)
+ .map_err(RE::from)
+ .map_err(RPG::Error::from)
+ }
- None => {
- let e = format!("Failed to get from store: {}", r);
- return Err(e).map_err(From::from)
- },
- }
+ fn create_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
+ -> Result<FileLockEntry<'a>, RPG::Error>
+ {
+ let path_str = path.as_ref().to_str().map(String::from).ok_or(REK::PathUTF8Error.into())?;
+ let hash = RPG::unique_hash(path)?;
+ let pathbuf = PathBuf::from(format!("{}/{}", RPG::collection(), hash));
+ let sid = StoreId::new_baseless(pathbuf).map_err(RE::from)?;
+
+ debug!("Creating: {:?}", sid);
+ self.create(sid)
+ .map_err(RE::from)
+ .and_then(|mut fle| {
+ let _ = fle.set_isflag::<IsRef>()?;
+ {
+ let hdr = fle.get_header_mut();
+ hdr.insert("ref.path", Value::String(String::from(path_str)))?;
+ hdr.insert("ref.hash", Value::String(hash))?;
}
-
- Ok(false)
+ Ok(fle)
})
+ .map_err(RPG::Error::from)
}
- /// Get a Ref object from the store by hash.
- ///
- /// Returns None if the hash cannot be found.
- fn get_by_hash<'a>(&'a self, hash: String) -> Result<Option<FileLockEntry<'a>>> {
- ModuleEntryPath::new(hash)
- .into_storeid()
- .and_then(|id| self.get(id))
- .map_err(From::from)
- }
-
- fn find_storeid_by_partial_hash(&self, hash: &String) -> Result<Option<StoreId>> {
- debug!("Trying to find '{}' in store...", hash);
- for id in self.retrieve_for_module("ref")? {
- let components_have_hash = id
- .components()
- .any(|c| c.as_os_str().to_str().map(|s| s.contains(hash)).unwrap_or(false));
-
- if components_have_hash {
- debug!("Found hash '{}' in {:?}", hash, id);
- return Ok(Some(id))
- }
- }
- Ok(None)
- }
-
- /// Get a Ref object from the store by (eventually partial) hash.
- ///
- /// If the hash is complete, `RefStore::get_by_hash()` should be used as it is cheaper.
- /// If the hash comes from user input and thus might be abbreviated, this function can be used.
- fn get_by_partitial_hash<'a>(&'a self, hash: &String) -> Result<Option<FileLockEntry<'a>>> {
- match self.find_storeid_by_partial_hash(hash)? {
- Some(id) => self.get(id).map_err(From::from),
- None => Ok(None),
- }
- }
-
- /// Delete a ref by hash
- ///
- /// If the returned Result contains an error, the ref might not be deleted.
- fn delete_by_hash(&self, hash: String) -> Result<()> {
- ModuleEntryPath::new(hash)
- .into_storeid()
- .and_then(|id| self.delete(id))
- .map_err(From::from)
- }
-
- /// Create a Ref object which refers to `pb`
- fn create<'a>(&'a self, pb: PathBuf, flags: RefFlags) -> Result<FileLockEntry<'a>> {
- self.create_with_hasher(pb, flags, DefaultHasher::new())
- }
-
- fn create_with_hasher<'a, H: Hasher>(&'a self, pb: PathBuf, flags: RefFlags, mut h: H)
- -> Result<FileLockEntry<'a>>
+ fn retrieve_ref<RPG: UniqueRefPathGenerator, A: AsRef<Path>>(&'a self, path: A)
+ -> Result<FileLockEntry<'a>, RPG::Error>
{
- use toml_query::insert::TomlValueInsertExt;
-
- if !pb.exists() {
- return Err(RE::from_kind(REK::RefTargetDoesNotExist));
- }
- if flags.get_content_hashing() && pb.is_dir() {
- return Err(RE::from_kind(REK::RefTargetCannotBeHashed));
+ match self.get_ref::<RPG, String>(RPG::unique_hash(path.as_ref())?)? {
+ Some(r) => Ok(r),
+ None => self.create_ref::<RPG, A>(path),
}
-
- let (mut fle, content_hash, permissions, canonical_path) = { // scope to be able to fold
- File::open(pb.clone())
- .chain_err(|| REK::RefTargetFileCannotBeOpened)
-
- // If we were able to open this file,
- // we hash the contents of the file and return (file, hash)
- .and_then(|mut file| {
- let opt_contenthash = if flags.get_content_hashing() {
- Some(h.create_hash(&pb, &mut file)?)
- } else {
- None
- };
-
- Ok((file, opt_contenthash))
- })
-
- // and then we get the permissions if we have to
- // and return (file, content hash, permissions)
- .and_then(|(file, opt_contenthash)| {
- let opt_permissions = if flags.get_permission_tracking() {
- Some(file.metadata()
- .map(|md| md.permissions())
- .chain_err(|| REK::RefTargetCannotReadPermissions)?)
- } else {
- None
- };
-
- Ok((opt_contenthash, opt_permissions))
- })
-
- // and then we try to canonicalize the PathBuf, because we want to store a
- // canonicalized path
- // and return (file, content hash, permissions, canonicalized path)
- .and_then(|(opt_contenthash, opt_permissions)| {
- pb.canonicalize()
- .map(|can| (opt_contenthash, opt_permissions, can))
- // if PathBuf::canonicalize() failed, build an error from the return value
- .chain_err(|| REK::PathCanonicalizationError)
- })
-
- // and then we hash the canonicalized path
- // and return (file, content hash, permissions, canonicalized path, path hash)
- .and_then(|(opt_contenthash, opt_permissions, can)| {
- let path_hash = hash_path(&can).chain_err(|| REK::PathHashingError)?;
-
- Ok((opt_contenthash, opt_permissions, can, path_hash))
- })
-
- // and then we convert the PathBuf of the canonicalized path to a String to be able
- // to save it in the Ref FileLockEntry obj
- // and return
- // (file, content hash, permissions, canonicalized path as String, path hash)
- .and_then(|(opt_conhash, opt_perm, can, path_hash)| {
- match can.to_str().map(String::from) {
- // UTF convert error in PathBuf::to_str(),
- None => Err(RE::from_kind(REK::PathUTF8Error)),
- Some(can) => Ok((opt_conhash, opt_perm, can, path_hash))
- }
- })
-
- // and then we create the FileLockEntry in the Store
- // and return (filelockentry, content hash, permissions, canonicalized path)
- .and_then(|(opt_conhash, opt_perm, can, path_hash)| {
- let fle = self.create(ModuleEntryPath::new(path_hash))?;
- Ok((fle, opt_conhash, opt_perm, can))
- })?
- };
-
- for tpl in [
- Some((String::from("ref"), Value::Table(BTreeMap::new()))),
- Some((String::from("ref.permissions"), Value::Table(BTreeMap::new()))),
- Some((String::from("ref.path"), Value::String(canonical_path))),
- Some((String::from("ref.content_hash"), Value::Table(BTreeMap::new()))),
-
- content_hash.map(|hash| {
- (format!("ref.content_hash.{}", h.hash_name()), Value::String(hash))
- }),
- permissions.map(|p| {
- (String::from("ref.permissions.ro"), Value::Boolean(p.readonly()))
- }),
- ].into_iter()
- {
- match tpl {
- &Some((ref s, ref v)) => {
- match fle.get_header_mut().insert(s, v.clone()) {
- Ok(Some(_)) => {
- let e = RE::from_kind(REK::HeaderFieldAlreadyExistsError);
- return Err(e).chain_err(|| REK::HeaderFieldWriteError);
- },
- Ok(None) => {
- // Okay, we just inserted a new header value...
- },
- Err(e) => return Err(e).chain_err(|| REK::HeaderFieldWriteError),
- }
- }
- &None => {
- debug!("Not going to insert.");
- }
- }
- }
-
- let _ = fle.set_isflag::<IsRef>()?;
-
- Ok(fle)
}
- fn all_references(&self) -> Result<StoreIdIterator> {
- self.retrieve_for_module("ref").map_err(From::from)
- }
}
diff --git a/lib/entry/libimagentryref/src/util.rs b/lib/entry/libimagentryref/src/util.rs
deleted file mode 100644
index 878b98c..0000000
--- a/lib/entry/libimagentryref/src/util.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-//
-// imag - the personal information management suite for the commandline
-// Copyright (C) 2015-2018 Matthias Beyer <mail@beyermatthias.de> and contributors
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; version
-// 2.1 of the License.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public
-// License along with this library; if not, write to the Free Software
-// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-//
-
-use std::path::PathBuf;
-
-use error::RefErrorKind as REK;
-use error::RefError as RE;
-use error::Result;
-
-use libimagstore::store::Entry;
-
-use toml_query::read::TomlValueReadTypeExt;
-
-/// Creates a Hash from a PathBuf by making the PathBuf absolute and then running a hash
-/// algorithm on it
-pub fn hash_path(pb: &PathBuf) -> Result<String> {
- use crypto::sha1::Sha1;
- use crypto::digest::Digest;
-
- pb.to_str()
- .ok_or(RE::from_kind(REK::PathUTF8Error))
- .map(|s| {
- let mut hasher = Sha1::new();
- hasher.input_str(s);
- hasher.result_str()
- })
-}
-
-/// Read the reference from a file
-pub fn read_reference(refentry: &Entry) -> Result<PathBuf> {
- refentry.get_header()
- .read_string("ref.path")?
- .ok_or(RE::from_kind(REK::HeaderFieldMissingError))
- .map(PathBuf::from)
-}
-