515 lines
22 KiB
Rust
515 lines
22 KiB
Rust
//! This modules abstracts data sources and merges them in a single virtual one
|
|
//!
|
|
//! It is also responsible for syncing them together
|
|
|
|
use std::error::Error;
|
|
use std::collections::HashSet;
|
|
use std::marker::PhantomData;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::fmt::{Display, Formatter};
|
|
|
|
use url::Url;
|
|
use itertools::Itertools;
|
|
|
|
use crate::traits::{BaseCalendar, CalDavSource, DavCalendar};
|
|
use crate::traits::CompleteCalendar;
|
|
use crate::item::SyncStatus;
|
|
|
|
pub mod sync_progress;
|
|
use sync_progress::SyncProgress;
|
|
use sync_progress::{FeedbackSender, SyncEvent};
|
|
|
|
/// How many items will be batched in a single HTTP request when downloading from the server
|
|
#[cfg(not(test))]
|
|
const DOWNLOAD_BATCH_SIZE: usize = 30;
|
|
/// How many items will be batched in a single HTTP request when downloading from the server
|
|
#[cfg(test)]
|
|
const DOWNLOAD_BATCH_SIZE: usize = 3;
|
|
|
|
// I am too lazy to actually make `fetch_and_apply` generic over an async closure.
|
|
// Let's work around by passing an enum, so that `fetch_and_apply` will know what to do
|
|
enum BatchDownloadType {
|
|
RemoteAdditions,
|
|
RemoteChanges,
|
|
}
|
|
|
|
impl Display for BatchDownloadType {
|
|
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
|
|
match self {
|
|
Self::RemoteAdditions => write!(f, "remote additions"),
|
|
Self::RemoteChanges => write!(f, "remote changes"),
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
/// A data source that combines two `CalDavSource`s, which is able to sync both sources.
|
|
///
|
|
/// Usually, you will only need to use a provider between a server and a local cache, that is to say a [`CalDavProvider`](crate::CalDavProvider), i.e. a `Provider<Cache, CachedCalendar, Client, RemoteCalendar>`. \
|
|
/// However, providers can be used for integration tests, where the remote source is mocked by a `Cache`.
|
|
#[derive(Debug)]
|
|
pub struct Provider<L, T, R, U>
|
|
where
|
|
L: CalDavSource<T>,
|
|
T: CompleteCalendar + Sync + Send,
|
|
R: CalDavSource<U>,
|
|
U: DavCalendar + Sync + Send,
|
|
{
|
|
/// The remote source (usually a server)
|
|
remote: R,
|
|
/// The local cache
|
|
local: L,
|
|
|
|
phantom_t: PhantomData<T>,
|
|
phantom_u: PhantomData<U>,
|
|
}
|
|
|
|
impl<L, T, R, U> Provider<L, T, R, U>
|
|
where
|
|
L: CalDavSource<T>,
|
|
T: CompleteCalendar + Sync + Send,
|
|
R: CalDavSource<U>,
|
|
U: DavCalendar + Sync + Send,
|
|
{
|
|
/// Create a provider.
|
|
///
|
|
/// `remote` is usually a [`Client`](crate::client::Client), `local` is usually a [`Cache`](crate::cache::Cache).
|
|
/// However, both can be interchangeable. The only difference is that `remote` always wins in case of a sync conflict
|
|
pub fn new(remote: R, local: L) -> Self {
|
|
Self { remote, local,
|
|
phantom_t: PhantomData, phantom_u: PhantomData,
|
|
}
|
|
}
|
|
|
|
/// Returns the data source described as `local`
|
|
pub fn local(&self) -> &L { &self.local }
|
|
/// Returns the data source described as `local`
|
|
pub fn local_mut(&mut self) -> &mut L { &mut self.local }
|
|
/// Returns the data source described as `remote`.
|
|
///
|
|
/// Apart from tests, there are very few (if any) reasons to access `remote` directly.
|
|
/// Usually, you should rather use the `local` source, which (usually) is a much faster local cache.
|
|
/// To be sure `local` accurately mirrors the `remote` source, you can run [`Provider::sync`]
|
|
pub fn remote(&self) -> &R { &self.remote }
|
|
|
|
/// Performs a synchronisation between `local` and `remote`, and provide feeedback to the user about the progress.
|
|
///
|
|
/// This bidirectional sync applies additions/deletions made on a source to the other source.
|
|
/// In case of conflicts (the same item has been modified on both ends since the last sync, `remote` always wins).
|
|
///
|
|
/// It returns whether the sync was totally successful (details about errors are logged using the `log::*` macros).
|
|
/// In case errors happened, the sync might have been partially executed but your data will never be correupted (either locally nor in the server).
|
|
/// Simply run this function again, it will re-start a sync, picking up where it failed.
|
|
pub async fn sync_with_feedback(&mut self, feedback_sender: FeedbackSender) -> bool {
|
|
let mut progress = SyncProgress::new_with_feedback_channel(feedback_sender);
|
|
self.run_sync(&mut progress).await
|
|
}
|
|
|
|
/// Performs a synchronisation between `local` and `remote`, without giving any feedback.
|
|
///
|
|
/// See [`Self::sync_with_feedback`]
|
|
pub async fn sync(&mut self) -> bool {
|
|
let mut progress = SyncProgress::new();
|
|
self.run_sync(&mut progress).await
|
|
}
|
|
|
|
async fn run_sync(&mut self, progress: &mut SyncProgress) -> bool {
|
|
if let Err(err) = self.run_sync_inner(progress).await {
|
|
progress.error(&format!("Sync terminated because of an error: {}", err));
|
|
}
|
|
progress.feedback(SyncEvent::Finished{ success: progress.is_success() });
|
|
progress.is_success()
|
|
}
|
|
|
|
async fn run_sync_inner(&mut self, progress: &mut SyncProgress) -> Result<(), Box<dyn Error>> {
|
|
progress.info("Starting a sync.");
|
|
progress.feedback(SyncEvent::Started);
|
|
|
|
let mut handled_calendars = HashSet::new();
|
|
|
|
// Sync every remote calendar
|
|
let cals_remote = self.remote.get_calendars().await?;
|
|
for (cal_url, cal_remote) in cals_remote {
|
|
let counterpart = match self.get_or_insert_local_counterpart_calendar(&cal_url, cal_remote.clone()).await {
|
|
Err(err) => {
|
|
progress.warn(&format!("Unable to get or insert local counterpart calendar for {} ({}). Skipping this time", cal_url, err));
|
|
continue;
|
|
},
|
|
Ok(arc) => arc,
|
|
};
|
|
|
|
if let Err(err) = Self::sync_calendar_pair(counterpart, cal_remote, progress).await {
|
|
progress.warn(&format!("Unable to sync calendar {}: {}, skipping this time.", cal_url, err));
|
|
continue;
|
|
}
|
|
handled_calendars.insert(cal_url);
|
|
}
|
|
|
|
// Sync every local calendar that would not be in the remote yet
|
|
let cals_local = self.local.get_calendars().await?;
|
|
for (cal_url, cal_local) in cals_local {
|
|
if handled_calendars.contains(&cal_url) {
|
|
continue;
|
|
}
|
|
|
|
let counterpart = match self.get_or_insert_remote_counterpart_calendar(&cal_url, cal_local.clone()).await {
|
|
Err(err) => {
|
|
progress.warn(&format!("Unable to get or insert remote counterpart calendar for {} ({}). Skipping this time", cal_url, err));
|
|
continue;
|
|
},
|
|
Ok(arc) => arc,
|
|
};
|
|
|
|
if let Err(err) = Self::sync_calendar_pair(cal_local, counterpart, progress).await {
|
|
progress.warn(&format!("Unable to sync calendar {}: {}, skipping this time.", cal_url, err));
|
|
continue;
|
|
}
|
|
}
|
|
|
|
progress.info("Sync ended");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
|
|
async fn get_or_insert_local_counterpart_calendar(&mut self, cal_url: &Url, needle: Arc<Mutex<U>>) -> Result<Arc<Mutex<T>>, Box<dyn Error>> {
|
|
get_or_insert_counterpart_calendar("local", &mut self.local, cal_url, needle).await
|
|
}
|
|
async fn get_or_insert_remote_counterpart_calendar(&mut self, cal_url: &Url, needle: Arc<Mutex<T>>) -> Result<Arc<Mutex<U>>, Box<dyn Error>> {
|
|
get_or_insert_counterpart_calendar("remote", &mut self.remote, cal_url, needle).await
|
|
}
|
|
|
|
|
|
async fn sync_calendar_pair(cal_local: Arc<Mutex<T>>, cal_remote: Arc<Mutex<U>>, progress: &mut SyncProgress) -> Result<(), Box<dyn Error>> {
|
|
let mut cal_remote = cal_remote.lock().unwrap();
|
|
let mut cal_local = cal_local.lock().unwrap();
|
|
let cal_name = cal_local.name().to_string();
|
|
|
|
progress.info(&format!("Syncing calendar {}", cal_name));
|
|
progress.reset_counter();
|
|
progress.feedback(SyncEvent::InProgress{
|
|
calendar: cal_name.clone(),
|
|
items_done_already: 0,
|
|
details: "started".to_string()
|
|
});
|
|
|
|
// Step 1 - find the differences
|
|
progress.debug("Finding the differences to sync...");
|
|
let mut local_del = HashSet::new();
|
|
let mut remote_del = HashSet::new();
|
|
let mut local_changes = HashSet::new();
|
|
let mut remote_changes = HashSet::new();
|
|
let mut local_additions = HashSet::new();
|
|
let mut remote_additions = HashSet::new();
|
|
|
|
let remote_items = cal_remote.get_item_version_tags().await?;
|
|
progress.feedback(SyncEvent::InProgress{
|
|
calendar: cal_name.clone(),
|
|
items_done_already: 0,
|
|
details: format!("{} remote items", remote_items.len()),
|
|
});
|
|
|
|
let mut local_items_to_handle = cal_local.get_item_urls().await?;
|
|
for (url, remote_tag) in remote_items {
|
|
progress.trace(&format!("***** Considering remote item {}...", url));
|
|
match cal_local.get_item_by_url(&url).await {
|
|
None => {
|
|
// This was created on the remote
|
|
progress.debug(&format!("* {} is a remote addition", url));
|
|
remote_additions.insert(url);
|
|
},
|
|
Some(local_item) => {
|
|
if local_items_to_handle.remove(&url) == false {
|
|
progress.error(&format!("Inconsistent state: missing task {} from the local tasks", url));
|
|
}
|
|
|
|
match local_item.sync_status() {
|
|
SyncStatus::NotSynced => {
|
|
progress.error(&format!("URL reuse between remote and local sources ({}). Ignoring this item in the sync", url));
|
|
continue;
|
|
},
|
|
SyncStatus::Synced(local_tag) => {
|
|
if &remote_tag != local_tag {
|
|
// This has been modified on the remote
|
|
progress.debug(&format!("* {} is a remote change", url));
|
|
remote_changes.insert(url);
|
|
}
|
|
},
|
|
SyncStatus::LocallyModified(local_tag) => {
|
|
if &remote_tag == local_tag {
|
|
// This has been changed locally
|
|
progress.debug(&format!("* {} is a local change", url));
|
|
local_changes.insert(url);
|
|
} else {
|
|
progress.info(&format!("Conflict: task {} has been modified in both sources. Using the remote version.", url));
|
|
progress.debug(&format!("* {} is considered a remote change", url));
|
|
remote_changes.insert(url);
|
|
}
|
|
},
|
|
SyncStatus::LocallyDeleted(local_tag) => {
|
|
if &remote_tag == local_tag {
|
|
// This has been locally deleted
|
|
progress.debug(&format!("* {} is a local deletion", url));
|
|
local_del.insert(url);
|
|
} else {
|
|
progress.info(&format!("Conflict: task {} has been locally deleted and remotely modified. Reverting to the remote version.", url));
|
|
progress.debug(&format!("* {} is a considered a remote change", url));
|
|
remote_changes.insert(url);
|
|
}
|
|
},
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Also iterate on the local tasks that are not on the remote
|
|
for url in local_items_to_handle {
|
|
progress.trace(&format!("##### Considering local item {}...", url));
|
|
let local_item = match cal_local.get_item_by_url(&url).await {
|
|
None => {
|
|
progress.error(&format!("Inconsistent state: missing task {} from the local tasks", url));
|
|
continue;
|
|
},
|
|
Some(item) => item,
|
|
};
|
|
|
|
match local_item.sync_status() {
|
|
SyncStatus::Synced(_) => {
|
|
// This item has been removed from the remote
|
|
progress.debug(&format!("# {} is a deletion from the server", url));
|
|
remote_del.insert(url);
|
|
},
|
|
SyncStatus::NotSynced => {
|
|
// This item has just been locally created
|
|
progress.debug(&format!("# {} has been locally created", url));
|
|
local_additions.insert(url);
|
|
},
|
|
SyncStatus::LocallyDeleted(_) => {
|
|
// This item has been deleted from both sources
|
|
progress.debug(&format!("# {} has been deleted from both sources", url));
|
|
remote_del.insert(url);
|
|
},
|
|
SyncStatus::LocallyModified(_) => {
|
|
progress.info(&format!("Conflict: item {} has been deleted from the server and locally modified. Deleting the local copy", url));
|
|
remote_del.insert(url);
|
|
},
|
|
}
|
|
}
|
|
|
|
|
|
// Step 2 - commit changes
|
|
progress.trace("Committing changes...");
|
|
for url_del in local_del {
|
|
progress.debug(&format!("> Pushing local deletion {} to the server", url_del));
|
|
progress.increment_counter(1);
|
|
progress.feedback(SyncEvent::InProgress{
|
|
calendar: cal_name.clone(),
|
|
items_done_already: progress.counter(),
|
|
details: Self::item_name(&cal_local, &url_del).await,
|
|
});
|
|
|
|
match cal_remote.delete_item(&url_del).await {
|
|
Err(err) => {
|
|
progress.warn(&format!("Unable to delete remote item {}: {}", url_del, err));
|
|
},
|
|
Ok(()) => {
|
|
// Change the local copy from "marked to deletion" to "actually deleted"
|
|
if let Err(err) = cal_local.immediately_delete_item(&url_del).await {
|
|
progress.error(&format!("Unable to permanently delete local item {}: {}", url_del, err));
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
for url_del in remote_del {
|
|
progress.debug(&format!("> Applying remote deletion {} locally", url_del));
|
|
progress.increment_counter(1);
|
|
progress.feedback(SyncEvent::InProgress{
|
|
calendar: cal_name.clone(),
|
|
items_done_already: progress.counter(),
|
|
details: Self::item_name(&cal_local, &url_del).await,
|
|
});
|
|
if let Err(err) = cal_local.immediately_delete_item(&url_del).await {
|
|
progress.warn(&format!("Unable to delete local item {}: {}", url_del, err));
|
|
}
|
|
}
|
|
|
|
Self::apply_remote_additions(
|
|
remote_additions,
|
|
&mut *cal_local,
|
|
&mut *cal_remote,
|
|
progress,
|
|
&cal_name
|
|
).await;
|
|
|
|
Self::apply_remote_changes(
|
|
remote_changes,
|
|
&mut *cal_local,
|
|
&mut *cal_remote,
|
|
progress,
|
|
&cal_name
|
|
).await;
|
|
|
|
|
|
for url_add in local_additions {
|
|
progress.debug(&format!("> Pushing local addition {} to the server", url_add));
|
|
progress.increment_counter(1);
|
|
progress.feedback(SyncEvent::InProgress{
|
|
calendar: cal_name.clone(),
|
|
items_done_already: progress.counter(),
|
|
details: Self::item_name(&cal_local, &url_add).await,
|
|
});
|
|
match cal_local.get_item_by_url_mut(&url_add).await {
|
|
None => {
|
|
progress.error(&format!("Inconsistency: created item {} has been marked for upload but is locally missing", url_add));
|
|
continue;
|
|
},
|
|
Some(item) => {
|
|
match cal_remote.add_item(item.clone()).await {
|
|
Err(err) => progress.error(&format!("Unable to add item {} to remote calendar: {}", url_add, err)),
|
|
Ok(new_ss) => {
|
|
// Update local sync status
|
|
item.set_sync_status(new_ss);
|
|
},
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
for url_change in local_changes {
|
|
progress.debug(&format!("> Pushing local change {} to the server", url_change));
|
|
progress.increment_counter(1);
|
|
progress.feedback(SyncEvent::InProgress{
|
|
calendar: cal_name.clone(),
|
|
items_done_already: progress.counter(),
|
|
details: Self::item_name(&cal_local, &url_change).await,
|
|
});
|
|
match cal_local.get_item_by_url_mut(&url_change).await {
|
|
None => {
|
|
progress.error(&format!("Inconsistency: modified item {} has been marked for upload but is locally missing", url_change));
|
|
continue;
|
|
},
|
|
Some(item) => {
|
|
match cal_remote.update_item(item.clone()).await {
|
|
Err(err) => progress.error(&format!("Unable to update item {} in remote calendar: {}", url_change, err)),
|
|
Ok(new_ss) => {
|
|
// Update local sync status
|
|
item.set_sync_status(new_ss);
|
|
},
|
|
};
|
|
}
|
|
};
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
|
|
async fn item_name(cal: &T, url: &Url) -> String {
|
|
cal.get_item_by_url(url).await.map(|item| item.name()).unwrap_or_default().to_string()
|
|
}
|
|
|
|
async fn apply_remote_additions(
|
|
mut remote_additions: HashSet<Url>,
|
|
cal_local: &mut T,
|
|
cal_remote: &mut U,
|
|
progress: &mut SyncProgress,
|
|
cal_name: &str
|
|
) {
|
|
for batch in remote_additions.drain().chunks(DOWNLOAD_BATCH_SIZE).into_iter() {
|
|
Self::fetch_batch_and_apply(BatchDownloadType::RemoteAdditions, batch, cal_local, cal_remote, progress, cal_name).await;
|
|
}
|
|
}
|
|
|
|
async fn apply_remote_changes(
|
|
mut remote_changes: HashSet<Url>,
|
|
cal_local: &mut T,
|
|
cal_remote: &mut U,
|
|
progress: &mut SyncProgress,
|
|
cal_name: &str
|
|
) {
|
|
for batch in remote_changes.drain().chunks(DOWNLOAD_BATCH_SIZE).into_iter() {
|
|
Self::fetch_batch_and_apply(BatchDownloadType::RemoteChanges, batch, cal_local, cal_remote, progress, cal_name).await;
|
|
}
|
|
}
|
|
|
|
async fn fetch_batch_and_apply<I: Iterator<Item = Url>>(
|
|
batch_type: BatchDownloadType,
|
|
remote_additions: I,
|
|
cal_local: &mut T,
|
|
cal_remote: &mut U,
|
|
progress: &mut SyncProgress,
|
|
cal_name: &str
|
|
) {
|
|
progress.debug(&format!("> Applying a batch of {} locally", batch_type) /* too bad Chunks does not implement ExactSizeIterator, that could provide useful debug info. See https://github.com/rust-itertools/itertools/issues/171 */);
|
|
|
|
let list_of_additions: Vec<Url> = remote_additions.map(|url| url.clone()).collect();
|
|
match cal_remote.get_items_by_url(&list_of_additions).await {
|
|
Err(err) => {
|
|
progress.warn(&format!("Unable to get the batch of {} {:?}: {}. Skipping them.", batch_type, list_of_additions, err));
|
|
},
|
|
Ok(items) => {
|
|
for item in items {
|
|
match item {
|
|
None => {
|
|
progress.error(&format!("Inconsistency: an item from the batch has vanished from the remote end"));
|
|
continue;
|
|
},
|
|
Some(new_item) => {
|
|
let local_update_result = match batch_type {
|
|
BatchDownloadType::RemoteAdditions => cal_local.add_item(new_item.clone()).await,
|
|
BatchDownloadType::RemoteChanges => cal_local.update_item(new_item.clone()).await,
|
|
};
|
|
if let Err(err) = local_update_result {
|
|
progress.error(&format!("Not able to add item {} to local calendar: {}", new_item.url(), err));
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
// Notifying every item at the same time would not make sense. Let's notify only one of them
|
|
let one_item_name = match list_of_additions.get(0) {
|
|
Some(url) => Self::item_name(&cal_local, &url).await,
|
|
None => String::from("<unable to get the name of the first batched item>"),
|
|
};
|
|
progress.increment_counter(list_of_additions.len());
|
|
progress.feedback(SyncEvent::InProgress{
|
|
calendar: cal_name.to_string(),
|
|
items_done_already: progress.counter(),
|
|
details: one_item_name,
|
|
});
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
async fn get_or_insert_counterpart_calendar<H, N, I>(haystack_descr: &str, haystack: &mut H, cal_url: &Url, needle: Arc<Mutex<N>>)
|
|
-> Result<Arc<Mutex<I>>, Box<dyn Error>>
|
|
where
|
|
H: CalDavSource<I>,
|
|
I: BaseCalendar,
|
|
N: BaseCalendar,
|
|
{
|
|
loop {
|
|
if let Some(cal) = haystack.get_calendar(&cal_url).await {
|
|
break Ok(cal);
|
|
}
|
|
|
|
// This calendar does not exist locally yet, let's add it
|
|
log::debug!("Adding a {} calendar {}", haystack_descr, cal_url);
|
|
let src = needle.lock().unwrap();
|
|
let name = src.name().to_string();
|
|
let supported_comps = src.supported_components();
|
|
let color = src.color();
|
|
if let Err(err) = haystack.create_calendar(
|
|
cal_url.clone(),
|
|
name,
|
|
supported_comps,
|
|
color.cloned(),
|
|
).await{
|
|
return Err(err);
|
|
}
|
|
}
|
|
}
|
|
|