fix Error code 3850: I/O error in the advisory file locking layer

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "onehistory"
-version = "0.1.2"
+version = "0.1.3"
 edition = "2021"
 authors = ["Jiacai Liu <jiacai2050+1history@gmail.com>"]
 description = "All your history in one place"

README.org

@@ -13,6 +13,7 @@ All your history in one place.
 - Rich dashboards to visualize your history
 - Export as CSV file
 - Entirely offline, No need to worry about privacy leaks
+- Well-designed schemas to avoid history duplication when backup multiple times
 
 * Screenshots
 ** Daily Page View
@@ -83,6 +84,12 @@ Users can also use =-f= option to set other history files to backup, the file ha
 | Firefox | places.sqlite    |
 | Safari  | History.db       |
 
+* FAQ
+- =Error code 5: The database file is locked= :: This error happens if your browser is opened during backup, as SQLite allow only one open connection.
+
+  Close the browser is one solution, or you can copy history file to other directory other than default location.
+
+
 * LICENSE
 Copyright (c) 2022 Jiacai Liu <jiacai2050+1history@gmail.com>
 

src/database.rs

@@ -102,7 +102,7 @@ CREATE TABLE IF NOT EXISTS import_records (
         }
     }
 
-    fn persist_visits(&self, src_path: &str, batch: Vec<HistoryVisit>) -> Result<usize> {
+    fn persist_visits(&self, src_path: &str, batch: Vec<HistoryVisit>) -> Result<(usize, usize)> {
         assert!(!batch.is_empty());
 
         let sql = r#"
@@ -114,6 +114,7 @@ INSERT INTO onehistory_visits (item_id, visit_time, visit_type)
         let tx = conn.transaction()?;
         let last_ts = batch[batch.len() - 1].visit_time;
         let mut affected = 0;
+        let mut duplicated = 0;
         for HistoryVisit {
             item_id,
             visit_time,
@@ -125,6 +126,7 @@ INSERT INTO onehistory_visits (item_id, visit_time, visit_type)
                 Err(e) => {
                     if let sqlError::SqliteFailure(ffi_err, _msg) = &e {
                         if ffi_err.code == ErrorCode::ConstraintViolation {
+                            duplicated += 1;
                             let ext_code = ffi_err.extended_code;
                             debug!(
                                 "[ignore]onehistory_visits duplicated. id:{item_id}, \
@@ -140,13 +142,14 @@ INSERT INTO onehistory_visits (item_id, visit_time, visit_type)
         Self::update_process(&tx, src_path, last_ts)?;
         tx.commit()?;
 
-        Ok(affected)
+        Ok((affected, duplicated))
     }
 
-    pub fn persist(&self, src_path: &str, details: Vec<VisitDetail>) -> Result<usize> {
+    pub fn persist(&self, src_path: &str, details: Vec<VisitDetail>) -> Result<(usize, usize)> {
         let mut i = 0;
         let mut batch = None; // Use Option so we can take it out later
         let mut affected = 0;
+        let mut duplicated = 0;
         for VisitDetail {
             url,
             title,
@@ -163,14 +166,18 @@ INSERT INTO onehistory_visits (item_id, visit_time, visit_type)
                 visit_type,
             });
             if i % self.persist_batch == 0 {
-                affected += self.persist_visits(src_path, batch.take().unwrap())?;
+                let (a, d) = self.persist_visits(src_path, batch.take().unwrap())?;
+                affected += a;
+                duplicated += d;
             }
         }
         if batch.is_some() {
-            affected += self.persist_visits(src_path, batch.take().unwrap())?;
+            let (a, d) = self.persist_visits(src_path, batch.take().unwrap())?;
+            affected += a;
+            duplicated += d;
         }
 
-        Ok(affected)
+        Ok((affected, duplicated))
     }
 
     fn update_process(tx: &Transaction<'_>, src_path: &str, ts: i64) -> Result<()> {

src/export.rs

@@ -7,12 +7,11 @@ use std::{
 
 use crate::{
     database::Database,
-    util::{now_as_unixepoch_ms, unixepoch_as_ymdhms},
+    util::{full_timerange, unixepoch_as_ymdhms},
 };
 
 pub fn export_csv(csv_file: String, db_file: String) -> Result<()> {
-    let start = 0;
-    let end = now_as_unixepoch_ms() + 3_600_000;
+    let (start, end) = full_timerange();
     debug!("start:{}, end:{}", start, end);
 
     let db = Database::open(db_file).context("open 1history db")?;

src/main.rs

@@ -7,7 +7,7 @@ mod web;
 
 use crate::database::Database;
 use crate::source::Source;
-use crate::util::{now_as_unixepoch_ms, DEFAULT_CSV_FILE, DEFAULT_DB_FILE};
+use crate::util::{full_timerange, DEFAULT_CSV_FILE, DEFAULT_DB_FILE};
 use anyhow::{Context, Result};
 use clap::{Parser, Subcommand};
 use export::export_csv;
@@ -87,28 +87,34 @@ fn show() -> Result<()> {
 }
 
 fn backup(history_files: Vec<String>, db_file: String, dry_run: bool) -> Result<()> {
-    let start = 0; // Currently use 0 to backup all histories
-    let end = now_as_unixepoch_ms() + 3_600_000; // 1 hour later
+    let (start, end) = full_timerange();
     debug!("start:{}, end:{}", start, end);
 
     let db = Database::open(db_file).context("open dst db")?;
 
     let mut found = 0;
     let mut total_affected = 0;
+    let mut total_duplicated = 0;
     for his_file in history_files {
         let s = Source::open(his_file.to_string()).context("open")?;
         let rows = s.select(start, end).context("select")?.collect::<Vec<_>>();
         debug!("{:?} select {} histories", s.name(), rows.len());
         found += rows.len();
 
         if !dry_run {
-            let affected = db.persist(s.path(), rows).context("persist")?;
-            debug!("{:?} save {} histories", s.name(), affected);
+            let (affected, duplicated) = db.persist(s.path(), rows).context("persist")?;
+            debug!(
+                "{:?} affected:{}, duplicated:{}",
+                s.name(),
+                affected,
+                duplicated
+            );
             total_affected += affected;
+            total_duplicated += duplicated;
         }
     }
 
-    info!("Summary:found {found}, imported {total_affected}");
+    info!("Summary\nFound:{found}, Imported:{total_affected}, Duplicated: {total_duplicated}");
     Ok(())
 }
 

src/source.rs

@@ -13,7 +13,7 @@ pub struct Source {
 
 impl Source {
     pub fn open(path: String) -> Result<Source> {
-        let flags = OpenFlags::SQLITE_OPEN_READ_ONLY;
+        let flags = OpenFlags::SQLITE_OPEN_READ_WRITE;
         let conn = Connection::open_with_flags(&path, flags).context(path.clone())?;
         let name = Self::detect_name(&conn).context(format!("detect {path}"))?;
         Ok(Source { path, name, conn })

src/util.rs

@@ -144,6 +144,12 @@ pub fn domain_from(url: String) -> String {
     url
 }
 
+pub fn full_timerange() -> (i64, i64) {
+    let start = 0;
+    let end = now_as_unixepoch_ms() + 3_600_000; // 1 hour later
+    (start, end) // Use this timerange to represent ALL
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;