feat: add auto ignore, add lockfile

.github/workflows/from_commit_to_build_test.yml

@@ -79,5 +79,5 @@ jobs:
         if: ${{ env.ACT == '' }}
         with:
           name: coverage-reports
-          path: /home/runner/rusticlone-tests/coverage/
+          path: /home/runner/.cache/rusticlone-tests/coverage/
           retention-days: 30

Makefile

@@ -18,7 +18,7 @@ test:
 	uv run bash tests/tests.sh
 
 ci:
-	act --workflows ".github/workflows/test.yml"
+	act --workflows ".github/workflows/from_commit_to_build_test.yml"
 
 toc:
 	find * -type f ! -name 'CHANGELOG.md' -exec toc -f {} \; 2>/dev/null

README.md

@@ -242,20 +242,17 @@ keep-quarter-yearly = 4
 keep-yearly = 1
 ```
 
+As it doesn't contain a "\[repository]" section, it will not be treated as a standalone profile by Rusticlone.
+
 This "common.toml" profile can be referenced from our documents by adding to "Documents.toml" the following:
 
 ```toml
 [global]
 use-profile = ["common"]
-```
 
-To exclude "common.toml" from Rusticlone (since it cannot be used alone), add the `--ignore` argument followed by "common":
-
-```bash
-rusticlone --ignore "common" -r "gdrive:/PC" backup
+# [...]
 ```
 
-All the profiles containing "common" in their name will be excluded, but will still be sourced from other profiles when needed.
 
 ### Custom log file
 
@@ -296,10 +293,10 @@ Description=Rusticlone service
 
 [Service]
 Type=oneshot
-ExecStart=rusticlone --ignore "common" --remote "gdrive:/PC" backup
+ExecStart=rusticlone --remote "gdrive:/PC" backup
 ```
 
-Adjust your `--ignore` and `--remote` as needed.
+Adjust your `--remote` as needed.
 
 Apply your changes and enable the timer:
 

example/systemd/rusticlone.service

@@ -3,5 +3,5 @@ Description=Rusticlone Backup - service
 
 [Service]
 Type=oneshot
-# replace the remote and ignore pattern
-ExecStart=rusticlone --remote "gdrive:/PC" --ignore "common" backup
+# replace the remote
+ExecStart=rusticlone --remote "gdrive:/PC" backup

rusticlone/cli.py

@@ -58,9 +58,8 @@ def parse_args():
         "-i",
         "--ignore",
         type=str,
-        default="🫣🫣🫣",
-        env_var="IGNORE",
-        help="Ignore rustic profiles containing this pattern",
+        default="",
+        help="Deprecated argument, does nothing. Will be removed in a future release",
     )
     parser.add_argument(
         "-l",

rusticlone/helpers/custom.py

@@ -26,8 +26,12 @@
 # exit
 import sys
 
+# toml parsing
+import tomllib
+
 # rusticlone
 from rusticlone.helpers.action import Action
+from rusticlone.helpers.rustic import Rustic
 from rusticlone.helpers.notification import notify_user
 from rusticlone.processing.parallel import (
     system_backup_parallel,
@@ -45,6 +49,7 @@
     system_download_sequential,
     system_extract_sequential,
 )
+from rusticlone.processing.profile import parse_repo, parse_sources
 
 # ################################################################ CLASSES
 
@@ -82,17 +87,12 @@ def __init__(self, args) -> None:
                 Path("/etc/rustic"),
             ]
         else:
-            self.profiles_dirs = [
-                Path.home() / ".config/rustic",
-                Path("/etc/rustic"),
-            ]
+            self.profiles_dirs = [Path.home() / ".config/rustic", Path("/etc/rustic")]
         # remote prefix: rclone remote + subdirectory without trailing slash
         if args.remote is not None:
             self.remote_prefix = args.remote.rstrip("/")
         else:
             self.remote_prefix = "None:/"
-        # ignore pattern for profiles
-        self.ignore_pattern = args.ignore
         # test profile
         if args.profile:
             self.provided_profile = args.profile
@@ -141,26 +141,41 @@ def check_profiles_dirs(self) -> None:
 # ################################################################ FUNCTIONS
 
 
-def list_profiles(
-    profiles_dirs: list, provided_profile: str = "*", ignore_pattern: str = "🫣🫣🫣"
-) -> list:
+def has_needed_config_components(profile_path: Path) -> bool:
+    """
+    Check if a profile should be processed by running rustic show-config
+    """
+    try:
+        profile_name = profile_path.stem
+        rustic = Rustic(profile_name, "show-config")
+        config = tomllib.loads(rustic.stdout)
+        has_repo = parse_repo(config) is not None
+        has_sources = parse_sources(config) is not None
+        return has_repo and has_sources
+    except (AttributeError, tomllib.TOMLDecodeError, KeyError, IndexError):
+        return False
+
+
+def list_profiles(profiles_dirs: list, provided_profile: str = "*") -> list:
     """
     Scan profiles from directories if none have been provided explicitely
     Don't scan from /etc/rustic if ~/.config/rustic has some profiles'
     """
     action = Action("Reading profiles")
     profiles: list[str] = []
+    opened_files = 0
     if not provided_profile:
         provided_profile = "*"
     for profiles_dir in profiles_dirs:
         if not profiles:
             action.stop(f'Scanning "{profiles_dir}"', "")
             files = sorted(list(profiles_dir.glob(f"{provided_profile}.toml")))
             for file in files:
+                opened_files += 1
                 if (
                     file.is_file()
-                    and ignore_pattern not in file.stem
                     and file.stem not in profiles
+                    and has_needed_config_components(file)
                 ):
                     profiles.append(file.stem)
     # remove duplicates
@@ -169,9 +184,10 @@ def list_profiles(
         action.stop(f"Profiles: {str(profiles)}", "")
         return profiles
     else:
-        print(provided_profile)
-        print(provided_profile)
-        action.abort("Could not find any rustic profile")
+        print("")
+        print(f"Glob pattern: {provided_profile}")
+        print(f"Files tried: {opened_files}")
+        action.abort("Could not find any valid profile")
         sys.exit(1)
 
 
@@ -243,6 +259,7 @@ def process_profiles(
                 )
             case _:
                 print(f"Invalid command '{command}'")
+                sys.exit(1)
     if apprise_url and results:
         notify_user(results, apprise_url)
 
@@ -254,9 +271,7 @@ def load_customizations(args: Namespace):
     custom = Custom(args)
     custom.check_log_file()
     custom.check_profiles_dirs()
-    profiles = list_profiles(
-        custom.profiles_dirs, custom.provided_profile, custom.ignore_pattern
-    )
+    profiles = list_profiles(custom.profiles_dirs, custom.provided_profile)
     process_profiles(
         profiles,
         custom.parallel,

rusticlone/helpers/rclone.py

@@ -48,6 +48,8 @@ def __init__(self, **kwargs):
             "--drive-chunk-size=128M",
             "--drive-acknowledge-abuse",
             "--drive-stop-on-upload-limit",
+            "--no-update-modtime",
+            "--no-update-dir-modtime",
         ]
         default_kwargs: dict[str, Any] = {
             "env": {},

rusticlone/processing/atomic.py

@@ -40,12 +40,14 @@ def profile_archive(
     profile.set_log_file(log_file)
     profile.check_sources_exist()
     profile.check_local_repo_exists()
+    profile.create_lockfile("archive")
     profile.check_local_repo_health()
     profile.init()
     profile.backup()
     profile.forget()
     profile.source_stats()
     profile.repo_stats()
+    profile.delete_lockfile()
     timer.stop()
     # action.stop(" ", "")
     return profile.result, timer.duration
@@ -64,7 +66,9 @@ def profile_upload(
     profile.set_log_file(log_file)
     profile.check_rclone_config_exists()
     profile.check_local_repo_exists()
+    profile.create_lockfile("upload")
     profile.upload(remote_prefix)
+    profile.delete_lockfile()
     timer.stop()
     # action.stop(" ", "")
     return profile.result, timer.duration
@@ -87,7 +91,9 @@ def profile_download(
     profile.check_rclone_config_exists()
     profile.check_remote_repo_exists(remote_prefix)
     profile.check_local_repo_exists()
+    profile.create_lockfile("download")
     profile.download(remote_prefix)
+    profile.delete_lockfile()
     timer.stop()
     # print_stats("", "")
     return profile.result, timer.duration
@@ -105,9 +111,11 @@ def profile_extract(
     profile.parse_rustic_config()
     profile.set_log_file(log_file)
     profile.check_local_repo_exists()
+    profile.create_lockfile("extract")
     profile.check_latest_snapshot()
     profile.check_sources_type()
     profile.restore()
+    profile.delete_lockfile()
     timer.stop()
     # print_stats("", "")
     return profile.result, timer.duration