From 44cd440a176f104b8d401c1c9949655732418b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Helle=C5=9Ben?= Date: Mon, 9 Mar 2026 05:27:11 +0100 Subject: [PATCH] Add watch_imports command --- tools/systemd/ttvdrops-import-drops.service | 30 ++- tools/systemd/ttvdrops-import-drops.timer | 10 - twitch/management/commands/watch_imports.py | 116 +++++++++++ twitch/tests/test_watch_imports.py | 214 ++++++++++++++++++++ 4 files changed, 355 insertions(+), 15 deletions(-) delete mode 100644 tools/systemd/ttvdrops-import-drops.timer create mode 100644 twitch/management/commands/watch_imports.py create mode 100644 twitch/tests/test_watch_imports.py diff --git a/tools/systemd/ttvdrops-import-drops.service b/tools/systemd/ttvdrops-import-drops.service index 585d10e..85f2b4d 100644 --- a/tools/systemd/ttvdrops-import-drops.service +++ b/tools/systemd/ttvdrops-import-drops.service @@ -1,12 +1,32 @@ [Unit] -Description=TTVDrops import drops from pending directory +Description=TTVDrops watch and import drops from pending directory +After=network-online.target +Wants=network-online.target [Service] -Type=oneshot +Type=simple User=ttvdrops Group=ttvdrops WorkingDirectory=/home/ttvdrops/ttvdrops EnvironmentFile=/home/ttvdrops/ttvdrops/.env -ExecStart=/usr/bin/uv run python manage.py better_import_drops /mnt/fourteen/Data/Responses/pending --ExecStartPost=/usr/bin/uv run python manage.py download_box_art --ExecStartPost=/usr/bin/uv run python manage.py download_campaign_images +ExecStart=/usr/bin/uv run python manage.py watch_imports /mnt/fourteen/Data/Responses/pending --verbose + +# Restart policy +Restart=on-failure +RestartSec=5s + +# Process management +KillMode=mixed +KillSignal=SIGTERM + +# Resource limits +MemoryLimit=512M +CPUQuota=50% + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ttvdrops-watch + +[Install] +WantedBy=multi-user.target diff --git a/tools/systemd/ttvdrops-import-drops.timer b/tools/systemd/ttvdrops-import-drops.timer deleted file mode 100644 index a715037..0000000 --- a/tools/systemd/ttvdrops-import-drops.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Frequent TTVDrops import drops timer - -[Timer] -OnBootSec=0 -OnUnitActiveSec=1min -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/twitch/management/commands/watch_imports.py b/twitch/management/commands/watch_imports.py new file mode 100644 index 0000000..a95bb20 --- /dev/null +++ b/twitch/management/commands/watch_imports.py @@ -0,0 +1,116 @@ +import logging +from pathlib import Path +from time import sleep +from typing import TYPE_CHECKING + +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError + +from twitch.management.commands.better_import_drops import ( + Command as BetterImportDropsCommand, +) + +if TYPE_CHECKING: + from django.core.management.base import CommandParser + +logger: logging.Logger = logging.getLogger("ttvdrops.watch_imports") + + +class Command(BaseCommand): + """Watch for JSON files in a directory and import them automatically.""" + + help = "Watch a directory for JSON files and import them automatically" + requires_migrations_checks = True + + def add_arguments(self, parser: CommandParser) -> None: + """Populate the command with arguments.""" + parser.add_argument( + "path", + type=str, + help="Path to directory to watch for JSON files", + ) + + def handle(self, *args, **options) -> None: # noqa: ARG002 + """Main entry point for the watch command. + + Args: + *args: Variable length argument list (unused). + **options: Arbitrary keyword arguments. + + Raises: + CommandError: If the provided path does not exist or is not a directory. + """ + watch_path: Path = self.get_watch_path(options) + self.stdout.write( + self.style.SUCCESS(f"Watching {watch_path} for JSON files..."), + ) + self.stdout.write("Press Ctrl+C to stop\n") + + importer_command: BetterImportDropsCommand = BetterImportDropsCommand() + + while True: + try: + sleep(1) + self.import_json_files( + importer_command=importer_command, + watch_path=watch_path, + ) + + except KeyboardInterrupt: + msg = "Received keyboard interrupt. Stopping watch..." + self.stdout.write(self.style.WARNING(msg)) + break + except CommandError as e: + msg = f"Import command error: {e}" + self.stdout.write(self.style.ERROR(msg)) + except Exception as e: + msg: str = f"Error while watching directory: {e}" + raise CommandError(msg) from e + + logger.info("Stopped watching directory: %s", watch_path) + + def import_json_files( + self, + importer_command: BetterImportDropsCommand, + watch_path: Path, + ) -> None: + """Import all JSON files in the watch directory using the provided importer command. + + Args: + importer_command: An instance of the BetterImportDropsCommand to handle the import logic. + watch_path: The directory path to watch for JSON files. + """ + # TODO(TheLovinator): Implement actual file watching using watchdog or similar library. # noqa: TD003 + json_files: list[Path] = [ + f for f in watch_path.iterdir() if f.suffix == ".json" and f.is_file() + ] + if not json_files: + return + + for json_file in json_files: + self.stdout.write(f"Importing {json_file}...") + importer_command.handle(path=json_file) + + def get_watch_path(self, options: dict[str, str]) -> Path: + """Validate and return the watch path from the command options. + + Args: + options: The command options containing the 'path' key. + + Returns: + The validated watch path as a Path object. + + Raises: + CommandError: If the provided path does not exist or is not a directory. + """ + watch_path: Path = Path(options["path"]).resolve() + + if not watch_path.exists(): + msg: str = f"Path does not exist: {watch_path}" + raise CommandError(msg) + + if not watch_path.is_dir(): + msg: str = f"Path is not a directory: {watch_path}, it is a {watch_path.stat().st_mode}" + raise CommandError(msg) + + return watch_path diff --git a/twitch/tests/test_watch_imports.py b/twitch/tests/test_watch_imports.py new file mode 100644 index 0000000..37d0005 --- /dev/null +++ b/twitch/tests/test_watch_imports.py @@ -0,0 +1,214 @@ +from pathlib import Path +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +import pytest +from _pytest.capture import CaptureResult +from django.core.management.base import CommandError + +from twitch.management.commands.watch_imports import Command + +if TYPE_CHECKING: + from pathlib import Path + + from _pytest.capture import CaptureResult + + +class TestWatchImportsCommand: + """Tests for the watch_imports management command.""" + + def test_get_watch_path_returns_resolved_directory(self, tmp_path: Path) -> None: + """It should return the resolved path when a valid directory is provided.""" + command = Command() + + result: Path = command.get_watch_path({"path": str(tmp_path)}) + + assert result == tmp_path.resolve() + + def test_get_watch_path_raises_for_missing_path(self, tmp_path: Path) -> None: + """It should raise CommandError when the path does not exist.""" + command = Command() + missing_path: Path = tmp_path / "missing" + + with pytest.raises(CommandError, match="Path does not exist"): + command.get_watch_path({"path": str(missing_path)}) + + def test_get_watch_path_raises_for_file_path(self, tmp_path: Path) -> None: + """It should raise CommandError when the path is not a directory.""" + command = Command() + file_path: Path = tmp_path / "data.json" + file_path.write_text("{}", encoding="utf-8") + + with pytest.raises(CommandError, match="Path is not a directory"): + command.get_watch_path({"path": str(file_path)}) + + def test_import_json_files_imports_only_json_files(self, tmp_path: Path) -> None: + """It should call importer for .json files and ignore other entries.""" + command = Command() + importer_command = MagicMock() + + json_file_1: Path = tmp_path / "one.json" + json_file_2: Path = tmp_path / "two.json" + ignored_txt: Path = tmp_path / "notes.txt" + ignored_dir: Path = tmp_path / "nested.json" + + json_file_1.write_text("{}", encoding="utf-8") + json_file_2.write_text("[]", encoding="utf-8") + ignored_txt.write_text("ignored", encoding="utf-8") + ignored_dir.mkdir() + + command.import_json_files( + importer_command=importer_command, + watch_path=tmp_path, + ) + + imported_paths: list[Path] = [ + call.kwargs["path"] for call in importer_command.handle.call_args_list + ] + assert set(imported_paths) == {json_file_1, json_file_2} + + def test_import_json_files_no_json_files_does_nothing(self, tmp_path: Path) -> None: + """It should not call importer when no JSON files are present.""" + command = Command() + importer_command = MagicMock() + + (tmp_path / "notes.txt").write_text("ignored", encoding="utf-8") + + command.import_json_files( + importer_command=importer_command, + watch_path=tmp_path, + ) + + importer_command.handle.assert_not_called() + + def test_handle_stops_cleanly_on_keyboard_interrupt( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + """It should print a warning and stop when interrupted by keyboard.""" + command = Command() + importer_instance = MagicMock() + + monkeypatch.setattr(command, "get_watch_path", lambda options: tmp_path) + monkeypatch.setattr( + "twitch.management.commands.watch_imports.BetterImportDropsCommand", + lambda: importer_instance, + ) + monkeypatch.setattr( + "twitch.management.commands.watch_imports.sleep", + lambda _seconds: (_ for _ in ()).throw(KeyboardInterrupt()), + ) + + command.handle(path=str(tmp_path)) + + captured: CaptureResult[str] = capsys.readouterr() + assert "Watching" in captured.out + assert "Received keyboard interrupt. Stopping watch..." in captured.out + + def test_handle_reports_command_errors_and_continues_until_interrupt( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + """It should report CommandError from import and continue the loop.""" + command = Command() + importer_instance = MagicMock() + sleep_calls: dict[str, int] = {"count": 0} + + def fake_sleep(_seconds: int) -> None: + """Simulate sleep and raise KeyboardInterrupt after 2 calls. + + Args: + _seconds: The number of seconds to sleep (ignored). + + Raises: + KeyboardInterrupt: After being called twice to simulate user interrupt. + """ + sleep_calls["count"] += 1 + if sleep_calls["count"] == 2: + raise KeyboardInterrupt + + def fake_import_json_files( + *, + importer_command: MagicMock, + watch_path: Path, + ) -> None: + """Simulate an import that raises CommandError. + + Args: + importer_command: The mock importer command instance (ignored). + watch_path: The path being watched (ignored). + + Raises: + CommandError: Always raised to simulate an import error. + """ + msg = "bad import" + raise CommandError(msg) + + monkeypatch.setattr( + command, + "get_watch_path", + lambda options: tmp_path, + ) + monkeypatch.setattr( + "twitch.management.commands.watch_imports.BetterImportDropsCommand", + lambda: importer_instance, + ) + monkeypatch.setattr( + "twitch.management.commands.watch_imports.sleep", + fake_sleep, + ) + monkeypatch.setattr( + command, + "import_json_files", + fake_import_json_files, + ) + + command.handle(path=str(tmp_path)) + + captured: CaptureResult[str] = capsys.readouterr() + assert "Import command error: bad import" in captured.out + assert "Received keyboard interrupt. Stopping watch..." in captured.out + + def test_handle_wraps_unexpected_errors_in_command_error( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """It should wrap unexpected exceptions in a CommandError.""" + command = Command() + importer_instance = MagicMock() + + monkeypatch.setattr( + command, + "get_watch_path", + lambda options: tmp_path, + ) + monkeypatch.setattr( + "twitch.management.commands.watch_imports.BetterImportDropsCommand", + lambda: importer_instance, + ) + monkeypatch.setattr( + "twitch.management.commands.watch_imports.sleep", + lambda _seconds: None, + ) + + def fake_import_json_files( + *, + importer_command: MagicMock, + watch_path: Path, + ) -> None: + msg = "boom" + raise RuntimeError(msg) + + monkeypatch.setattr( + command, + "import_json_files", + fake_import_json_files, + ) + + with pytest.raises(CommandError, match="Error while watching directory: boom"): + command.handle(path=str(tmp_path))