TheLovinator/ttvdrops
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add user interruption support

Browse source
This commit is contained in:
Joakim Hellsén 2025-09-24 03:07:08 +02:00
commit 13affa2382
1 changed files with 36 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

61
twitch/management/commands/import_drops.py
View file

@ -86,10 +86,10 @@ class Command(BaseCommand):
CommandError: If the file/directory doesn't exist, isn't a JSON file, CommandError: If the file/directory doesn't exist, isn't a JSON file,
or has an invalid JSON structure. or has an invalid JSON structure.
ValueError: If the JSON file has an invalid structure. ValueError: If the JSON file has an invalid structure.
TypeError: If the JSON file has an invalid structure. TypeError: If the JSON file has an invalid JSON structure.
AttributeError: If the JSON file has an invalid structure. AttributeError: If the JSON file has an invalid JSON structure.
KeyError: If the JSON file has an invalid structure. KeyError: If the JSON file has an invalid JSON structure.
IndexError: If the JSON file has an invalid structure. IndexError: If the JSON file has an invalid JSON structure.
""" """
paths: list[str] = options["paths"] paths: list[str] = options["paths"]
processed_dir: str = options["processed_dir"] processed_dir: str = options["processed_dir"]
@ -118,6 +118,10 @@ class Command(BaseCommand):
raise raise
self.stdout.write(self.style.ERROR(f"Data error processing path {p}")) self.stdout.write(self.style.ERROR(f"Data error processing path {p}"))
self.stdout.write(self.style.ERROR(traceback.format_exc())) self.stdout.write(self.style.ERROR(traceback.format_exc()))
except KeyboardInterrupt:
# Gracefully handle Ctrl+C
self.stdout.write(self.style.WARNING("Interrupted by user, exiting import."))
return
def process_drops(self, *, continue_on_error: bool, path: Path, processed_path: Path) -> None: def process_drops(self, *, continue_on_error: bool, path: Path, processed_path: Path) -> None:
"""Process drops from a file or directory. """Process drops from a file or directory.
@ -171,6 +175,7 @@ class Command(BaseCommand):
AttributeError: If a JSON file has an invalid structure. AttributeError: If a JSON file has an invalid structure.
KeyError: If a JSON file has an invalid structure. KeyError: If a JSON file has an invalid structure.
IndexError: If a JSON file has an invalid structure. IndexError: If a JSON file has an invalid structure.
KeyboardInterrupt: If processing is interrupted by the user.
""" """
json_files: list[Path] = list(directory.glob("*.json")) json_files: list[Path] = list(directory.glob("*.json"))
if not json_files: if not json_files:
@ -182,29 +187,33 @@ class Command(BaseCommand):
start_time: float = time.time() start_time: float = time.time()
processed = 0 processed = 0
with concurrent.futures.ThreadPoolExecutor() as executor: try:
future_to_file: dict[concurrent.futures.Future[None], Path] = { with concurrent.futures.ThreadPoolExecutor() as executor:
executor.submit(self._process_file, json_file, processed_path): json_file for json_file in json_files future_to_file: dict[concurrent.futures.Future[None], Path] = {
} executor.submit(self._process_file, json_file, processed_path): json_file for json_file in json_files
for future in concurrent.futures.as_completed(future_to_file): }
json_file: Path = future_to_file[future] for future in concurrent.futures.as_completed(future_to_file):
self.stdout.write(f"Processing file {json_file.name}...") json_file: Path = future_to_file[future]
try: self.stdout.write(f"Processing file {json_file.name}...")
future.result() try:
except CommandError as e: future.result()
if not continue_on_error: except CommandError as e:
raise if not continue_on_error:
self.stdout.write(self.style.ERROR(f"Error processing {json_file}: {e}")) raise
except (ValueError, TypeError, AttributeError, KeyError, IndexError): self.stdout.write(self.style.ERROR(f"Error processing {json_file}: {e}"))
if not continue_on_error: except (ValueError, TypeError, AttributeError, KeyError, IndexError):
raise if not continue_on_error:
self.stdout.write(self.style.ERROR(f"Data error processing {json_file}")) raise
self.stdout.write(self.style.ERROR(traceback.format_exc())) self.stdout.write(self.style.ERROR(f"Data error processing {json_file}"))
self.stdout.write(self.style.ERROR(traceback.format_exc()))
self.update_processing_progress(total_files=total_files, start_time=start_time, processed=processed) self.update_processing_progress(total_files=total_files, start_time=start_time, processed=processed)
except KeyboardInterrupt:
msg: str = f"Processed {total_files} JSON files in {directory}. Moved processed files to {processed_path}." self.stdout.write(self.style.WARNING("Interrupted by user, exiting import."))
self.stdout.write(self.style.SUCCESS(msg)) raise
else:
msg: str = f"Processed {total_files} JSON files in {directory}. Moved processed files to {processed_path}."
self.stdout.write(self.style.SUCCESS(msg))
def update_processing_progress(self, total_files: int, start_time: float, processed: int) -> None: def update_processing_progress(self, total_files: int, start_time: float, processed: int) -> None:
"""Update and display processing progress. """Update and display processing progress.