From 7c4bb9acca3b97973e661c6031b429769f35ba05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Sat, 21 Feb 2026 06:05:07 +0100 Subject: [PATCH] rip Docker, long live systemd --- .dockerignore | 27 ------ .env.example | 4 + .github/workflows/docker.yaml | 46 +---------- .vscode/settings.json | 1 + Dockerfile | 30 ------- README.md | 91 +++++++++++++++++++++ docker-compose.yml | 89 -------------------- pyproject.toml | 2 + start.sh | 11 --- tools/systemd/ttvdrops-backup.service | 10 +++ tools/systemd/ttvdrops-backup.timer | 9 ++ tools/systemd/ttvdrops-import-drops.service | 12 +++ tools/systemd/ttvdrops-import-drops.timer | 10 +++ tools/systemd/ttvdrops.service | 32 ++++++++ tools/systemd/ttvdrops.socket | 11 +++ 15 files changed, 184 insertions(+), 201 deletions(-) delete mode 100644 .dockerignore delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml delete mode 100755 start.sh create mode 100644 tools/systemd/ttvdrops-backup.service create mode 100644 tools/systemd/ttvdrops-backup.timer create mode 100644 tools/systemd/ttvdrops-import-drops.service create mode 100644 tools/systemd/ttvdrops-import-drops.timer create mode 100644 tools/systemd/ttvdrops.service create mode 100644 tools/systemd/ttvdrops.socket diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 696f042..0000000 --- a/.dockerignore +++ /dev/null @@ -1,27 +0,0 @@ -.env -.env.example -.git/ -.github/ -.pre-commit-config.yaml -.pytest_cache/ -.ruff_cache/ -.venv -.vscode/ -.vscode/ -*.json -*.log -*.py[codz] -**/__pycache__/ -*$py.class -archive/ -check_these_please/ -db.sqlite3 -db.sqlite3-journal -env.bak/ -env/ -ENV/ -responses/ -staticfiles/ -tests/ -venv.bak/ -venv/ diff --git a/.env.example b/.env.example index ea1c6e9..0e6117f 100644 --- a/.env.example +++ b/.env.example @@ -31,3 +31,7 @@ EMAIL_USE_SSL=False # Connection timeout in seconds EMAIL_TIMEOUT=10 + +# Where to store Twitch API responses +TTVDROPS_IMPORTED_DIR=/mnt/fourteen/Data/Responses/imported +TTVDROPS_BROKEN_DIR=/mnt/fourteen/Data/Responses/broken diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 08f000b..4ae5f35 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -1,10 +1,10 @@ -name: Test and Build Docker Image +name: Run Pytest on: push: pull_request: schedule: - - cron: "0 14 * * *" # Run every day at 14:00 CET + - cron: "0 14 * * 0" # Run weekly at 14:00 UTC workflow_dispatch: jobs: @@ -16,23 +16,7 @@ jobs: DJANGO_SECRET_KEY: 1234567890 steps: - - uses: docker/login-action@v3 - if: github.event_name != 'pull_request' - with: - registry: ghcr.io - username: thelovinator1 - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/setup-buildx-action@v3 - - uses: actions/checkout@v6 - - - uses: astral-sh/ruff-action@v3 - with: - version: "latest" - - run: ruff check --exit-non-zero-on-fix --verbose - - run: ruff format --check --verbose - - - run: docker build --check . - uses: actions/setup-python@v6 with: python-version: 3.14 @@ -40,29 +24,3 @@ jobs: - uses: astral-sh/setup-uv@v7 - run: uv sync --all-extras --dev - run: uv run pytest - - - run: uv run python manage.py makemigrations --check - env: - TESTING: True - - run: uv run python manage.py migrate - env: - TESTING: True - - run: uv run python manage.py collectstatic --noinput - - id: meta - uses: docker/metadata-action@v5 - env: - DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index - with: - images: | - ghcr.io/thelovinator1/ttvdrops - tags: | - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - - - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - labels: ${{ steps.meta.outputs.labels }} - tags: ${{ steps.meta.outputs.tags }} - annotations: ${{ steps.meta.outputs.annotations }} diff --git a/.vscode/settings.json b/.vscode/settings.json index 7dd2244..6e557e2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -79,6 +79,7 @@ "ttvdrops", "twid", "twitchgamedata", + "usermod", "venv", "wrongpassword", "wsgi", diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 66aaa1b..0000000 --- a/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM python:3.14-slim-trixie -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ - -ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy -ENV UV_NO_DEV=1 -ENV UV_PYTHON_DOWNLOADS=0 -ENV UV_NO_CACHE=1 - -WORKDIR /app -COPY . /app/ - -RUN uv sync -RUN chmod +x /app/start.sh - -ENV PATH="/app/.venv/bin:$PATH" - -RUN groupadd -g 1000 ttvdrops \ - && useradd -m -u 1000 -g 1000 -d /home/ttvdrops -s /bin/sh ttvdrops \ - && mkdir -p /home/ttvdrops/.local/share/TTVDrops \ - && chown -R ttvdrops:ttvdrops /home/ttvdrops /app - -HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ - CMD curl -f http://localhost:8000/ || exit 1 - -VOLUME ["/home/ttvdrops/.local/share/TTVDrops"] -EXPOSE 8000 -USER ttvdrops -ENTRYPOINT [ "/app/start.sh" ] -CMD ["uv", "run", "gunicorn", "config.wsgi:application", "--workers", "27", "--bind", "0.0.0.0:8000"] diff --git a/README.md b/README.md index ace0d5b..9fa842f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,74 @@ Get notified when a new drop is available on Twitch +## TL;DR (Arch Linux + PostgreSQL) + +Install and initialize Postgres, then start the service: + +```bash +sudo pacman -S postgresql +sudo -u postgres initdb -D /var/lib/postgres/data +sudo systemctl enable --now postgresql +``` + +Create a local role and database: + +```bash +sudo -u postgres createuser -P ttvdrops +sudo -u postgres createdb -O ttvdrops ttvdrops +``` + +Point Django at the unix socket used by Arch (`/run/postgresql`): + +```bash +POSTGRES_USER=ttvdrops +POSTGRES_PASSWORD=your_password +POSTGRES_DB=ttvdrops +POSTGRES_HOST=/run/postgresql +POSTGRES_PORT=5432 +``` + +### Linux (Systemd) + +```bash +sudo useradd --create-home --home-dir /home/ttvdrops --shell /bin/fish ttvdrops +sudo passwd ttvdrops +sudo usermod -aG wheel ttvdrops +su - ttvdrops +git clone https://github.com/TheLovinator1/ttvdrops.git +cd ttvdrops +uv sync --no-dev + +# Modify .env with the correct database credentials and other settings, then run migrations: +uv run python manage.py migrate +``` + +Install the systemd service from the repo: + +```bash +sudo install -m 0644 tools/systemd/ttvdrops.socket /etc/systemd/system/ttvdrops.socket +sudo install -m 0644 tools/systemd/ttvdrops.service /etc/systemd/system/ttvdrops.service +``` + +Enable and start the service: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now ttvdrops.socket +sudo systemctl enable --now ttvdrops.service + +curl --unix-socket /run/ttvdrops/ttvdrops.sock http://ttvdrops.lovinator.space +``` + +Install and enable timers: + +```bash +sudo install -m 0644 tools/systemd/ttvdrops-backup.{service,timer} /etc/systemd/system/ +sudo install -m 0644 tools/systemd/ttvdrops-import-drops.{service,timer} /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now ttvdrops-backup.timer ttvdrops-import-drops.timer +``` + ## Development ```bash @@ -42,3 +110,26 @@ Optional arguments: ```bash uv run python manage.py backup_db --output-dir "" --prefix "ttvdrops" ``` + +### How the duck does permissions work on Linux? + +```bash +sudo groupadd responses +sudo usermod -aG responses lovinator +sudo usermod -aG responses ttvdrops + +sudo chown -R lovinator:responses /mnt/fourteen/Data/Responses +sudo chown -R lovinator:responses /mnt/fourteen/Data/ttvdrops +sudo chmod -R 2775 /mnt/fourteen/Data/Responses +sudo chmod -R 2775 /mnt/fourteen/Data/ttvdrops + +# Import dir +sudo setfacl -b /mnt/fourteen/Data/Responses /mnt/fourteen/Data/Responses/imported +sudo setfacl -m g:responses:rwx /mnt/fourteen/Data/Responses /mnt/fourteen/Data/Responses/imported +sudo setfacl -d -m g:responses:rwx /mnt/fourteen/Data/Responses /mnt/fourteen/Data/Responses/imported + +# Backup dir +sudo setfacl -b /mnt/fourteen/Data/ttvdrops +sudo setfacl -m g:responses:rwx /mnt/fourteen/Data/ttvdrops +sudo setfacl -d -m g:responses:rwx /mnt/fourteen/Data/ttvdrops +``` diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index f755f88..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,89 +0,0 @@ -services: - ttvdrops_postgres: - container_name: ttvdrops_postgres - image: docker.io/pgautoupgrade/pgautoupgrade:latest - command: - - "-c" - - "max_connections=200" - - "-c" - - "shared_buffers=4GB" - - "-c" - - "effective_cache_size=12GB" - - "-c" - - "maintenance_work_mem=1GB" - - "-c" - - "checkpoint_completion_target=0.9" - - "-c" - - "wal_buffers=16MB" - - "-c" - - "default_statistics_target=100" - - "-c" - - "random_page_cost=1.1" - - "-c" - - "effective_io_concurrency=200" - - "-c" - - "work_mem=20MB" - - "-c" - - "huge_pages=off" - - "-c" - - "min_wal_size=1GB" - - "-c" - - "max_wal_size=4GB" - - "-c" - - "max_worker_processes=12" - - "-c" - - "max_parallel_workers_per_gather=4" - - "-c" - - "max_parallel_workers=12" - - "-c" - - "max_parallel_maintenance_workers=4" - environment: - - POSTGRES_DB=ttvdrops - - POSTGRES_USER=ttvdrops - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - ports: - - "5442:5432" - volumes: - - /mnt/Docker/Data/ttvdrops/postgres:/var/lib/postgresql - shm_size: '8gb' - restart: unless-stopped - networks: - - internal - - ttvdrops: - container_name: ttvdrops - image: ghcr.io/thelovinator1/ttvdrops:latest - expose: - - "8000" - user: 1000:1000 - environment: - - DEBUG=False - - DJANGO_SECRET_KEY=$(DJANGO_SECRET_KEY) - - TWITCH_CLIENT_ID=$(TWITCH_CLIENT_ID) - - TWITCH_CLIENT_SECRET=$(TWITCH_CLIENT_SECRET) - - EMAIL_HOST=smtp.gmail.com - - EMAIL_PORT=587 - - EMAIL_HOST_USER=$(EMAIL_HOST_USER) - - EMAIL_HOST_PASSWORD=$(EMAIL_HOST_PASSWORD) - - EMAIL_USE_TLS=True - - EMAIL_USE_SSL=False - - POSTGRES_DB=ttvdrops - - POSTGRES_USER=ttvdrops - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_HOST=ttvdrops_postgres - - POSTGRES_PORT=5432 - volumes: - - /mnt/Docker/Data/ttvdrops/data:/home/ttvdrops/.local/share/TTVDrops - restart: unless-stopped - networks: - - web - - internal - depends_on: - ttvdrops_postgres: - condition: service_started - -networks: - web: - external: true - internal: - driver: bridge diff --git a/pyproject.toml b/pyproject.toml index a006fc5..6126a14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,9 @@ dependencies = [ "pydantic", "pygments", "python-dotenv", + "systemd; sys_platform == 'linux'", "tqdm", + "setproctitle>=1.3.7", ] [dependency-groups] diff --git a/start.sh b/start.sh deleted file mode 100755 index a6fdf33..0000000 --- a/start.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -e - -echo "Running database migrations..." -uv run python manage.py migrate --noinput - -echo "Collecting static files..." -uv run python manage.py collectstatic --noinput - -echo "Starting Django server..." -exec "$@" diff --git a/tools/systemd/ttvdrops-backup.service b/tools/systemd/ttvdrops-backup.service new file mode 100644 index 0000000..c4f73c8 --- /dev/null +++ b/tools/systemd/ttvdrops-backup.service @@ -0,0 +1,10 @@ +[Unit] +Description=TTVDrops database backup + +[Service] +Type=oneshot +User=ttvdrops +Group=ttvdrops +WorkingDirectory=/home/ttvdrops/ttvdrops +EnvironmentFile=/home/ttvdrops/ttvdrops/.env +ExecStart=/usr/bin/uv run python manage.py backup_db diff --git a/tools/systemd/ttvdrops-backup.timer b/tools/systemd/ttvdrops-backup.timer new file mode 100644 index 0000000..1546d5e --- /dev/null +++ b/tools/systemd/ttvdrops-backup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Nightly TTVDrops database backup + +[Timer] +OnCalendar=*-*-* 02:15:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/tools/systemd/ttvdrops-import-drops.service b/tools/systemd/ttvdrops-import-drops.service new file mode 100644 index 0000000..585d10e --- /dev/null +++ b/tools/systemd/ttvdrops-import-drops.service @@ -0,0 +1,12 @@ +[Unit] +Description=TTVDrops import drops from pending directory + +[Service] +Type=oneshot +User=ttvdrops +Group=ttvdrops +WorkingDirectory=/home/ttvdrops/ttvdrops +EnvironmentFile=/home/ttvdrops/ttvdrops/.env +ExecStart=/usr/bin/uv run python manage.py better_import_drops /mnt/fourteen/Data/Responses/pending +-ExecStartPost=/usr/bin/uv run python manage.py download_box_art +-ExecStartPost=/usr/bin/uv run python manage.py download_campaign_images diff --git a/tools/systemd/ttvdrops-import-drops.timer b/tools/systemd/ttvdrops-import-drops.timer new file mode 100644 index 0000000..a715037 --- /dev/null +++ b/tools/systemd/ttvdrops-import-drops.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Frequent TTVDrops import drops timer + +[Timer] +OnBootSec=0 +OnUnitActiveSec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/tools/systemd/ttvdrops.service b/tools/systemd/ttvdrops.service new file mode 100644 index 0000000..de87217 --- /dev/null +++ b/tools/systemd/ttvdrops.service @@ -0,0 +1,32 @@ +[Unit] +Description=TTVDrops +Requires=ttvdrops.socket +After=network.target + +[Service] +Type=simple +User=ttvdrops +Group=ttvdrops +WorkingDirectory=/home/ttvdrops/ttvdrops +EnvironmentFile=/home/ttvdrops/ttvdrops/.env +RuntimeDirectory=ttvdrops +UMask=0077 +ExecStart=/usr/bin/uv run gunicorn config.wsgi:application --bind unix:/run/ttvdrops/ttvdrops.sock --workers 13 --name ttvdrops --max-requests-jitter 50 --max-requests 1200 +ExecReload=/bin/kill -s HUP $MAINPID + +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=full +ProtectHome=no +ReadWritePaths=/home/ttvdrops/ttvdrops /run/ttvdrops /mnt/fourteen/Data/Responses +PrivateDevices=yes +CapabilityBoundingSet= +AmbientCapabilities= +RestrictRealtime=yes +LockPersonality=yes + +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/tools/systemd/ttvdrops.socket b/tools/systemd/ttvdrops.socket new file mode 100644 index 0000000..9b4963d --- /dev/null +++ b/tools/systemd/ttvdrops.socket @@ -0,0 +1,11 @@ +[Unit] +Description=TTVDrops Socket + +[Socket] +ListenStream=/run/ttvdrops/ttvdrops.sock +SocketUser=ttvdrops +SocketGroup=ttvdrops +SocketMode=0660 + +[Install] +WantedBy=sockets.target