From ed8ad1bee930fbd2c9f6aaf498c6938437288047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Sat, 25 Apr 2026 06:34:37 +0200 Subject: [PATCH] Begin Firecracker microVM support --- .github/copilot-instructions.md | 5 +- .gitignore | 4 + CONTRIBUTING.md | 40 +++++++++ Firecracker/get_rootfs.bash | 92 ++++++++++++++++++++ README.md | 4 + config/apps.py | 14 +++ config/checks.py | 59 +++++++++++++ config/settings.py | 1 + pyproject.toml | 7 +- tests/test_dev_command_checks.py | 36 ++++++++ tests/test_dev_command_checks_integration.py | 35 ++++++++ 11 files changed, 290 insertions(+), 7 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100755 Firecracker/get_rootfs.bash create mode 100644 config/apps.py create mode 100644 config/checks.py create mode 100644 tests/test_dev_command_checks.py create mode 100644 tests/test_dev_command_checks_integration.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ca33841..500b62d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -13,7 +13,7 @@ Auto-Clarity: drop caveman for security warnings, irreversible actions, user con # Context: Tussilago A platform to run and host applications, with a focus on Python applications. - **Tech Stack**: Python 3.14+, Django 6, Celery, SQLite (platform), PostgreSQL/Redis (tenants). -- **Infrastructure**: Podman, gVisor (sandboxing), Caddy (reverse proxy/load balancer). +- **Infrastructure**: Firecracker microVMs (sandboxing), Podman (image/build workflow), Caddy (reverse proxy/load balancer). - **Tooling**: `uv` (package manager/runner), `pytest`, `ruff`. ## CRITICAL: Security & Architecture constraints @@ -22,6 +22,7 @@ A platform to run and host applications, with a focus on Python applications. - **Subprocess**: NEVER use `os.system`. MUST use `subprocess.run(..., check=True, capture_output=True, text=True)`. Handle `subprocess.CalledProcessError` explicitly. - **Database IDs**: NEVER expose internal Linux UIDs (integers) to the frontend/API. MUST use `UUIDv4` for external references. - **Caddy**: MUST interact with Caddy strictly via its JSON REST API. +- **Firecracker Isolation**: Tenant workloads MUST run inside Firecracker microVM boundaries. Do not assume host-level process trust; enforce per-VM network/storage isolation and explicit lifecycle control. - **PostgreSQL Isolation**: For shared-tier DBs, use logical isolation. Create roles with `NOSUPERUSER`, `CONNECTION LIMIT`, and configured `statement_timeout`. - **Connection Pooling**: Assume PgBouncer is active. In Django settings, `DISABLE_SERVER_SIDE_CURSORS = True` MUST be set. @@ -67,7 +68,7 @@ A platform to run and host applications, with a focus on Python applications. - **Celery Tasks**: - **Idempotency**: All tasks MUST be idempotent. If a task fails halfway and retries, it MUST NOT corrupt the system or create duplicate containers. - **Serialization**: NEVER pass Django ORM instances as task arguments. Pass `UUID`s or primitive types and refetch the object inside the task. - - **Retries**: Always implement bounded exponential backoff for external interactions (e.g., interacting with the Caddy API or waiting for gVisor). + - **Retries**: Always implement bounded exponential backoff for external interactions (e.g., interacting with the Caddy API or waiting for Firecracker microVM readiness). ## Testing Standards - **Mocking**: NEVER allow test suites to execute real `podman` subprocess calls or make real HTTP requests to Caddy. MUST use `unittest.mock.patch` or `responses`/`httpx-mock` for external boundaries. diff --git a/.gitignore b/.gitignore index 7e534e7..bac7f79 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,7 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Ignore files related to Firecracker microVM, except for get_rootfs.bash which is used in image preparation. +Firecracker/* +!Firecracker/get_rootfs.bash diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..94d8423 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,40 @@ +# Contributing + +## TL;DR + +### 1 Install system packages + +```bash +sudo pacman -Syu --needed \ + base-devel \ + git \ + python \ + uv \ + sqlite \ + curl \ + wget \ + openssh \ + squashfs-tools \ + e2fsprogs \ + caddy \ + podman +``` + +Notes: +- `uv` is package/dependency runner used by repo. +- `squashfs-tools`, `openssh`, and `e2fsprogs` are needed for `get_rootfs.bash` image prep flow. + +### 2 Bootstrap project + +```bash +uv sync +``` + +### 3 Day-to-day commands + +```bash +uv run python manage.py check +uv run pytest -n 5 -q +uv run ruff check . --fix +uv run ruff format . +``` diff --git a/Firecracker/get_rootfs.bash b/Firecracker/get_rootfs.bash new file mode 100755 index 0000000..c4abbe9 --- /dev/null +++ b/Firecracker/get_rootfs.bash @@ -0,0 +1,92 @@ +#!/bin/bash +# This script downloads the latest Linux kernel and rootfs images from Firecracker CI, patches the rootfs with an SSH key, and creates an ext4 filesystem image. + +required_commands=( + uname + curl + basename + grep + sort + tail + wget + unsquashfs + ssh-keygen + sudo + chown + truncate + mkfs.ext4 + e2fsck +) + +missing_commands=() +for cmd in "${required_commands[@]}"; do + if ! command -v "$cmd" >/dev/null 2>&1; then + missing_commands+=("$cmd") + fi +done + +if [ ${#missing_commands[@]} -ne 0 ]; then + echo "ERROR: Missing required program(s): ${missing_commands[*]}" >&2 + echo "Install missing dependencies and rerun." >&2 + exit 1 +fi + +ARCH="$(uname -m)" +release_url="https://github.com/firecracker-microvm/firecracker/releases" +latest_version=$(basename "$(curl -fsSLI -o /dev/null -w "%{url_effective}" "${release_url}/latest")") +CI_VERSION=${latest_version%.*} +latest_kernel_key=$(curl "http://spec.ccfc.min.s3.amazonaws.com/?prefix=firecracker-ci/$CI_VERSION/$ARCH/vmlinux-&list-type=2" \ + | grep -oP "(?<=)(firecracker-ci/$CI_VERSION/$ARCH/vmlinux-[0-9]+\.[0-9]+\.[0-9]{1,3})(?=)" \ + | sort -V | tail -1) + +# Download a linux kernel binary +wget "https://s3.amazonaws.com/spec.ccfc.min/${latest_kernel_key}" + +latest_ubuntu_key=$(curl "http://spec.ccfc.min.s3.amazonaws.com/?prefix=firecracker-ci/$CI_VERSION/$ARCH/ubuntu-&list-type=2" \ + | grep -oP "(?<=)(firecracker-ci/$CI_VERSION/$ARCH/ubuntu-[0-9]+\.[0-9]+\.squashfs)(?=)" \ + | sort -V | tail -1) +ubuntu_version=$(basename "$latest_ubuntu_key" .squashfs | grep -oE '[0-9]+\.[0-9]+') + +# Download a rootfs from Firecracker CI +wget -O "ubuntu-${ubuntu_version}.squashfs.upstream" "https://s3.amazonaws.com/spec.ccfc.min/$latest_ubuntu_key" + +# The rootfs in our CI doesn't contain SSH keys to connect to the VM +# For the purpose of this demo, let's create one and patch it in the rootfs +unsquashfs "ubuntu-${ubuntu_version}.squashfs.upstream" +ssh-keygen -f id_rsa -N "" +cp -v id_rsa.pub squashfs-root/root/.ssh/authorized_keys +mv -v id_rsa "./ubuntu-${ubuntu_version}.id_rsa" +# create ext4 filesystem image +sudo chown -R root:root squashfs-root +truncate -s 1G "ubuntu-${ubuntu_version}.ext4" +sudo mkfs.ext4 -d squashfs-root -F "ubuntu-${ubuntu_version}.ext4" + +# Verify everything was correctly set up and print versions +echo +echo "The following files were downloaded and set up:" +kernel_files=(./vmlinux-*) +if [ -e "${kernel_files[0]}" ]; then + last_kernel_index=$((${#kernel_files[@]} - 1)) + KERNEL="${kernel_files[$last_kernel_index]}" + echo "Kernel: $KERNEL" +else + echo "ERROR: Kernel image does not exist" +fi + +rootfs_files=(./*.ext4) +if [ -e "${rootfs_files[0]}" ]; then + last_rootfs_index=$((${#rootfs_files[@]} - 1)) + ROOTFS="${rootfs_files[$last_rootfs_index]}" + e2fsck -fn "$ROOTFS" &>/dev/null && echo "Rootfs: $ROOTFS" || echo "ERROR: $ROOTFS is not a valid ext4 fs" +else + echo "ERROR: Rootfs image does not exist" +fi + +key_files=(./*.id_rsa) +if [ -e "${key_files[0]}" ]; then + last_key_index=$((${#key_files[@]} - 1)) + KEY_NAME="${key_files[$last_key_index]}" + echo "SSH Key: $KEY_NAME" +else + echo "ERROR: SSH key does not exist" +fi diff --git a/README.md b/README.md index 2e9dc79..a5e5048 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,10 @@ Email: tlovinator@gmail.com Discord: TheLovinator#9276 +# Contributing + +See `CONTRIBUTING.md` for developer setup and workflow. + # License The AGPL license applies to the infrastructure platform (the SaaS backend, the deployment scripts, the web UI). It does not infect the user's code. If a tenant hosts a proprietary, closed-source application on this platform, their code remains completely theirs. diff --git a/config/apps.py b/config/apps.py new file mode 100644 index 0000000..9a8c292 --- /dev/null +++ b/config/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig + + +class TussilagoConfig(AppConfig): + """Application config used to register startup hooks for config package.""" + + # TODO(TheLovinator): Consider creating an app with a more descriptive name for its purpose, e.g. "core" or "common". # noqa: TD003 + + name = "config" + verbose_name = "Tussilago Config" + + def ready(self) -> None: + """Register Django system checks after app loading.""" + import config.checks # pyright: ignore[reportUnusedImport] # noqa: F401, PLC0415 diff --git a/config/checks.py b/config/checks.py new file mode 100644 index 0000000..e8f83cc --- /dev/null +++ b/config/checks.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import shutil +from typing import TYPE_CHECKING + +from django.conf import settings +from django.core.checks import Tags +from django.core.checks import Warning as DjangoWarning +from django.core.checks import register + +if TYPE_CHECKING: + from django.core.checks import CheckMessage + +REQUIRED_DEV_COMMANDS: tuple[str, ...] = ( + "uname", + "curl", + "basename", + "grep", + "sort", + "tail", + "wget", + "unsquashfs", + "ssh-keygen", + "sudo", + "chown", + "truncate", + "mkfs.ext4", + "e2fsck", +) + + +@register(Tags.compatibility) +def check_required_dev_commands(*_: object, **__: object) -> list[CheckMessage]: + """Warn in dev-like runtime when host tooling for init script is missing. + + Returns: + A list of CheckMessage objects representing any issues found. + """ + if not (settings.DEBUG or getattr(settings, "TESTING", False)): + return [] + + missing_commands: list[str] = [ + command for command in REQUIRED_DEV_COMMANDS if shutil.which(command) is None + ] + + if not missing_commands: + return [] + + return [ + DjangoWarning( + "Missing host commands used by get_rootfs.bash.", + hint=( + "Dev-only tooling missing: " + f"{', '.join(missing_commands)}. " + "These are required for local VM/rootfs setup during development, not production runtime." + ), + id="tussilago.W001", + ), + ] diff --git a/config/settings.py b/config/settings.py index fc0c151..f8e40d5 100644 --- a/config/settings.py +++ b/config/settings.py @@ -69,6 +69,7 @@ if DEBUG: INTERNAL_IPS: list[str] = ["127.0.0.1", "localhost"] INSTALLED_APPS: list[str] = [ + "config.apps.TussilagoConfig", "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", diff --git a/pyproject.toml b/pyproject.toml index b8fe5ce..480e7b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,7 @@ version = "0.1.0" description = "A platform to run and host applications, with a focus on Python applications." readme = "README.md" requires-python = ">=3.14" -dependencies = [ - "django>=6.0.4", - "platformdirs>=4.9.6", - "python-dotenv>=1.2.2", -] +dependencies = ["django>=6.0.4", "platformdirs>=4.9.6", "python-dotenv>=1.2.2"] license = "AGPL-3.0-or-later" authors = [{ name = "Joakim Hellsén", email = "tlovinator@gmail.com" }] @@ -99,3 +95,4 @@ lint.ignore = [ DJANGO_SETTINGS_MODULE = "config.settings" python_files = ["test_*.py", "*_test.py"] addopts = "-n 5 -q" +norecursedirs = ["Firecracker"] diff --git a/tests/test_dev_command_checks.py b/tests/test_dev_command_checks.py new file mode 100644 index 0000000..e996457 --- /dev/null +++ b/tests/test_dev_command_checks.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from unittest.mock import patch + +from django.test import override_settings + +from config.checks import check_required_dev_commands + + +@override_settings(DEBUG=False, TESTING=False) +def test_command_check_is_silent_in_production_runtime() -> None: + """Production runtime must not warn about dev-only command dependencies.""" + with patch("config.checks.shutil.which", return_value=None): + warnings = check_required_dev_commands() + + assert warnings == [] + + +@override_settings(DEBUG=True, TESTING=False) +def test_command_check_warns_and_mentions_dev_only_dependencies() -> None: + """Dev runtime must warn and clearly label these dependencies as dev-only.""" + + def fake_which(command: str) -> str | None: + if command == "curl": + return None + return f"/usr/bin/{command}" + + with patch("config.checks.shutil.which", side_effect=fake_which): + warnings = check_required_dev_commands() + + assert len(warnings) == 1 + warning = warnings[0] + assert warning.id == "tussilago.W001" + assert warning.hint is not None + assert "Dev-only tooling missing" in warning.hint + assert "curl" in warning.hint diff --git a/tests/test_dev_command_checks_integration.py b/tests/test_dev_command_checks_integration.py new file mode 100644 index 0000000..08d46b7 --- /dev/null +++ b/tests/test_dev_command_checks_integration.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from io import StringIO +from unittest.mock import patch + +from django.core.management import call_command +from django.test import override_settings + + +@override_settings(DEBUG=True, TESTING=False) +def test_dev_command_check_is_reported_by_check_command() -> None: + """System check must be visible through call_command without direct check import.""" + + def fake_which(command: str) -> str | None: + if command == "curl": + return None + return f"/usr/bin/{command}" + + stderr = StringIO() + with patch("shutil.which", side_effect=fake_which): + call_command("check", "-t", "compatibility", stderr=stderr) + + output = stderr.getvalue() + assert "(tussilago.W001) Missing host commands used by get_rootfs.bash." in output + assert "Dev-only tooling missing: curl." in output + + +@override_settings(DEBUG=False, TESTING=False) +def test_dev_command_check_is_not_reported_in_production_runtime() -> None: + """Production runtime must not report dev-only tooling warning.""" + stderr = StringIO() + with patch("shutil.which", return_value=None): + call_command("check", "-t", "compatibility", stderr=stderr) + + assert "(tussilago.W001)" not in stderr.getvalue()