diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..52aea2d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.11-slim + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# Set work directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + build-essential \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy project +COPY . . + +# Create necessary directories +RUN mkdir -p /app/logs /app/staticfiles /app/media + +# Collect static files +RUN python manage.py collectstatic --noinput + +# Expose port +EXPOSE 8000 + +# Run the application +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] \ No newline at end of file diff --git a/README.md b/README.md index 291f542..e4f0e9e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,193 @@ # postgres-handholder -Django application to manage Postgres clusters, including rolling updates, connection pooling, and more. \ No newline at end of file +Django application to manage Postgres clusters, including rolling updates, connection pooling, and more. + +## Overview + +Postgres hand-holder is a Django application designed to simplify the management of Postgres clusters in Docker and local Linux environments. It provides a user-friendly interface for performing various operations on Postgres databases, such as rolling updates, connection pooling, and backup management. + +postgres-handholder will automate the creation and management of Postgres clusters, making it easier for developers and startups to deploy and maintain their database infrastructure. It is particularly useful for those who want to manage Postgres clusters without the complexity of traditional database management systems. + +## Features + +- Rolling updates on Postgres cluster changes, incl. quick minor version updates +- Database connection pooling with PGBouncer +- Support fast in place major version upgrade. Supports global upgrade of all clusters. +- Restore and cloning Postgres clusters on demand +- Additionally logical backups to S3 or GCS bucket can be configured +- Standby cluster from S3 or GCS WAL archive +- Configurable for non-cloud environments +- Basic credential and user management on Django interface, eases application deployments +- Support for custom TLS certificates +- UI to create and edit Postgres cluster configurations +- Supports PostgreSQL 17, starting from 13+ +- Add easy way to preload libraries: bg_mon, pg_stat_statements, pgextwlist, pg_auth_mon +- Add easy way to include popular Postgres extensions such as decoderbufs, hypopg, pg_cron, pg_partman, pg_stat_kcache, pgq, pgvector, plpgsql_check, postgis, set_user and timescaledb + +## Quick Start + +### Prerequisites + +- Python 3.11+ +- Docker and Docker Compose +- PostgreSQL 13+ (for the Django app database) + +### Installation + +1. Clone the repository: +```bash +git clone https://github.com/yourusername/postgres-handholder.git +cd postgres-handholder +``` + +2. Set up the environment: +```bash +# Copy environment file +cp .env.example .env + +# Edit environment variables +nano .env +``` + +3. Start the application with Docker Compose: +```bash +docker-compose up -d +``` + +4. Run database migrations: +```bash +docker-compose exec web python manage.py migrate +``` + +5. Create a superuser: +```bash +docker-compose exec web python manage.py createsuperuser +``` + +6. Access the application at http://localhost:8000 + +### Development Setup + +1. Create a virtual environment: +```bash +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +``` + +2. Install dependencies: +```bash +pip install -r requirements.txt +``` + +3. Set up the database: +```bash +# Create PostgreSQL database +createdb postgres_handholder + +# Run migrations +python manage.py migrate + +# Create superuser +python manage.py createsuperuser +``` + +4. Start the development server: +```bash +python manage.py runserver +``` + +## Configuration + +### Environment Variables + +Create a `.env` file with the following variables: + +```env +# Django +DEBUG=True +SECRET_KEY=your-secret-key-here +ALLOWED_HOSTS=localhost,127.0.0.1 + +# Database +DB_NAME=postgres_handholder +DB_USER=postgres +DB_PASSWORD=your-password +DB_HOST=localhost +DB_PORT=5432 + +# Celery +CELERY_BROKER_URL=redis://localhost:6379/0 +CELERY_RESULT_BACKEND=redis://localhost:6379/0 + +# Cloud Storage (optional) +AWS_ACCESS_KEY_ID=your-aws-key +AWS_SECRET_ACCESS_KEY=your-aws-secret +AWS_STORAGE_BUCKET_NAME=your-bucket +AWS_S3_REGION_NAME=us-east-1 + +GOOGLE_CLOUD_STORAGE_BUCKET=your-gcs-bucket +GOOGLE_APPLICATION_CREDENTIALS=path/to/credentials.json +``` + +## Usage + +### Creating a Cluster + +1. Navigate to the Clusters section +2. Click "Create Cluster" +3. Fill in the configuration details: + - Cluster name and description + - PostgreSQL version (13-17) + - Deployment type (Docker, Kubernetes, Local) + - Resource limits (CPU, Memory, Storage) + - Network configuration + - Extensions and libraries + +### Managing Clusters + +- **Start/Stop**: Control cluster lifecycle +- **Update**: Perform rolling updates +- **Monitor**: View real-time metrics and logs +- **Backup**: Create and manage backups +- **Users**: Manage database users and permissions +- **Databases**: Create and manage databases + +### Backup Management + +- Create automated backups +- Store backups in S3 or Google Cloud Storage +- Restore clusters from backups +- Clone clusters for testing + +## Architecture + +The application is built with: + +- **Django 4.2+**: Web framework +- **PostgreSQL**: Primary database +- **Redis**: Message broker for Celery +- **Celery**: Background task processing +- **Docker**: Containerization +- **Bootstrap 5**: UI framework + +### Apps Structure + +- `clusters/`: Core cluster management +- `backups/`: Backup and restore functionality +- `monitoring/`: Metrics and monitoring + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests +5. Submit a pull request + +## License + +This project is licensed under the GNU General Public License v3.0 - see the [LICENSE](LICENSE) file for details. + +## Support + +For support and questions, please open an issue on GitHub. diff --git a/backups/__init__.py b/backups/__init__.py new file mode 100644 index 0000000..e8ea7c6 --- /dev/null +++ b/backups/__init__.py @@ -0,0 +1 @@ +# Backups app for managing Postgres backups \ No newline at end of file diff --git a/backups/apps.py b/backups/apps.py new file mode 100644 index 0000000..5a16c7b --- /dev/null +++ b/backups/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class BackupsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'backups' + verbose_name = 'Database Backups' \ No newline at end of file diff --git a/backups/urls.py b/backups/urls.py new file mode 100644 index 0000000..a906a11 --- /dev/null +++ b/backups/urls.py @@ -0,0 +1,13 @@ +from django.urls import path +from . import views + +app_name = 'backups' + +urlpatterns = [ + path('', views.backup_list, name='backup_list'), + path('create/', views.backup_create, name='backup_create'), + path('/', views.backup_detail, name='backup_detail'), + path('/download/', views.backup_download, name='backup_download'), + path('/restore/', views.backup_restore, name='backup_restore'), + path('/delete/', views.backup_delete, name='backup_delete'), +] \ No newline at end of file diff --git a/clusters/__init__.py b/clusters/__init__.py new file mode 100644 index 0000000..5db5a7b --- /dev/null +++ b/clusters/__init__.py @@ -0,0 +1 @@ +# Clusters app for managing Postgres clusters \ No newline at end of file diff --git a/clusters/apps.py b/clusters/apps.py new file mode 100644 index 0000000..59b64b7 --- /dev/null +++ b/clusters/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class ClustersConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'clusters' + verbose_name = 'Postgres Clusters' \ No newline at end of file diff --git a/clusters/forms.py b/clusters/forms.py new file mode 100644 index 0000000..6aceb44 --- /dev/null +++ b/clusters/forms.py @@ -0,0 +1,152 @@ +from django import forms +from django.contrib.auth.models import User +from .models import PostgresCluster, ClusterUser, ClusterDatabase + + +class PostgresClusterForm(forms.ModelForm): + """Form for creating and editing Postgres clusters.""" + + class Meta: + model = PostgresCluster + fields = [ + 'name', 'description', 'cluster_type', 'deployment_type', + 'postgres_version', 'port', 'data_directory', + 'cpu_limit', 'memory_limit', 'storage_size', + 'host', 'external_port', 'admin_user', 'admin_password', + 'extensions', 'libraries', 'tls_enabled', 'tls_cert_path', + 'tls_key_path', 'tls_ca_path', 'pgpool_enabled', 'pgpool_instances' + ] + widgets = { + 'name': forms.TextInput(attrs={'class': 'form-control'}), + 'description': forms.Textarea(attrs={'class': 'form-control', 'rows': 3}), + 'cluster_type': forms.Select(attrs={'class': 'form-select'}), + 'deployment_type': forms.Select(attrs={'class': 'form-select'}), + 'postgres_version': forms.Select(attrs={'class': 'form-select'}, choices=[ + ('13', 'PostgreSQL 13'), + ('14', 'PostgreSQL 14'), + ('15', 'PostgreSQL 15'), + ('16', 'PostgreSQL 16'), + ('17', 'PostgreSQL 17'), + ]), + 'port': forms.NumberInput(attrs={'class': 'form-control'}), + 'data_directory': forms.TextInput(attrs={'class': 'form-control'}), + 'cpu_limit': forms.TextInput(attrs={'class': 'form-control'}), + 'memory_limit': forms.TextInput(attrs={'class': 'form-control'}), + 'storage_size': forms.TextInput(attrs={'class': 'form-control'}), + 'host': forms.TextInput(attrs={'class': 'form-control'}), + 'external_port': forms.NumberInput(attrs={'class': 'form-control'}), + 'admin_user': forms.TextInput(attrs={'class': 'form-control'}), + 'admin_password': forms.PasswordInput(attrs={'class': 'form-control'}), + 'tls_cert_path': forms.TextInput(attrs={'class': 'form-control'}), + 'tls_key_path': forms.TextInput(attrs={'class': 'form-control'}), + 'tls_ca_path': forms.TextInput(attrs={'class': 'form-control'}), + 'pgpool_instances': forms.NumberInput(attrs={'class': 'form-control'}), + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Set default values for extensions and libraries + if not self.instance.pk: + self.fields['extensions'].initial = [ + 'pg_stat_statements', + 'pg_auth_mon', + ] + self.fields['libraries'].initial = [ + 'bg_mon', + 'pg_stat_statements', + 'pgextwlist', + 'pg_auth_mon', + ] + + def clean_name(self): + name = self.cleaned_data['name'] + if not name.replace('-', '').replace('_', '').isalnum(): + raise forms.ValidationError( + 'Cluster name can only contain letters, numbers, hyphens, and underscores.' + ) + return name + + def clean_admin_password(self): + password = self.cleaned_data['admin_password'] + if len(password) < 8: + raise forms.ValidationError( + 'Admin password must be at least 8 characters long.' + ) + return password + + +class ClusterUserForm(forms.ModelForm): + """Form for creating and editing cluster users.""" + + confirm_password = forms.CharField( + widget=forms.PasswordInput(attrs={'class': 'form-control'}), + help_text='Confirm the password' + ) + + class Meta: + model = ClusterUser + fields = [ + 'username', 'password', 'is_superuser', 'can_create_db', + 'can_login', 'permissions' + ] + widgets = { + 'username': forms.TextInput(attrs={'class': 'form-control'}), + 'password': forms.PasswordInput(attrs={'class': 'form-control'}), + 'is_superuser': forms.CheckboxInput(attrs={'class': 'form-check-input'}), + 'can_create_db': forms.CheckboxInput(attrs={'class': 'form-check-input'}), + 'can_login': forms.CheckboxInput(attrs={'class': 'form-check-input'}), + 'permissions': forms.Textarea(attrs={'class': 'form-control', 'rows': 3}), + } + + def clean_username(self): + username = self.cleaned_data['username'] + if not username.replace('_', '').isalnum(): + raise forms.ValidationError( + 'Username can only contain letters, numbers, and underscores.' + ) + return username + + def clean(self): + cleaned_data = super().clean() + password = cleaned_data.get('password') + confirm_password = cleaned_data.get('confirm_password') + + if password and confirm_password and password != confirm_password: + raise forms.ValidationError('Passwords do not match.') + + return cleaned_data + + +class ClusterDatabaseForm(forms.ModelForm): + """Form for creating and editing cluster databases.""" + + class Meta: + model = ClusterDatabase + fields = ['name', 'owner', 'encoding', 'collation', 'ctype'] + widgets = { + 'name': forms.TextInput(attrs={'class': 'form-control'}), + 'owner': forms.Select(attrs={'class': 'form-select'}), + 'encoding': forms.Select(attrs={'class': 'form-select'}, choices=[ + ('UTF8', 'UTF8'), + ('LATIN1', 'LATIN1'), + ('SQL_ASCII', 'SQL_ASCII'), + ]), + 'collation': forms.TextInput(attrs={'class': 'form-control'}), + 'ctype': forms.TextInput(attrs={'class': 'form-control'}), + } + + def __init__(self, *args, **kwargs): + cluster = kwargs.pop('cluster', None) + super().__init__(*args, **kwargs) + + if cluster: + self.fields['owner'].queryset = cluster.users.all() + + def clean_name(self): + name = self.cleaned_data['name'] + if not name.replace('_', '').isalnum(): + raise forms.ValidationError( + 'Database name can only contain letters, numbers, and underscores.' + ) + return name \ No newline at end of file diff --git a/clusters/models.py b/clusters/models.py new file mode 100644 index 0000000..683b085 --- /dev/null +++ b/clusters/models.py @@ -0,0 +1,200 @@ +from django.db import models +from django.contrib.auth.models import User +from django.core.validators import MinValueValidator, MaxValueValidator +import json + + +class PostgresCluster(models.Model): + """Model for managing Postgres cluster configurations.""" + + CLUSTER_TYPES = [ + ('single', 'Single Instance'), + ('replica', 'Primary with Replicas'), + ('standby', 'Standby Cluster'), + ] + + DEPLOYMENT_TYPES = [ + ('docker', 'Docker'), + ('kubernetes', 'Kubernetes'), + ('local', 'Local Linux'), + ] + + name = models.CharField(max_length=100, unique=True) + description = models.TextField(blank=True) + cluster_type = models.CharField(max_length=20, choices=CLUSTER_TYPES, default='single') + deployment_type = models.CharField(max_length=20, choices=DEPLOYMENT_TYPES, default='docker') + + # PostgreSQL Configuration + postgres_version = models.CharField(max_length=10, default='15') + port = models.IntegerField(default=5432, validators=[MinValueValidator(1024), MaxValueValidator(65535)]) + data_directory = models.CharField(max_length=255, default='/var/lib/postgresql/data') + + # Resource Configuration + cpu_limit = models.CharField(max_length=20, default='2') + memory_limit = models.CharField(max_length=20, default='4Gi') + storage_size = models.CharField(max_length=20, default='10Gi') + + # Network Configuration + host = models.CharField(max_length=255, default='localhost') + external_port = models.IntegerField(default=5432, validators=[MinValueValidator(1024), MaxValueValidator(65535)]) + + # Authentication + admin_user = models.CharField(max_length=50, default='postgres') + admin_password = models.CharField(max_length=255) + + # Extensions and Libraries + extensions = models.JSONField(default=list, blank=True) + libraries = models.JSONField(default=list, blank=True) + + # TLS Configuration + tls_enabled = models.BooleanField(default=False) + tls_cert_path = models.CharField(max_length=255, blank=True) + tls_key_path = models.CharField(max_length=255, blank=True) + tls_ca_path = models.CharField(max_length=255, blank=True) + + # Connection Pooling + pgpool_enabled = models.BooleanField(default=False) + pgpool_instances = models.IntegerField(default=1, validators=[MinValueValidator(1), MaxValueValidator(10)]) + + # Status + status = models.CharField(max_length=20, default='stopped', choices=[ + ('running', 'Running'), + ('stopped', 'Stopped'), + ('starting', 'Starting'), + ('stopping', 'Stopping'), + ('error', 'Error'), + ('updating', 'Updating'), + ]) + + # Metadata + created_by = models.ForeignKey(User, on_delete=models.CASCADE) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ['-created_at'] + + def __str__(self): + return self.name + + def get_connection_string(self): + """Get the connection string for this cluster.""" + return f"postgresql://{self.admin_user}:{self.admin_password}@{self.host}:{self.external_port}/postgres" + + def get_extensions_list(self): + """Get the list of enabled extensions.""" + return self.extensions if isinstance(self.extensions, list) else [] + + def get_libraries_list(self): + """Get the list of enabled libraries.""" + return self.libraries if isinstance(self.libraries, list) else [] + + +class PostgresInstance(models.Model): + """Model for managing individual Postgres instances within a cluster.""" + + INSTANCE_TYPES = [ + ('primary', 'Primary'), + ('replica', 'Replica'), + ('standby', 'Standby'), + ] + + cluster = models.ForeignKey(PostgresCluster, on_delete=models.CASCADE, related_name='instances') + name = models.CharField(max_length=100) + instance_type = models.CharField(max_length=20, choices=INSTANCE_TYPES, default='primary') + + # Instance Configuration + host = models.CharField(max_length=255) + port = models.IntegerField(validators=[MinValueValidator(1024), MaxValueValidator(65535)]) + data_directory = models.CharField(max_length=255) + + # Status + status = models.CharField(max_length=20, default='stopped', choices=[ + ('running', 'Running'), + ('stopped', 'Stopped'), + ('starting', 'Starting'), + ('stopping', 'Stopping'), + ('error', 'Error'), + ('syncing', 'Syncing'), + ]) + + # Replication Configuration (for replicas) + replication_slot = models.CharField(max_length=100, blank=True) + lag_seconds = models.IntegerField(default=0) + + # Resource Usage + cpu_usage = models.FloatField(default=0.0) + memory_usage = models.FloatField(default=0.0) + disk_usage = models.FloatField(default=0.0) + + # Metadata + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ['cluster', 'name'] + ordering = ['instance_type', 'name'] + + def __str__(self): + return f"{self.cluster.name} - {self.name}" + + +class ClusterUser(models.Model): + """Model for managing database users within clusters.""" + + cluster = models.ForeignKey(PostgresCluster, on_delete=models.CASCADE, related_name='users') + username = models.CharField(max_length=50) + password = models.CharField(max_length=255) + is_superuser = models.BooleanField(default=False) + can_create_db = models.BooleanField(default=False) + can_login = models.BooleanField(default=True) + + # Permissions + permissions = models.JSONField(default=dict, blank=True) + + # Metadata + created_by = models.ForeignKey(User, on_delete=models.CASCADE) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ['cluster', 'username'] + ordering = ['username'] + + def __str__(self): + return f"{self.cluster.name} - {self.username}" + + +class ClusterDatabase(models.Model): + """Model for managing databases within clusters.""" + + cluster = models.ForeignKey(PostgresCluster, on_delete=models.CASCADE, related_name='databases') + name = models.CharField(max_length=50) + owner = models.ForeignKey(ClusterUser, on_delete=models.CASCADE) + + # Configuration + encoding = models.CharField(max_length=20, default='UTF8') + collation = models.CharField(max_length=50, default='en_US.utf8') + ctype = models.CharField(max_length=50, default='en_US.utf8') + + # Size tracking + size_bytes = models.BigIntegerField(default=0) + + # Metadata + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = ['cluster', 'name'] + ordering = ['name'] + + def __str__(self): + return f"{self.cluster.name} - {self.name}" + + def get_size_mb(self): + """Get database size in MB.""" + return round(self.size_bytes / (1024 * 1024), 2) + + def get_size_gb(self): + """Get database size in GB.""" + return round(self.size_bytes / (1024 * 1024 * 1024), 2) \ No newline at end of file diff --git a/clusters/urls.py b/clusters/urls.py new file mode 100644 index 0000000..633a9ad --- /dev/null +++ b/clusters/urls.py @@ -0,0 +1,19 @@ +from django.urls import path +from . import views + +app_name = 'clusters' + +urlpatterns = [ + path('', views.cluster_list, name='cluster_list'), + path('create/', views.cluster_create, name='cluster_create'), + path('/', views.cluster_detail, name='cluster_detail'), + path('/edit/', views.cluster_edit, name='cluster_edit'), + path('/delete/', views.cluster_delete, name='cluster_delete'), + path('/start/', views.cluster_start, name='cluster_start'), + path('/stop/', views.cluster_stop, name='cluster_stop'), + path('/restart/', views.cluster_restart, name='cluster_restart'), + path('/update/', views.cluster_update, name='cluster_update'), + path('/users/', views.cluster_users, name='cluster_users'), + path('/databases/', views.cluster_databases, name='cluster_databases'), + path('/instances/', views.cluster_instances, name='cluster_instances'), +] \ No newline at end of file diff --git a/clusters/views.py b/clusters/views.py new file mode 100644 index 0000000..913c364 --- /dev/null +++ b/clusters/views.py @@ -0,0 +1,261 @@ +from django.shortcuts import render, get_object_or_404, redirect +from django.contrib.auth.decorators import login_required +from django.contrib import messages +from django.http import JsonResponse +from django.views.decorators.http import require_http_methods +from django.core.paginator import Paginator +from .models import PostgresCluster, PostgresInstance, ClusterUser, ClusterDatabase +from .forms import PostgresClusterForm, ClusterUserForm, ClusterDatabaseForm + + +@login_required +def cluster_list(request): + """Display list of all clusters.""" + clusters = PostgresCluster.objects.filter(created_by=request.user) + + # Filtering + status_filter = request.GET.get('status') + if status_filter: + clusters = clusters.filter(status=status_filter) + + # Pagination + paginator = Paginator(clusters, 10) + page_number = request.GET.get('page') + page_obj = paginator.get_page(page_number) + + context = { + 'page_obj': page_obj, + 'status_choices': PostgresCluster._meta.get_field('status').choices, + } + return render(request, 'clusters/cluster_list.html', context) + + +@login_required +def cluster_create(request): + """Create a new Postgres cluster.""" + if request.method == 'POST': + form = PostgresClusterForm(request.POST) + if form.is_valid(): + cluster = form.save(commit=False) + cluster.created_by = request.user + cluster.save() + messages.success(request, f'Cluster "{cluster.name}" created successfully.') + return redirect('clusters:cluster_detail', cluster_id=cluster.id) + else: + form = PostgresClusterForm() + + context = { + 'form': form, + 'title': 'Create New Cluster', + } + return render(request, 'clusters/cluster_form.html', context) + + +@login_required +def cluster_detail(request, cluster_id): + """Display detailed information about a cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + context = { + 'cluster': cluster, + 'instances': cluster.instances.all(), + 'users': cluster.users.all(), + 'databases': cluster.databases.all(), + } + return render(request, 'clusters/cluster_detail.html', context) + + +@login_required +def cluster_edit(request, cluster_id): + """Edit an existing cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + if request.method == 'POST': + form = PostgresClusterForm(request.POST, instance=cluster) + if form.is_valid(): + form.save() + messages.success(request, f'Cluster "{cluster.name}" updated successfully.') + return redirect('clusters:cluster_detail', cluster_id=cluster.id) + else: + form = PostgresClusterForm(instance=cluster) + + context = { + 'form': form, + 'cluster': cluster, + 'title': f'Edit Cluster: {cluster.name}', + } + return render(request, 'clusters/cluster_form.html', context) + + +@login_required +def cluster_delete(request, cluster_id): + """Delete a cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + if request.method == 'POST': + cluster_name = cluster.name + cluster.delete() + messages.success(request, f'Cluster "{cluster_name}" deleted successfully.') + return redirect('clusters:cluster_list') + + context = { + 'cluster': cluster, + } + return render(request, 'clusters/cluster_confirm_delete.html', context) + + +@login_required +@require_http_methods(["POST"]) +def cluster_start(request, cluster_id): + """Start a cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + try: + # TODO: Implement actual cluster start logic + cluster.status = 'starting' + cluster.save() + messages.success(request, f'Starting cluster "{cluster.name}"...') + except Exception as e: + messages.error(request, f'Failed to start cluster: {str(e)}') + + return redirect('clusters:cluster_detail', cluster_id=cluster.id) + + +@login_required +@require_http_methods(["POST"]) +def cluster_stop(request, cluster_id): + """Stop a cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + try: + # TODO: Implement actual cluster stop logic + cluster.status = 'stopping' + cluster.save() + messages.success(request, f'Stopping cluster "{cluster.name}"...') + except Exception as e: + messages.error(request, f'Failed to stop cluster: {str(e)}') + + return redirect('clusters:cluster_detail', cluster_id=cluster.id) + + +@login_required +@require_http_methods(["POST"]) +def cluster_restart(request, cluster_id): + """Restart a cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + try: + # TODO: Implement actual cluster restart logic + cluster.status = 'starting' + cluster.save() + messages.success(request, f'Restarting cluster "{cluster.name}"...') + except Exception as e: + messages.error(request, f'Failed to restart cluster: {str(e)}') + + return redirect('clusters:cluster_detail', cluster_id=cluster.id) + + +@login_required +@require_http_methods(["POST"]) +def cluster_update(request, cluster_id): + """Update a cluster (rolling update).""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + try: + # TODO: Implement actual rolling update logic + cluster.status = 'updating' + cluster.save() + messages.success(request, f'Updating cluster "{cluster.name}"...') + except Exception as e: + messages.error(request, f'Failed to update cluster: {str(e)}') + + return redirect('clusters:cluster_detail', cluster_id=cluster.id) + + +@login_required +def cluster_users(request, cluster_id): + """Manage users for a cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + if request.method == 'POST': + form = ClusterUserForm(request.POST) + if form.is_valid(): + user = form.save(commit=False) + user.cluster = cluster + user.created_by = request.user + user.save() + messages.success(request, f'User "{user.username}" created successfully.') + return redirect('clusters:cluster_users', cluster_id=cluster.id) + else: + form = ClusterUserForm() + + context = { + 'cluster': cluster, + 'form': form, + 'users': cluster.users.all(), + } + return render(request, 'clusters/cluster_users.html', context) + + +@login_required +def cluster_databases(request, cluster_id): + """Manage databases for a cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + if request.method == 'POST': + form = ClusterDatabaseForm(request.POST) + if form.is_valid(): + database = form.save(commit=False) + database.cluster = cluster + database.save() + messages.success(request, f'Database "{database.name}" created successfully.') + return redirect('clusters:cluster_databases', cluster_id=cluster.id) + else: + form = ClusterDatabaseForm() + + context = { + 'cluster': cluster, + 'form': form, + 'databases': cluster.databases.all(), + } + return render(request, 'clusters/cluster_databases.html', context) + + +@login_required +def cluster_instances(request, cluster_id): + """View instances for a cluster.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + context = { + 'cluster': cluster, + 'instances': cluster.instances.all(), + } + return render(request, 'clusters/cluster_instances.html', context) + + +@login_required +def cluster_status_api(request, cluster_id): + """API endpoint to get cluster status.""" + cluster = get_object_or_404(PostgresCluster, id=cluster_id, created_by=request.user) + + data = { + 'id': cluster.id, + 'name': cluster.name, + 'status': cluster.status, + 'instances': [ + { + 'id': instance.id, + 'name': instance.name, + 'type': instance.instance_type, + 'status': instance.status, + 'host': instance.host, + 'port': instance.port, + 'cpu_usage': instance.cpu_usage, + 'memory_usage': instance.memory_usage, + 'disk_usage': instance.disk_usage, + } + for instance in cluster.instances.all() + ] + } + + return JsonResponse(data) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e622438 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,102 @@ +version: '3.8' + +services: + # PostgreSQL database for the Django app + db: + image: postgres:15 + environment: + POSTGRES_DB: postgres_handholder + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + # Redis for Celery + redis: + image: redis:7-alpine + ports: + - "6379:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # Django web application + web: + build: . + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/app + - static_volume:/app/staticfiles + - media_volume:/app/media + ports: + - "8000:8000" + environment: + - DEBUG=True + - DB_HOST=db + - DB_NAME=postgres_handholder + - DB_USER=postgres + - DB_PASSWORD=postgres + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + + # Celery worker + celery: + build: . + command: celery -A postgres_handholder worker -l info + volumes: + - .:/app + environment: + - DEBUG=True + - DB_HOST=db + - DB_NAME=postgres_handholder + - DB_USER=postgres + - DB_PASSWORD=postgres + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + + # Celery beat scheduler + celery-beat: + build: . + command: celery -A postgres_handholder beat -l info + volumes: + - .:/app + environment: + - DEBUG=True + - DB_HOST=db + - DB_NAME=postgres_handholder + - DB_USER=postgres + - DB_PASSWORD=postgres + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + +volumes: + postgres_data: + redis_data: + static_volume: + media_volume: \ No newline at end of file diff --git a/env.example b/env.example new file mode 100644 index 0000000..2d2bad7 --- /dev/null +++ b/env.example @@ -0,0 +1,31 @@ +# Django Configuration +DEBUG=True +SECRET_KEY=your-secret-key-here-change-this-in-production +ALLOWED_HOSTS=localhost,127.0.0.1 + +# Database Configuration +DB_NAME=postgres_handholder +DB_USER=postgres +DB_PASSWORD=your-database-password +DB_HOST=localhost +DB_PORT=5432 + +# Celery Configuration +CELERY_BROKER_URL=redis://localhost:6379/0 +CELERY_RESULT_BACKEND=redis://localhost:6379/0 + +# AWS S3 Configuration (optional) +AWS_ACCESS_KEY_ID=your-aws-access-key +AWS_SECRET_ACCESS_KEY=your-aws-secret-key +AWS_STORAGE_BUCKET_NAME=your-backup-bucket +AWS_S3_REGION_NAME=us-east-1 + +# Google Cloud Storage Configuration (optional) +GOOGLE_CLOUD_STORAGE_BUCKET=your-gcs-bucket +GOOGLE_APPLICATION_CREDENTIALS=path/to/your/credentials.json + +# Docker Configuration +DOCKER_HOST=unix://var/run/docker.sock + +# Kubernetes Configuration (optional) +KUBECONFIG=path/to/your/kubeconfig \ No newline at end of file diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..7c22157 --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'postgres_handholder.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/monitoring/__init__.py b/monitoring/__init__.py new file mode 100644 index 0000000..6fc279f --- /dev/null +++ b/monitoring/__init__.py @@ -0,0 +1 @@ +# Monitoring app for Postgres cluster monitoring \ No newline at end of file diff --git a/monitoring/apps.py b/monitoring/apps.py new file mode 100644 index 0000000..f89febf --- /dev/null +++ b/monitoring/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class MonitoringConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'monitoring' + verbose_name = 'Cluster Monitoring' \ No newline at end of file diff --git a/monitoring/urls.py b/monitoring/urls.py new file mode 100644 index 0000000..6fb1e1b --- /dev/null +++ b/monitoring/urls.py @@ -0,0 +1,12 @@ +from django.urls import path +from . import views + +app_name = 'monitoring' + +urlpatterns = [ + path('', views.dashboard, name='dashboard'), + path('clusters/', views.cluster_metrics, name='cluster_metrics'), + path('clusters//', views.cluster_detail_metrics, name='cluster_detail_metrics'), + path('alerts/', views.alerts, name='alerts'), + path('logs/', views.logs, name='logs'), +] \ No newline at end of file diff --git a/postgres_handholder/__init__.py b/postgres_handholder/__init__.py new file mode 100644 index 0000000..8eff6b8 --- /dev/null +++ b/postgres_handholder/__init__.py @@ -0,0 +1 @@ +# Postgres Handholder Django Application \ No newline at end of file diff --git a/postgres_handholder/asgi.py b/postgres_handholder/asgi.py new file mode 100644 index 0000000..420cdce --- /dev/null +++ b/postgres_handholder/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for postgres_handholder project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'postgres_handholder.settings') + +application = get_asgi_application() \ No newline at end of file diff --git a/postgres_handholder/celery.py b/postgres_handholder/celery.py new file mode 100644 index 0000000..a22dcc2 --- /dev/null +++ b/postgres_handholder/celery.py @@ -0,0 +1,19 @@ +import os +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'postgres_handholder.settings') + +app = Celery('postgres_handholder') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): + print(f'Request: {self.request!r}') \ No newline at end of file diff --git a/postgres_handholder/settings.py b/postgres_handholder/settings.py new file mode 100644 index 0000000..94d8f1e --- /dev/null +++ b/postgres_handholder/settings.py @@ -0,0 +1,193 @@ +""" +Django settings for postgres_handholder project. +""" + +import os +from pathlib import Path +from django.core.management.utils import get_random_secret_key + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get('SECRET_KEY', get_random_secret_key()) + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = os.environ.get('DEBUG', 'True').lower() == 'true' + +ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', 'localhost,127.0.0.1').split(',') + +# Application definition +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.sites', + + # Third party apps + 'allauth', + 'allauth.account', + 'allauth.socialaccount', + 'crispy_forms', + 'crispy_bootstrap5', + 'django_extensions', + 'debug_toolbar', + + # Local apps + 'clusters', + 'backups', + 'monitoring', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'allauth.account.middleware.AccountMiddleware', + 'debug_toolbar.middleware.DebugToolbarMiddleware', +] + +ROOT_URLCONF = 'postgres_handholder.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [BASE_DIR / 'templates'], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'postgres_handholder.wsgi.application' + +# Database +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': os.environ.get('DB_NAME', 'postgres_handholder'), + 'USER': os.environ.get('DB_USER', 'postgres'), + 'PASSWORD': os.environ.get('DB_PASSWORD', ''), + 'HOST': os.environ.get('DB_HOST', 'localhost'), + 'PORT': os.environ.get('DB_PORT', '5432'), + } +} + +# Password validation +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# Internationalization +LANGUAGE_CODE = 'en-us' +TIME_ZONE = 'UTC' +USE_I18N = True +USE_TZ = True + +# Static files (CSS, JavaScript, Images) +STATIC_URL = 'static/' +STATIC_ROOT = BASE_DIR / 'staticfiles' +STATICFILES_DIRS = [ + BASE_DIR / 'static', +] + +# Media files +MEDIA_URL = 'media/' +MEDIA_ROOT = BASE_DIR / 'media' + +# Default primary key field type +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +# Crispy Forms +CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap5" +CRISPY_TEMPLATE_PACK = "bootstrap5" + +# Django Allauth +AUTHENTICATION_BACKENDS = [ + 'django.contrib.auth.backends.ModelBackend', + 'allauth.account.auth_backends.AuthenticationBackend', +] + +SITE_ID = 1 + +ACCOUNT_EMAIL_REQUIRED = True +ACCOUNT_USERNAME_REQUIRED = False +ACCOUNT_AUTHENTICATION_METHOD = 'email' +ACCOUNT_EMAIL_VERIFICATION = 'mandatory' + +# Celery Configuration +CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379/0') +CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0') +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = TIME_ZONE + +# Cloud Storage Configuration +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME', 'us-east-1') + +# Google Cloud Storage Configuration +GOOGLE_CLOUD_STORAGE_BUCKET = os.environ.get('GOOGLE_CLOUD_STORAGE_BUCKET') +GOOGLE_APPLICATION_CREDENTIALS = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + +# Docker Configuration +DOCKER_HOST = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock') + +# Kubernetes Configuration +KUBERNETES_CONFIG_PATH = os.environ.get('KUBECONFIG') + +# Debug Toolbar +INTERNAL_IPS = [ + '127.0.0.1', +] + +# Logging +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', + 'style': '{', + }, + }, + 'handlers': { + 'file': { + 'level': 'INFO', + 'class': 'logging.FileHandler', + 'filename': BASE_DIR / 'logs' / 'django.log', + 'formatter': 'verbose', + }, + }, + 'root': { + 'handlers': ['file'], + 'level': 'INFO', + }, +} \ No newline at end of file diff --git a/postgres_handholder/urls.py b/postgres_handholder/urls.py new file mode 100644 index 0000000..e10ad77 --- /dev/null +++ b/postgres_handholder/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for postgres_handholder project. +""" +from django.contrib import admin +from django.urls import path, include +from django.conf import settings +from django.conf.urls.static import static + +urlpatterns = [ + path('admin/', admin.site.urls), + path('accounts/', include('allauth.urls')), + path('', include('clusters.urls')), + path('backups/', include('backups.urls')), + path('monitoring/', include('monitoring.urls')), +] + +if settings.DEBUG: + urlpatterns += [ + path('__debug__/', include('debug_toolbar.urls')), + ] + urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) \ No newline at end of file diff --git a/postgres_handholder/wsgi.py b/postgres_handholder/wsgi.py new file mode 100644 index 0000000..e668ec6 --- /dev/null +++ b/postgres_handholder/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for postgres_handholder project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'postgres_handholder.settings') + +application = get_wsgi_application() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7a5428b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +Django>=4.2.0,<5.0 +psycopg2-binary>=2.9.0 +django-environ>=0.11.0 +celery>=5.3.0 +redis>=4.5.0 +boto3>=1.26.0 +google-cloud-storage>=2.8.0 +docker>=6.1.0 +kubernetes>=26.1.0 +django-crispy-forms>=2.0 +crispy-bootstrap5>=0.7 +django-allauth>=0.54.0 +django-extensions>=3.2.0 +django-debug-toolbar>=4.0.0 +pytest-django>=4.5.0 +black>=23.0.0 +flake8>=6.0.0 +isort>=5.12.0 \ No newline at end of file diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..a10c6ef --- /dev/null +++ b/templates/base.html @@ -0,0 +1,96 @@ + + + + + + {% block title %}Postgres Handholder{% endblock %} + + + + + + + {% block extra_css %}{% endblock %} + + + + + + +
+ + {% if messages %} + {% for message in messages %} + + {% endfor %} + {% endif %} + + + {% block content %}{% endblock %} +
+ + +
+
+

+ Postgres Handholder - Simplify Postgres cluster management +

+
+
+ + + + + {% block extra_js %}{% endblock %} + + \ No newline at end of file diff --git a/templates/clusters/cluster_list.html b/templates/clusters/cluster_list.html new file mode 100644 index 0000000..1572041 --- /dev/null +++ b/templates/clusters/cluster_list.html @@ -0,0 +1,134 @@ +{% extends 'base.html' %} + +{% block title %}Postgres Clusters - Postgres Handholder{% endblock %} + +{% block content %} +
+

Postgres Clusters

+ + Create Cluster + +
+ + +
+
+
+
+ + +
+
+ + + Clear + +
+
+
+
+ + +{% if page_obj %} +
+ {% for cluster in page_obj %} +
+
+
+
{{ cluster.name }}
+ + {{ cluster.get_status_display }} + +
+
+

{{ cluster.description|truncatewords:20 }}

+ +
+
+ Type +
{{ cluster.get_cluster_type_display }}
+
+
+ Version +
PostgreSQL {{ cluster.postgres_version }}
+
+
+ Deployment +
{{ cluster.get_deployment_type_display }}
+
+
+ +
+ + Created {{ cluster.created_at|date:"M j, Y" }} + +
+ + + + + + +
+
+
+
+
+ {% endfor %} +
+ + + {% if page_obj.has_other_pages %} + + {% endif %} + +{% else %} +
+ +

No clusters found

+

Get started by creating your first Postgres cluster.

+ + Create Your First Cluster + +
+{% endif %} +{% endblock %} \ No newline at end of file