From 51ec52499f9650a34441ceeba8dd7723a09efadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Fri, 5 Sep 2025 13:59:02 +0200 Subject: [PATCH] Implement full-text search functionality --- templates/base.html | 10 +++ templates/twitch/search_results.html | 59 +++++++++++++++++ .../commands/update_search_vectors.py | 31 +++++++-- ...arch_vector_game_search_vector_and_more.py | 51 +++++++++++++++ twitch/models.py | 20 ++++++ twitch/urls.py | 1 + twitch/views.py | 64 +++++++++++++++++++ 7 files changed, 229 insertions(+), 7 deletions(-) create mode 100644 templates/twitch/search_results.html create mode 100644 twitch/migrations/0012_dropbenefit_search_vector_game_search_vector_and_more.py diff --git a/templates/base.html b/templates/base.html index 55844f8..c4a06e8 100644 --- a/templates/base.html +++ b/templates/base.html @@ -48,6 +48,16 @@ Games | Organizations | RSS Docs | +
+ + +
+ | {% if user.is_authenticated %} Debug | {% if user.is_staff %} diff --git a/templates/twitch/search_results.html b/templates/twitch/search_results.html new file mode 100644 index 0000000..268fc6a --- /dev/null +++ b/templates/twitch/search_results.html @@ -0,0 +1,59 @@ +{% extends "base.html" %} +{% block title %} + Search Results for "{{ query }}" +{% endblock title %} +{% block content %} +
+

Search Results for "{{ query }}"

+ {% if not results.organizations and not results.games and not results.campaigns and not results.drops and not results.benefits %} +

No results found.

+ {% else %} + {% if results.organizations %} +

Organizations

+ + {% endif %} + {% if results.games %} +

Games

+ + {% endif %} + {% if results.campaigns %} +

Campaigns

+ + {% endif %} + {% if results.drops %} +

Drops

+ + {% endif %} + {% if results.benefits %} +

Benefits

+ + {% endif %} + {% endif %} +
+{% endblock content %} diff --git a/twitch/management/commands/update_search_vectors.py b/twitch/management/commands/update_search_vectors.py index 7e4219f..ad71127 100644 --- a/twitch/management/commands/update_search_vectors.py +++ b/twitch/management/commands/update_search_vectors.py @@ -5,19 +5,36 @@ from __future__ import annotations from django.contrib.postgres.search import SearchVector from django.core.management.base import BaseCommand -from twitch.models import DropCampaign +from twitch.models import DropBenefit, DropCampaign, Game, Organization, TimeBasedDrop class Command(BaseCommand): - """Update search vectors for existing campaign records.""" + """Update search vectors for existing records.""" - help = "Update PostgreSQL search vectors for existing campaigns" + help = "Update PostgreSQL search vectors for existing records" def handle(self, *_args, **_options) -> None: - """Update search vectors for all campaigns.""" - self.stdout.write("Updating search vectors for campaigns...") + """Update search vectors for all models.""" + self.stdout.write("Updating search vectors...") - DropCampaign.objects.update(search_vector=SearchVector("name", "description", weight="A")) + # Update Organizations + org_count = Organization.objects.update(search_vector=SearchVector("name")) + self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {org_count} organizations")) - campaign_count: int = DropCampaign.objects.count() + # Update Games + game_count = Game.objects.update(search_vector=SearchVector("name", "display_name")) + self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {game_count} games")) + + # Update DropCampaigns + campaign_count = DropCampaign.objects.update(search_vector=SearchVector("name", "description")) self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {campaign_count} campaigns")) + + # Update TimeBasedDrops + drop_count = TimeBasedDrop.objects.update(search_vector=SearchVector("name")) + self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {drop_count} time-based drops")) + + # Update DropBenefits + benefit_count = DropBenefit.objects.update(search_vector=SearchVector("name")) + self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {benefit_count} drop benefits")) + + self.stdout.write(self.style.SUCCESS("All search vectors updated.")) diff --git a/twitch/migrations/0012_dropbenefit_search_vector_game_search_vector_and_more.py b/twitch/migrations/0012_dropbenefit_search_vector_game_search_vector_and_more.py new file mode 100644 index 0000000..f7c7fc8 --- /dev/null +++ b/twitch/migrations/0012_dropbenefit_search_vector_game_search_vector_and_more.py @@ -0,0 +1,51 @@ +# Generated by Django 5.2.5 on 2025-09-05 11:36 + +import django.contrib.postgres.indexes +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('twitch', '0011_remove_timebaseddrop_drop_positive_minutes_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='dropbenefit', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, null=True), + ), + migrations.AddField( + model_name='game', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, null=True), + ), + migrations.AddField( + model_name='organization', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, null=True), + ), + migrations.AddField( + model_name='timebaseddrop', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, null=True), + ), + migrations.AddIndex( + model_name='dropbenefit', + index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='benefit_search_vector_idx'), + ), + migrations.AddIndex( + model_name='game', + index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='game_search_vector_idx'), + ), + migrations.AddIndex( + model_name='organization', + index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='org_search_vector_idx'), + ), + migrations.AddIndex( + model_name='timebaseddrop', + index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='drop_search_vector_idx'), + ), + ] diff --git a/twitch/models.py b/twitch/models.py index a85d1e4..bfd3ebc 100644 --- a/twitch/models.py +++ b/twitch/models.py @@ -45,11 +45,16 @@ class Organization(models.Model): help_text="Timestamp when this organization record was last updated.", ) + # PostgreSQL full-text search field + search_vector = SearchVectorField(null=True, blank=True) + class Meta: ordering = ["name"] indexes: ClassVar[list] = [ # Regular B-tree index for name lookups models.Index(fields=["name"]), + # Full-text search index (GIN works with SearchVectorField) + GinIndex(fields=["search_vector"], name="org_search_vector_idx"), ] def __str__(self) -> str: @@ -90,6 +95,9 @@ class Game(models.Model): verbose_name="Box art URL", ) + # PostgreSQL full-text search field + search_vector = SearchVectorField(null=True, blank=True) + owner = models.ForeignKey( Organization, on_delete=models.SET_NULL, @@ -117,6 +125,8 @@ class Game(models.Model): # Regular B-tree indexes for name lookups models.Index(fields=["display_name"]), models.Index(fields=["name"]), + # Full-text search index (GIN works with SearchVectorField) + GinIndex(fields=["search_vector"], name="game_search_vector_idx"), # Partial index for games with owners only models.Index(fields=["owner"], condition=models.Q(owner__isnull=False), name="game_owner_partial_idx"), ] @@ -330,6 +340,9 @@ class DropBenefit(models.Model): help_text="Type of distribution for this benefit.", ) + # PostgreSQL full-text search field + search_vector = SearchVectorField(null=True, blank=True) + added_at = models.DateTimeField( auto_now_add=True, db_index=True, @@ -345,6 +358,8 @@ class DropBenefit(models.Model): indexes: ClassVar[list] = [ # Regular B-tree index for benefit name lookups models.Index(fields=["name"]), + # Full-text search index (GIN works with SearchVectorField) + GinIndex(fields=["search_vector"], name="benefit_search_vector_idx"), models.Index(fields=["created_at"]), models.Index(fields=["distribution_type"]), # Partial index for iOS available benefits @@ -392,6 +407,9 @@ class TimeBasedDrop(models.Model): help_text="Datetime when this drop expires.", ) + # PostgreSQL full-text search field + search_vector = SearchVectorField(null=True, blank=True) + # Foreign keys campaign = models.ForeignKey( DropCampaign, @@ -432,6 +450,8 @@ class TimeBasedDrop(models.Model): indexes: ClassVar[list] = [ # Regular B-tree index for drop name lookups models.Index(fields=["name"]), + # Full-text search index (GIN works with SearchVectorField) + GinIndex(fields=["search_vector"], name="drop_search_vector_idx"), models.Index(fields=["start_at", "end_at"]), models.Index(fields=["required_minutes_watched"]), # Covering index for common queries (includes campaign_id from FK) diff --git a/twitch/urls.py b/twitch/urls.py index b459ffc..60b6262 100644 --- a/twitch/urls.py +++ b/twitch/urls.py @@ -13,6 +13,7 @@ app_name = "twitch" urlpatterns = [ path("", views.dashboard, name="dashboard"), + path("search/", views.search_view, name="search"), path("debug/", views.debug_view, name="debug"), path("campaigns/", views.DropCampaignListView.as_view(), name="campaign_list"), path("campaigns//", views.DropCampaignDetailView.as_view(), name="campaign_detail"), diff --git a/twitch/views.py b/twitch/views.py index fe1f931..bf3b48e 100644 --- a/twitch/views.py +++ b/twitch/views.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, cast from django.contrib import messages from django.contrib.auth.decorators import login_required +from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector from django.core.serializers import serialize from django.db.models import Count, F, Prefetch, Q from django.db.models.functions import Trim @@ -27,6 +28,69 @@ if TYPE_CHECKING: logger: logging.Logger = logging.getLogger(__name__) +MIN_QUERY_LENGTH_FOR_FTS = 3 +MIN_SEARCH_RANK = 0.05 + + +def search_view(request: HttpRequest) -> HttpResponse: + """Search view for all models. + + Args: + request: The HTTP request. + + Returns: + HttpResponse: The rendered search results. + """ + query = request.GET.get("q", "") + results = {} + + if query: + if len(query) < MIN_QUERY_LENGTH_FOR_FTS: + results["organizations"] = Organization.objects.filter(name__istartswith=query) + results["games"] = Game.objects.filter(Q(name__istartswith=query) | Q(display_name__istartswith=query)) + results["campaigns"] = DropCampaign.objects.filter(Q(name__istartswith=query) | Q(description__icontains=query)).select_related("game") + results["drops"] = TimeBasedDrop.objects.filter(name__istartswith=query).select_related("campaign") + results["benefits"] = DropBenefit.objects.filter(name__istartswith=query) + else: + search_query = SearchQuery(query) + + # Search Organizations + org_vector = SearchVector("name") + org_results = Organization.objects.annotate(rank=SearchRank(org_vector, search_query)).filter(rank__gte=MIN_SEARCH_RANK).order_by("-rank") + results["organizations"] = org_results + + # Search Games + game_vector = SearchVector("name", "display_name") + game_results = Game.objects.annotate(rank=SearchRank(game_vector, search_query)).filter(rank__gte=MIN_SEARCH_RANK).order_by("-rank") + results["games"] = game_results + + # Search DropCampaigns + campaign_vector = SearchVector("name", "description") + campaign_results = ( + DropCampaign.objects.annotate(rank=SearchRank(campaign_vector, search_query)) + .filter(rank__gte=MIN_SEARCH_RANK) + .select_related("game") + .order_by("-rank") + ) + results["campaigns"] = campaign_results + + # Search TimeBasedDrops + drop_vector = SearchVector("name") + drop_results = ( + TimeBasedDrop.objects.annotate(rank=SearchRank(drop_vector, search_query)) + .filter(rank__gte=MIN_SEARCH_RANK) + .select_related("campaign") + .order_by("-rank") + ) + results["drops"] = drop_results + + # Search DropBenefits + benefit_vector = SearchVector("name") + benefit_results = DropBenefit.objects.annotate(rank=SearchRank(benefit_vector, search_query)).filter(rank__gte=MIN_SEARCH_RANK).order_by("-rank") + results["benefits"] = benefit_results + + return render(request, "twitch/search_results.html", {"query": query, "results": results}) + class OrgListView(ListView): """List view for organization."""