Implement full-text search functionality

This commit is contained in:
Joakim Hellsén 2025-09-05 13:59:02 +02:00
commit 51ec52499f
7 changed files with 229 additions and 7 deletions

View file

@ -48,6 +48,16 @@
<a href="{% url 'twitch:game_list' %}">Games</a> | <a href="{% url 'twitch:game_list' %}">Games</a> |
<a href="{% url 'twitch:org_list' %}">Organizations</a> | <a href="{% url 'twitch:org_list' %}">Organizations</a> |
<a href="{% url 'twitch:docs_rss' %}">RSS Docs</a> | <a href="{% url 'twitch:docs_rss' %}">RSS Docs</a> |
<form action="{% url 'twitch:search' %}"
method="get"
style="display: inline">
<input type="search"
name="q"
placeholder="Search..."
value="{{ request.GET.q }}">
<button type="submit">Search</button>
</form>
|
{% if user.is_authenticated %} {% if user.is_authenticated %}
<a href="{% url 'twitch:debug' %}">Debug</a> | <a href="{% url 'twitch:debug' %}">Debug</a> |
{% if user.is_staff %} {% if user.is_staff %}

View file

@ -0,0 +1,59 @@
{% extends "base.html" %}
{% block title %}
Search Results for "{{ query }}"
{% endblock title %}
{% block content %}
<div class="container">
<h1>Search Results for "{{ query }}"</h1>
{% if not results.organizations and not results.games and not results.campaigns and not results.drops and not results.benefits %}
<p>No results found.</p>
{% else %}
{% if results.organizations %}
<h2>Organizations</h2>
<ul>
{% for org in results.organizations %}
<li>
<a href="{% url 'twitch:organization_detail' org.pk %}">{{ org.name }}</a>
</li>
{% endfor %}
</ul>
{% endif %}
{% if results.games %}
<h2>Games</h2>
<ul>
{% for game in results.games %}
<li>
<a href="{% url 'twitch:game_detail' game.pk %}">{{ game.display_name }}</a>
</li>
{% endfor %}
</ul>
{% endif %}
{% if results.campaigns %}
<h2>Campaigns</h2>
<ul>
{% for campaign in results.campaigns %}
<li>
<a href="{% url 'twitch:campaign_detail' campaign.pk %}">{{ campaign.name }}</a>
</li>
{% endfor %}
</ul>
{% endif %}
{% if results.drops %}
<h2>Drops</h2>
<ul>
{% for drop in results.drops %}
<li>
<a href="{% url 'twitch:campaign_detail' drop.campaign.pk %}#drop-{{ drop.id }}">{{ drop.name }}</a> (in {{ drop.campaign.name }})
</li>
{% endfor %}
</ul>
{% endif %}
{% if results.benefits %}
<h2>Benefits</h2>
<ul>
{% for benefit in results.benefits %}<li>{{ benefit.name }}</li>{% endfor %}
</ul>
{% endif %}
{% endif %}
</div>
{% endblock content %}

View file

@ -5,19 +5,36 @@ from __future__ import annotations
from django.contrib.postgres.search import SearchVector from django.contrib.postgres.search import SearchVector
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from twitch.models import DropCampaign from twitch.models import DropBenefit, DropCampaign, Game, Organization, TimeBasedDrop
class Command(BaseCommand): class Command(BaseCommand):
"""Update search vectors for existing campaign records.""" """Update search vectors for existing records."""
help = "Update PostgreSQL search vectors for existing campaigns" help = "Update PostgreSQL search vectors for existing records"
def handle(self, *_args, **_options) -> None: def handle(self, *_args, **_options) -> None:
"""Update search vectors for all campaigns.""" """Update search vectors for all models."""
self.stdout.write("Updating search vectors for campaigns...") self.stdout.write("Updating search vectors...")
DropCampaign.objects.update(search_vector=SearchVector("name", "description", weight="A")) # Update Organizations
org_count = Organization.objects.update(search_vector=SearchVector("name"))
self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {org_count} organizations"))
campaign_count: int = DropCampaign.objects.count() # Update Games
game_count = Game.objects.update(search_vector=SearchVector("name", "display_name"))
self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {game_count} games"))
# Update DropCampaigns
campaign_count = DropCampaign.objects.update(search_vector=SearchVector("name", "description"))
self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {campaign_count} campaigns")) self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {campaign_count} campaigns"))
# Update TimeBasedDrops
drop_count = TimeBasedDrop.objects.update(search_vector=SearchVector("name"))
self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {drop_count} time-based drops"))
# Update DropBenefits
benefit_count = DropBenefit.objects.update(search_vector=SearchVector("name"))
self.stdout.write(self.style.SUCCESS(f"Successfully updated search vectors for {benefit_count} drop benefits"))
self.stdout.write(self.style.SUCCESS("All search vectors updated."))

View file

@ -0,0 +1,51 @@
# Generated by Django 5.2.5 on 2025-09-05 11:36
import django.contrib.postgres.indexes
import django.contrib.postgres.search
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('twitch', '0011_remove_timebaseddrop_drop_positive_minutes_and_more'),
]
operations = [
migrations.AddField(
model_name='dropbenefit',
name='search_vector',
field=django.contrib.postgres.search.SearchVectorField(blank=True, null=True),
),
migrations.AddField(
model_name='game',
name='search_vector',
field=django.contrib.postgres.search.SearchVectorField(blank=True, null=True),
),
migrations.AddField(
model_name='organization',
name='search_vector',
field=django.contrib.postgres.search.SearchVectorField(blank=True, null=True),
),
migrations.AddField(
model_name='timebaseddrop',
name='search_vector',
field=django.contrib.postgres.search.SearchVectorField(blank=True, null=True),
),
migrations.AddIndex(
model_name='dropbenefit',
index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='benefit_search_vector_idx'),
),
migrations.AddIndex(
model_name='game',
index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='game_search_vector_idx'),
),
migrations.AddIndex(
model_name='organization',
index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='org_search_vector_idx'),
),
migrations.AddIndex(
model_name='timebaseddrop',
index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='drop_search_vector_idx'),
),
]

View file

@ -45,11 +45,16 @@ class Organization(models.Model):
help_text="Timestamp when this organization record was last updated.", help_text="Timestamp when this organization record was last updated.",
) )
# PostgreSQL full-text search field
search_vector = SearchVectorField(null=True, blank=True)
class Meta: class Meta:
ordering = ["name"] ordering = ["name"]
indexes: ClassVar[list] = [ indexes: ClassVar[list] = [
# Regular B-tree index for name lookups # Regular B-tree index for name lookups
models.Index(fields=["name"]), models.Index(fields=["name"]),
# Full-text search index (GIN works with SearchVectorField)
GinIndex(fields=["search_vector"], name="org_search_vector_idx"),
] ]
def __str__(self) -> str: def __str__(self) -> str:
@ -90,6 +95,9 @@ class Game(models.Model):
verbose_name="Box art URL", verbose_name="Box art URL",
) )
# PostgreSQL full-text search field
search_vector = SearchVectorField(null=True, blank=True)
owner = models.ForeignKey( owner = models.ForeignKey(
Organization, Organization,
on_delete=models.SET_NULL, on_delete=models.SET_NULL,
@ -117,6 +125,8 @@ class Game(models.Model):
# Regular B-tree indexes for name lookups # Regular B-tree indexes for name lookups
models.Index(fields=["display_name"]), models.Index(fields=["display_name"]),
models.Index(fields=["name"]), models.Index(fields=["name"]),
# Full-text search index (GIN works with SearchVectorField)
GinIndex(fields=["search_vector"], name="game_search_vector_idx"),
# Partial index for games with owners only # Partial index for games with owners only
models.Index(fields=["owner"], condition=models.Q(owner__isnull=False), name="game_owner_partial_idx"), models.Index(fields=["owner"], condition=models.Q(owner__isnull=False), name="game_owner_partial_idx"),
] ]
@ -330,6 +340,9 @@ class DropBenefit(models.Model):
help_text="Type of distribution for this benefit.", help_text="Type of distribution for this benefit.",
) )
# PostgreSQL full-text search field
search_vector = SearchVectorField(null=True, blank=True)
added_at = models.DateTimeField( added_at = models.DateTimeField(
auto_now_add=True, auto_now_add=True,
db_index=True, db_index=True,
@ -345,6 +358,8 @@ class DropBenefit(models.Model):
indexes: ClassVar[list] = [ indexes: ClassVar[list] = [
# Regular B-tree index for benefit name lookups # Regular B-tree index for benefit name lookups
models.Index(fields=["name"]), models.Index(fields=["name"]),
# Full-text search index (GIN works with SearchVectorField)
GinIndex(fields=["search_vector"], name="benefit_search_vector_idx"),
models.Index(fields=["created_at"]), models.Index(fields=["created_at"]),
models.Index(fields=["distribution_type"]), models.Index(fields=["distribution_type"]),
# Partial index for iOS available benefits # Partial index for iOS available benefits
@ -392,6 +407,9 @@ class TimeBasedDrop(models.Model):
help_text="Datetime when this drop expires.", help_text="Datetime when this drop expires.",
) )
# PostgreSQL full-text search field
search_vector = SearchVectorField(null=True, blank=True)
# Foreign keys # Foreign keys
campaign = models.ForeignKey( campaign = models.ForeignKey(
DropCampaign, DropCampaign,
@ -432,6 +450,8 @@ class TimeBasedDrop(models.Model):
indexes: ClassVar[list] = [ indexes: ClassVar[list] = [
# Regular B-tree index for drop name lookups # Regular B-tree index for drop name lookups
models.Index(fields=["name"]), models.Index(fields=["name"]),
# Full-text search index (GIN works with SearchVectorField)
GinIndex(fields=["search_vector"], name="drop_search_vector_idx"),
models.Index(fields=["start_at", "end_at"]), models.Index(fields=["start_at", "end_at"]),
models.Index(fields=["required_minutes_watched"]), models.Index(fields=["required_minutes_watched"]),
# Covering index for common queries (includes campaign_id from FK) # Covering index for common queries (includes campaign_id from FK)

View file

@ -13,6 +13,7 @@ app_name = "twitch"
urlpatterns = [ urlpatterns = [
path("", views.dashboard, name="dashboard"), path("", views.dashboard, name="dashboard"),
path("search/", views.search_view, name="search"),
path("debug/", views.debug_view, name="debug"), path("debug/", views.debug_view, name="debug"),
path("campaigns/", views.DropCampaignListView.as_view(), name="campaign_list"), path("campaigns/", views.DropCampaignListView.as_view(), name="campaign_list"),
path("campaigns/<str:pk>/", views.DropCampaignDetailView.as_view(), name="campaign_detail"), path("campaigns/<str:pk>/", views.DropCampaignDetailView.as_view(), name="campaign_detail"),

View file

@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, cast
from django.contrib import messages from django.contrib import messages
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector
from django.core.serializers import serialize from django.core.serializers import serialize
from django.db.models import Count, F, Prefetch, Q from django.db.models import Count, F, Prefetch, Q
from django.db.models.functions import Trim from django.db.models.functions import Trim
@ -27,6 +28,69 @@ if TYPE_CHECKING:
logger: logging.Logger = logging.getLogger(__name__) logger: logging.Logger = logging.getLogger(__name__)
MIN_QUERY_LENGTH_FOR_FTS = 3
MIN_SEARCH_RANK = 0.05
def search_view(request: HttpRequest) -> HttpResponse:
"""Search view for all models.
Args:
request: The HTTP request.
Returns:
HttpResponse: The rendered search results.
"""
query = request.GET.get("q", "")
results = {}
if query:
if len(query) < MIN_QUERY_LENGTH_FOR_FTS:
results["organizations"] = Organization.objects.filter(name__istartswith=query)
results["games"] = Game.objects.filter(Q(name__istartswith=query) | Q(display_name__istartswith=query))
results["campaigns"] = DropCampaign.objects.filter(Q(name__istartswith=query) | Q(description__icontains=query)).select_related("game")
results["drops"] = TimeBasedDrop.objects.filter(name__istartswith=query).select_related("campaign")
results["benefits"] = DropBenefit.objects.filter(name__istartswith=query)
else:
search_query = SearchQuery(query)
# Search Organizations
org_vector = SearchVector("name")
org_results = Organization.objects.annotate(rank=SearchRank(org_vector, search_query)).filter(rank__gte=MIN_SEARCH_RANK).order_by("-rank")
results["organizations"] = org_results
# Search Games
game_vector = SearchVector("name", "display_name")
game_results = Game.objects.annotate(rank=SearchRank(game_vector, search_query)).filter(rank__gte=MIN_SEARCH_RANK).order_by("-rank")
results["games"] = game_results
# Search DropCampaigns
campaign_vector = SearchVector("name", "description")
campaign_results = (
DropCampaign.objects.annotate(rank=SearchRank(campaign_vector, search_query))
.filter(rank__gte=MIN_SEARCH_RANK)
.select_related("game")
.order_by("-rank")
)
results["campaigns"] = campaign_results
# Search TimeBasedDrops
drop_vector = SearchVector("name")
drop_results = (
TimeBasedDrop.objects.annotate(rank=SearchRank(drop_vector, search_query))
.filter(rank__gte=MIN_SEARCH_RANK)
.select_related("campaign")
.order_by("-rank")
)
results["drops"] = drop_results
# Search DropBenefits
benefit_vector = SearchVector("name")
benefit_results = DropBenefit.objects.annotate(rank=SearchRank(benefit_vector, search_query)).filter(rank__gte=MIN_SEARCH_RANK).order_by("-rank")
results["benefits"] = benefit_results
return render(request, "twitch/search_results.html", {"query": query, "results": results})
class OrgListView(ListView): class OrgListView(ListView):
"""List view for organization.""" """List view for organization."""