Add API
This commit is contained in:
2
.vscode/launch.json
vendored
2
.vscode/launch.json
vendored
@ -8,7 +8,7 @@
|
|||||||
"program": "${workspaceFolder}/manage.py",
|
"program": "${workspaceFolder}/manage.py",
|
||||||
"args": [
|
"args": [
|
||||||
"runserver",
|
"runserver",
|
||||||
"--noreload",
|
"--reload",
|
||||||
"--nothreading"
|
"--nothreading"
|
||||||
],
|
],
|
||||||
"django": true,
|
"django": true,
|
||||||
|
@ -70,6 +70,7 @@ INSTALLED_APPS: list[str] = [
|
|||||||
"django.contrib.sessions",
|
"django.contrib.sessions",
|
||||||
"django.contrib.messages",
|
"django.contrib.messages",
|
||||||
"django.contrib.staticfiles",
|
"django.contrib.staticfiles",
|
||||||
|
"ninja",
|
||||||
]
|
]
|
||||||
|
|
||||||
MIDDLEWARE: list[str] = [
|
MIDDLEWARE: list[str] = [
|
||||||
@ -130,3 +131,26 @@ STORAGES: dict[str, dict[str, str]] = {
|
|||||||
"BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",
|
"BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOGGING = {
|
||||||
|
"version": 1,
|
||||||
|
"disable_existing_loggers": False,
|
||||||
|
"handlers": {
|
||||||
|
"console": {
|
||||||
|
"level": "DEBUG",
|
||||||
|
"class": "logging.StreamHandler",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"loggers": {
|
||||||
|
"": {
|
||||||
|
"handlers": ["console"],
|
||||||
|
"level": "DEBUG",
|
||||||
|
"propagate": True,
|
||||||
|
},
|
||||||
|
"django.utils.autoreload": { # Remove spam
|
||||||
|
"handlers": ["console"],
|
||||||
|
"level": "INFO",
|
||||||
|
"propagate": True,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
@ -1,10 +1,21 @@
|
|||||||
from django.contrib import admin
|
from django.contrib import admin
|
||||||
from django.urls import include, path
|
from django.urls import URLPattern, include, path
|
||||||
from django.urls.resolvers import URLResolver
|
from django.urls.resolvers import URLResolver
|
||||||
|
from ninja import NinjaAPI
|
||||||
|
|
||||||
|
from twitch.api import router as twitch_router
|
||||||
|
|
||||||
|
api = NinjaAPI(
|
||||||
|
title="TTVDrops API",
|
||||||
|
version="1.0.0",
|
||||||
|
description="No rate limits, but don't abuse it.",
|
||||||
|
)
|
||||||
|
api.add_router(prefix="/twitch", router=twitch_router)
|
||||||
|
|
||||||
app_name: str = "config"
|
app_name: str = "config"
|
||||||
|
|
||||||
urlpatterns: list[URLResolver] = [
|
urlpatterns: list[URLPattern | URLResolver] = [
|
||||||
path(route="admin/", view=admin.site.urls),
|
path(route="admin/", view=admin.site.urls),
|
||||||
path(route="", view=include(arg="core.urls")),
|
path(route="", view=include(arg="core.urls")),
|
||||||
|
path(route="api/", view=api.urls),
|
||||||
]
|
]
|
||||||
|
@ -1,120 +0,0 @@
|
|||||||
{% load static %}
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en" data-bs-theme="dark">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="description"
|
|
||||||
content="{% block description %}
|
|
||||||
Subscribe to TTVDrop for the latest Twitch drops.
|
|
||||||
{% endblock description %}">
|
|
||||||
<meta name="keywords"
|
|
||||||
content="{% block keywords %}
|
|
||||||
Twitch, Drops, TTVDrop, Twitch Drops, Twitch Drops Tracker
|
|
||||||
{% endblock keywords %}">
|
|
||||||
<meta name="author" content="TheLovinator">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
{% block title %}
|
|
||||||
<title>Twitch drops</title>
|
|
||||||
{% endblock title %}
|
|
||||||
<style>
|
|
||||||
html {
|
|
||||||
max-width: 88ch;
|
|
||||||
padding: calc(1vmin + 0.5rem);
|
|
||||||
margin-inline: auto;
|
|
||||||
font-size: clamp(1em, 0.909em + 0.45vmin, 1.25em);
|
|
||||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
|
|
||||||
Helvetica, Arial, sans-serif;
|
|
||||||
color-scheme: light dark;
|
|
||||||
}
|
|
||||||
|
|
||||||
h1 {
|
|
||||||
font-size: 2.5rem;
|
|
||||||
font-weight: 600;
|
|
||||||
margin: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.title {
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search {
|
|
||||||
display: flex;
|
|
||||||
justify-content: center;
|
|
||||||
margin-top: 1rem;
|
|
||||||
margin-inline: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.leftright {
|
|
||||||
display: flex;
|
|
||||||
justify-content: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.left {
|
|
||||||
margin-right: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.right {
|
|
||||||
margin-left: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
textarea {
|
|
||||||
width: 100%;
|
|
||||||
height: 10rem;
|
|
||||||
resize: vertical;
|
|
||||||
}
|
|
||||||
|
|
||||||
.messages {
|
|
||||||
list-style-type: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.error {
|
|
||||||
color: red;
|
|
||||||
}
|
|
||||||
|
|
||||||
.success {
|
|
||||||
color: green;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
{% if messages %}
|
|
||||||
<ul class="messages">
|
|
||||||
{% for message in messages %}
|
|
||||||
<li {% if message.tags %}class="{{ message.tags }}"{% endif %}>{{ message }}</li>
|
|
||||||
{% endfor %}
|
|
||||||
</ul>
|
|
||||||
{% endif %}
|
|
||||||
<span class="title">
|
|
||||||
<h1>
|
|
||||||
<a href="{% url 'core:index' %}">TTVDrop</a>
|
|
||||||
</h1>
|
|
||||||
</span>
|
|
||||||
<nav>
|
|
||||||
<small>
|
|
||||||
<div class="leftright">
|
|
||||||
<div class="left">Hello.</div>
|
|
||||||
<div class="right">
|
|
||||||
<a href="/api/v1/docs">API</a> |
|
|
||||||
<a href="https://github.com/TheLovinator1/twitch-online-notifier">GitHub</a> |
|
|
||||||
<a href="https://github.com/sponsors/TheLovinator1">Donate</a>
|
|
||||||
{% if not user.is_authenticated %}| <a href=''>Login</a>{% endif %}
|
|
||||||
{% if user.is_authenticated %}| <a href=''>{{ user.username }}</a>{% endif %}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</small>
|
|
||||||
</nav>
|
|
||||||
<hr />
|
|
||||||
<main>
|
|
||||||
{% block content %}<!-- default content -->{% endblock %}
|
|
||||||
</main>
|
|
||||||
<hr />
|
|
||||||
<footer>
|
|
||||||
<small>
|
|
||||||
<div class="leftright">
|
|
||||||
<div class="left">TheLovinator#9276 on Discord</div>
|
|
||||||
<div class="right">No rights reserved.</div>
|
|
||||||
</div>
|
|
||||||
</small>
|
|
||||||
</footer>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
@ -93,13 +93,35 @@
|
|||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
margin: 0;
|
margin: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Django messages framework */
|
||||||
|
.messages {
|
||||||
|
list-style-type: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Make error messages red and success messages green */
|
||||||
|
.error {
|
||||||
|
color: red;
|
||||||
|
}
|
||||||
|
.success {
|
||||||
|
color: green;
|
||||||
|
}
|
||||||
|
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
{% if messages %}
|
||||||
|
<ul class="messages">
|
||||||
|
{% for message in messages %}
|
||||||
|
<li {% if message.tags %}class="{{ message.tags }}"{% endif %}>{{ message }}</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
{% endif %}
|
||||||
<h1 class="logo">Twitch Drops</h1>
|
<h1 class="logo">Twitch Drops</h1>
|
||||||
<div class="navbar">
|
<div class="navbar">
|
||||||
<a href="">API</a> |
|
<a href='{% url "api-1.0.0:openapi-view" %}'>API</a> |
|
||||||
<a href="https://github.com/sponsors/TheLovinator1">Donate</a>
|
<a href="https://github.com/sponsors/TheLovinator1">Donate</a> |
|
||||||
|
TheLovinator#9276
|
||||||
</div>
|
</div>
|
||||||
{% for organization, org_data in orgs_data.items %}
|
{% for organization, org_data in orgs_data.items %}
|
||||||
<ul>
|
<ul>
|
||||||
|
@ -1,11 +1,12 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from django.urls import URLPattern, path
|
from django.urls import URLPattern, URLResolver, path
|
||||||
|
|
||||||
from . import views
|
from . import views
|
||||||
|
|
||||||
app_name: str = "core"
|
app_name: str = "core"
|
||||||
|
|
||||||
urlpatterns: list[URLPattern] = [
|
|
||||||
|
urlpatterns: list[URLPattern | URLResolver] = [
|
||||||
path(route="", view=views.index, name="index"),
|
path(route="", view=views.index, name="index"),
|
||||||
]
|
]
|
||||||
|
@ -10,3 +10,4 @@ sentry-sdk[django]
|
|||||||
whitenoise[brotli]
|
whitenoise[brotli]
|
||||||
platformdirs
|
platformdirs
|
||||||
playwright
|
playwright
|
||||||
|
django-ninja
|
||||||
|
124
twitch/api.py
Normal file
124
twitch/api.py
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
import datetime
|
||||||
|
|
||||||
|
from django.db.models.manager import BaseManager
|
||||||
|
from django.http import HttpRequest
|
||||||
|
from ninja import Router, Schema
|
||||||
|
|
||||||
|
from .models import (
|
||||||
|
Channel,
|
||||||
|
DropBenefit,
|
||||||
|
DropCampaign,
|
||||||
|
Game,
|
||||||
|
Organization,
|
||||||
|
TimeBasedDrop,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = Router(
|
||||||
|
tags=["twitch"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class OrganizationSchema(Schema):
|
||||||
|
id: str | None = None
|
||||||
|
name: str | None = None
|
||||||
|
added_at: datetime.datetime | None = None
|
||||||
|
modified_at: datetime.datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ChannelSchema(Schema):
|
||||||
|
id: str
|
||||||
|
display_name: str | None = None
|
||||||
|
name: str | None = None
|
||||||
|
added_at: datetime.datetime | None = None
|
||||||
|
modified_at: datetime.datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class GameSchema(Schema):
|
||||||
|
id: str
|
||||||
|
slug: str | None = None
|
||||||
|
twitch_url: str | None = None
|
||||||
|
display_name: str | None = None
|
||||||
|
added_at: datetime.datetime | None = None
|
||||||
|
modified_at: datetime.datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class DropBenefitSchema(Schema):
|
||||||
|
id: str
|
||||||
|
created_at: datetime.datetime | None = None
|
||||||
|
entitlement_limit: int | None = None
|
||||||
|
image_asset_url: str | None = None
|
||||||
|
is_ios_available: bool | None = None
|
||||||
|
name: str | None = None
|
||||||
|
owner_organization: OrganizationSchema
|
||||||
|
game: GameSchema
|
||||||
|
added_at: datetime.datetime | None = None
|
||||||
|
modified_at: datetime.datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class TimeBasedDropSchema(Schema):
|
||||||
|
id: str
|
||||||
|
required_subs: int | None = None
|
||||||
|
end_at: datetime.datetime | None = None
|
||||||
|
name: str | None = None
|
||||||
|
required_minutes_watched: int | None = None
|
||||||
|
start_at: datetime.datetime | None = None
|
||||||
|
benefits: list[DropBenefitSchema]
|
||||||
|
added_at: datetime.datetime | None = None
|
||||||
|
modified_at: datetime.datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class DropCampaignSchema(Schema):
|
||||||
|
id: str
|
||||||
|
account_link_url: str | None = None
|
||||||
|
description: str | None = None
|
||||||
|
details_url: str | None = None
|
||||||
|
added_at: datetime.datetime | None = None
|
||||||
|
modified_at: datetime.datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
# http://localhost:8000/api/twitch/organizations
|
||||||
|
@router.get("/organizations", response=list[OrganizationSchema])
|
||||||
|
def get_organizations(
|
||||||
|
request: HttpRequest, # noqa: ARG001
|
||||||
|
) -> BaseManager[Organization]:
|
||||||
|
"""Get all organizations."""
|
||||||
|
return Organization.objects.all()
|
||||||
|
|
||||||
|
|
||||||
|
# http://localhost:8000/api/twitch/channels
|
||||||
|
@router.get("/channels", response=list[ChannelSchema])
|
||||||
|
def get_channels(request: HttpRequest) -> BaseManager[Channel]: # noqa: ARG001
|
||||||
|
"""Get all channels."""
|
||||||
|
return Channel.objects.all()
|
||||||
|
|
||||||
|
|
||||||
|
# http://localhost:8000/api/twitch/games
|
||||||
|
@router.get("/games", response=list[GameSchema])
|
||||||
|
def get_games(request: HttpRequest) -> BaseManager[Game]: # noqa: ARG001
|
||||||
|
"""Get all games."""
|
||||||
|
return Game.objects.all()
|
||||||
|
|
||||||
|
|
||||||
|
# http://localhost:8000/api/twitch/drop_benefits
|
||||||
|
@router.get("/drop_benefits", response=list[DropBenefitSchema])
|
||||||
|
def get_drop_benefits(request: HttpRequest) -> BaseManager[DropBenefit]: # noqa: ARG001
|
||||||
|
"""Get all drop benefits."""
|
||||||
|
return DropBenefit.objects.all()
|
||||||
|
|
||||||
|
|
||||||
|
# http://localhost:8000/api/twitch/drop_campaigns
|
||||||
|
@router.get("/drop_campaigns", response=list[DropCampaignSchema])
|
||||||
|
def get_drop_campaigns(
|
||||||
|
request: HttpRequest, # noqa: ARG001
|
||||||
|
) -> BaseManager[DropCampaign]:
|
||||||
|
"""Get all drop campaigns."""
|
||||||
|
return DropCampaign.objects.all()
|
||||||
|
|
||||||
|
|
||||||
|
# http://localhost:8000/api/twitch/time_based_drops
|
||||||
|
@router.get("/time_based_drops", response=list[TimeBasedDropSchema])
|
||||||
|
def get_time_based_drops(
|
||||||
|
request: HttpRequest, # noqa: ARG001
|
||||||
|
) -> BaseManager[TimeBasedDrop]:
|
||||||
|
"""Get all time-based drops."""
|
||||||
|
return TimeBasedDrop.objects.all()
|
@ -1,7 +1,8 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import typing
|
import typing
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from asgiref.sync import sync_to_async
|
from asgiref.sync import sync_to_async
|
||||||
from django.core.management.base import BaseCommand
|
from django.core.management.base import BaseCommand
|
||||||
@ -22,6 +23,7 @@ from twitch.models import (
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from playwright.async_api._generated import BrowserContext, Page
|
from playwright.async_api._generated import BrowserContext, Page
|
||||||
|
|
||||||
|
# Where to store the Firefox profile
|
||||||
data_dir = Path(
|
data_dir = Path(
|
||||||
user_data_dir(
|
user_data_dir(
|
||||||
appname="TTVDrops",
|
appname="TTVDrops",
|
||||||
@ -35,6 +37,8 @@ if not data_dir:
|
|||||||
msg = "DATA_DIR is not set in settings.py"
|
msg = "DATA_DIR is not set in settings.py"
|
||||||
raise ValueError(msg)
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
logger: logging.Logger = logging.getLogger("twitch.management.commands.scrape_twitch")
|
||||||
|
|
||||||
|
|
||||||
async def insert_data(data: dict) -> None: # noqa: PLR0914
|
async def insert_data(data: dict) -> None: # noqa: PLR0914
|
||||||
"""Insert data into the database.
|
"""Insert data into the database.
|
||||||
@ -44,32 +48,39 @@ async def insert_data(data: dict) -> None: # noqa: PLR0914
|
|||||||
"""
|
"""
|
||||||
user_data: dict = data.get("data", {}).get("user")
|
user_data: dict = data.get("data", {}).get("user")
|
||||||
if not user_data:
|
if not user_data:
|
||||||
|
logger.debug("No user data found")
|
||||||
return
|
return
|
||||||
|
|
||||||
user_id = user_data["id"]
|
user_id = user_data["id"]
|
||||||
drop_campaign_data = user_data["dropCampaign"]
|
drop_campaign_data = user_data["dropCampaign"]
|
||||||
if not drop_campaign_data:
|
if not drop_campaign_data:
|
||||||
|
logger.debug("No drop campaign data found")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
logger.info("Inserting data for user ID: %s", user_id)
|
||||||
|
|
||||||
# Create or get the organization
|
# Create or get the organization
|
||||||
owner_data = drop_campaign_data["owner"]
|
owner_data = drop_campaign_data["owner"]
|
||||||
owner, _ = await sync_to_async(Organization.objects.get_or_create)(
|
owner, created = await sync_to_async(Organization.objects.get_or_create)(
|
||||||
id=owner_data["id"],
|
id=owner_data["id"],
|
||||||
defaults={"name": owner_data["name"]},
|
defaults={"name": owner_data["name"]},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
logger.debug("Organization %s: %s", "created" if created else "retrieved", owner)
|
||||||
|
|
||||||
# Create or get the game
|
# Create or get the game
|
||||||
game_data = drop_campaign_data["game"]
|
game_data = drop_campaign_data["game"]
|
||||||
game, _ = await sync_to_async(Game.objects.get_or_create)(
|
game, created = await sync_to_async(Game.objects.get_or_create)(
|
||||||
id=game_data["id"],
|
id=game_data["id"],
|
||||||
defaults={
|
defaults={
|
||||||
"slug": game_data["slug"],
|
"slug": game_data["slug"],
|
||||||
"display_name": game_data["displayName"],
|
"display_name": game_data["displayName"],
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
logger.debug("Game %s: %s", "created" if created else "retrieved", game)
|
||||||
|
|
||||||
# Create the drop campaign
|
# Create the drop campaign
|
||||||
drop_campaign, _ = await sync_to_async(DropCampaign.objects.get_or_create)(
|
drop_campaign, created = await sync_to_async(DropCampaign.objects.get_or_create)(
|
||||||
id=drop_campaign_data["id"],
|
id=drop_campaign_data["id"],
|
||||||
defaults={
|
defaults={
|
||||||
"account_link_url": drop_campaign_data["accountLinkURL"],
|
"account_link_url": drop_campaign_data["accountLinkURL"],
|
||||||
@ -84,15 +95,23 @@ async def insert_data(data: dict) -> None: # noqa: PLR0914
|
|||||||
"owner": owner,
|
"owner": owner,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
logger.debug(
|
||||||
|
"Drop campaign %s: %s",
|
||||||
|
"created" if created else "retrieved",
|
||||||
|
drop_campaign,
|
||||||
|
)
|
||||||
|
|
||||||
if not drop_campaign_data["allow"]:
|
if not drop_campaign_data["allow"]:
|
||||||
|
logger.debug("No allowed data in drop campaign")
|
||||||
return
|
return
|
||||||
|
|
||||||
if not drop_campaign_data["allow"]["channels"]:
|
if not drop_campaign_data["allow"]["channels"]:
|
||||||
|
logger.debug("No allowed channels in drop campaign")
|
||||||
return
|
return
|
||||||
|
|
||||||
# Create channels
|
# Create channels
|
||||||
for channel_data in drop_campaign_data["allow"]["channels"]:
|
for channel_data in drop_campaign_data["allow"]["channels"]:
|
||||||
channel, _ = await sync_to_async(Channel.objects.get_or_create)(
|
channel, created = await sync_to_async(Channel.objects.get_or_create)(
|
||||||
id=channel_data["id"],
|
id=channel_data["id"],
|
||||||
defaults={
|
defaults={
|
||||||
"display_name": channel_data["displayName"],
|
"display_name": channel_data["displayName"],
|
||||||
@ -100,6 +119,7 @@ async def insert_data(data: dict) -> None: # noqa: PLR0914
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
await sync_to_async(drop_campaign.channels.add)(channel)
|
await sync_to_async(drop_campaign.channels.add)(channel)
|
||||||
|
logger.debug("Channel %s: %s", "created" if created else "retrieved", channel)
|
||||||
|
|
||||||
# Create time-based drops
|
# Create time-based drops
|
||||||
for drop_data in drop_campaign_data["timeBasedDrops"]:
|
for drop_data in drop_campaign_data["timeBasedDrops"]:
|
||||||
@ -110,18 +130,29 @@ async def insert_data(data: dict) -> None: # noqa: PLR0914
|
|||||||
benefit_data = edge["benefit"]
|
benefit_data = edge["benefit"]
|
||||||
benefit_owner_data = benefit_data["ownerOrganization"]
|
benefit_owner_data = benefit_data["ownerOrganization"]
|
||||||
|
|
||||||
benefit_owner, _ = await sync_to_async(Organization.objects.get_or_create)(
|
benefit_owner, created = await sync_to_async(
|
||||||
|
Organization.objects.get_or_create,
|
||||||
|
)(
|
||||||
id=benefit_owner_data["id"],
|
id=benefit_owner_data["id"],
|
||||||
defaults={"name": benefit_owner_data["name"]},
|
defaults={"name": benefit_owner_data["name"]},
|
||||||
)
|
)
|
||||||
|
logger.debug(
|
||||||
|
"Benefit owner %s: %s",
|
||||||
|
"created" if created else "retrieved",
|
||||||
|
benefit_owner,
|
||||||
|
)
|
||||||
benefit_game_data = benefit_data["game"]
|
benefit_game_data = benefit_data["game"]
|
||||||
benefit_game, _ = await sync_to_async(Game.objects.get_or_create)(
|
benefit_game, created = await sync_to_async(Game.objects.get_or_create)(
|
||||||
id=benefit_game_data["id"],
|
id=benefit_game_data["id"],
|
||||||
defaults={"name": benefit_game_data["name"]},
|
defaults={"name": benefit_game_data["name"]},
|
||||||
)
|
)
|
||||||
|
logger.debug(
|
||||||
|
"Benefit game %s: %s",
|
||||||
|
"created" if created else "retrieved",
|
||||||
|
benefit_game,
|
||||||
|
)
|
||||||
|
|
||||||
benefit, _ = await sync_to_async(DropBenefit.objects.get_or_create)(
|
benefit, created = await sync_to_async(DropBenefit.objects.get_or_create)(
|
||||||
id=benefit_data["id"],
|
id=benefit_data["id"],
|
||||||
defaults={
|
defaults={
|
||||||
"created_at": benefit_data["createdAt"],
|
"created_at": benefit_data["createdAt"],
|
||||||
@ -134,8 +165,15 @@ async def insert_data(data: dict) -> None: # noqa: PLR0914
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
drop_benefits.append(benefit)
|
drop_benefits.append(benefit)
|
||||||
|
logger.debug(
|
||||||
|
"Benefit %s: %s",
|
||||||
|
"created" if created else "retrieved",
|
||||||
|
benefit,
|
||||||
|
)
|
||||||
|
|
||||||
time_based_drop, _ = await sync_to_async(TimeBasedDrop.objects.get_or_create)(
|
time_based_drop, created = await sync_to_async(
|
||||||
|
TimeBasedDrop.objects.get_or_create,
|
||||||
|
)(
|
||||||
id=drop_data["id"],
|
id=drop_data["id"],
|
||||||
defaults={
|
defaults={
|
||||||
"required_subs": drop_data["requiredSubs"],
|
"required_subs": drop_data["requiredSubs"],
|
||||||
@ -147,23 +185,41 @@ async def insert_data(data: dict) -> None: # noqa: PLR0914
|
|||||||
)
|
)
|
||||||
await sync_to_async(time_based_drop.benefits.set)(drop_benefits)
|
await sync_to_async(time_based_drop.benefits.set)(drop_benefits)
|
||||||
await sync_to_async(drop_campaign.time_based_drops.add)(time_based_drop)
|
await sync_to_async(drop_campaign.time_based_drops.add)(time_based_drop)
|
||||||
|
logger.debug(
|
||||||
|
"Time-based drop %s: %s",
|
||||||
|
"created" if created else "retrieved",
|
||||||
|
time_based_drop,
|
||||||
|
)
|
||||||
|
|
||||||
# Create or get the user
|
# Create or get the user
|
||||||
user, _ = await sync_to_async(User.objects.get_or_create)(id=user_id)
|
user, created = await sync_to_async(User.objects.get_or_create)(id=user_id)
|
||||||
await sync_to_async(user.drop_campaigns.add)(drop_campaign)
|
await sync_to_async(user.drop_campaigns.add)(drop_campaign)
|
||||||
|
logger.debug(
|
||||||
|
"User %s: %s",
|
||||||
|
"created" if created else "retrieved",
|
||||||
|
user,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
help = "Scrape Twitch Drops Campaigns with login using Firefox"
|
help = "Scrape Twitch Drops Campaigns with login using Firefox"
|
||||||
|
|
||||||
async def run(self, playwright: Playwright) -> list[Any]:
|
async def run( # noqa: PLR6301
|
||||||
|
self,
|
||||||
|
playwright: Playwright,
|
||||||
|
) -> list[dict[str, typing.Any]]:
|
||||||
profile_dir: Path = Path(data_dir / "firefox-profile")
|
profile_dir: Path = Path(data_dir / "firefox-profile")
|
||||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
logger.debug(
|
||||||
|
"Launching Firefox browser with user data directory: %s",
|
||||||
|
profile_dir,
|
||||||
|
)
|
||||||
|
|
||||||
browser: BrowserContext = await playwright.firefox.launch_persistent_context(
|
browser: BrowserContext = await playwright.firefox.launch_persistent_context(
|
||||||
user_data_dir=profile_dir,
|
user_data_dir=profile_dir,
|
||||||
headless=True,
|
headless=True,
|
||||||
)
|
)
|
||||||
|
logger.debug("Launched Firefox browser")
|
||||||
|
|
||||||
page: Page = await browser.new_page()
|
page: Page = await browser.new_page()
|
||||||
json_data: list[dict] = []
|
json_data: list[dict] = []
|
||||||
@ -173,11 +229,19 @@ class Command(BaseCommand):
|
|||||||
try:
|
try:
|
||||||
body: typing.Any = await response.json()
|
body: typing.Any = await response.json()
|
||||||
json_data.extend(body)
|
json_data.extend(body)
|
||||||
except Exception: # noqa: BLE001
|
logger.debug(
|
||||||
self.stdout.write(f"Failed to parse JSON from {response.url}")
|
"Received JSON data from %s",
|
||||||
|
response.url,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"Failed to parse JSON from %s",
|
||||||
|
response.url,
|
||||||
|
)
|
||||||
|
|
||||||
page.on("response", handle_response)
|
page.on("response", handle_response)
|
||||||
await page.goto("https://www.twitch.tv/drops/campaigns")
|
await page.goto("https://www.twitch.tv/drops/campaigns")
|
||||||
|
logger.debug("Navigated to Twitch drops campaigns page")
|
||||||
|
|
||||||
logged_in = False
|
logged_in = False
|
||||||
while not logged_in:
|
while not logged_in:
|
||||||
@ -187,14 +251,16 @@ class Command(BaseCommand):
|
|||||||
timeout=30000,
|
timeout=30000,
|
||||||
)
|
)
|
||||||
logged_in = True
|
logged_in = True
|
||||||
self.stdout.write("Logged in")
|
logger.info("Logged in to Twitch")
|
||||||
except KeyboardInterrupt as e:
|
except KeyboardInterrupt as e:
|
||||||
raise KeyboardInterrupt from e
|
raise KeyboardInterrupt from e
|
||||||
except Exception: # noqa: BLE001
|
except Exception: # noqa: BLE001
|
||||||
await asyncio.sleep(5)
|
await asyncio.sleep(5)
|
||||||
self.stdout.write("Waiting for login")
|
logger.info("Waiting for login")
|
||||||
|
|
||||||
await page.wait_for_load_state("networkidle")
|
await page.wait_for_load_state("networkidle")
|
||||||
|
logger.debug("Page is idle")
|
||||||
|
|
||||||
await browser.close()
|
await browser.close()
|
||||||
|
|
||||||
for campaign in json_data:
|
for campaign in json_data:
|
||||||
|
@ -2,14 +2,15 @@
|
|||||||
|
|
||||||
import django.db.models.deletion
|
import django.db.models.deletion
|
||||||
from django.db import migrations, models
|
from django.db import migrations, models
|
||||||
|
from django.db.migrations.operations.base import Operation
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
initial = True
|
initial = True
|
||||||
|
|
||||||
dependencies = []
|
dependencies: list[tuple[str, str]] = []
|
||||||
|
|
||||||
operations = [
|
operations: list[Operation] = [
|
||||||
migrations.CreateModel(
|
migrations.CreateModel(
|
||||||
name="Channel",
|
name="Channel",
|
||||||
fields=[
|
fields=[
|
||||||
|
@ -2,14 +2,15 @@
|
|||||||
|
|
||||||
import django.db.models.deletion
|
import django.db.models.deletion
|
||||||
from django.db import migrations, models
|
from django.db import migrations, models
|
||||||
|
from django.db.migrations.operations.base import Operation
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
dependencies = [
|
dependencies: list[tuple[str, str]] = [
|
||||||
("twitch", "0001_initial"),
|
("twitch", "0001_initial"),
|
||||||
]
|
]
|
||||||
|
|
||||||
operations = [
|
operations: list[Operation] = [
|
||||||
migrations.AlterModelOptions(
|
migrations.AlterModelOptions(
|
||||||
name="dropcampaign",
|
name="dropcampaign",
|
||||||
options={"verbose_name_plural": "Drop Campaigns"},
|
options={"verbose_name_plural": "Drop Campaigns"},
|
||||||
|
@ -0,0 +1,83 @@
|
|||||||
|
# Generated by Django 5.1a1 on 2024-06-22 19:21
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
from django.db.migrations.operations.base import Operation
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies: list[tuple[str, str]] = [
|
||||||
|
("twitch", "0002_alter_dropcampaign_options_alter_dropcampaign_game_and_more"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations: list[Operation] = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="channel",
|
||||||
|
name="added_at",
|
||||||
|
field=models.DateTimeField(auto_now_add=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="channel",
|
||||||
|
name="modified_at",
|
||||||
|
field=models.DateTimeField(auto_now=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="dropbenefit",
|
||||||
|
name="added_at",
|
||||||
|
field=models.DateTimeField(auto_now_add=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="dropbenefit",
|
||||||
|
name="modified_at",
|
||||||
|
field=models.DateTimeField(auto_now=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="dropcampaign",
|
||||||
|
name="added_at",
|
||||||
|
field=models.DateTimeField(auto_now_add=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="dropcampaign",
|
||||||
|
name="modified_at",
|
||||||
|
field=models.DateTimeField(auto_now=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="game",
|
||||||
|
name="added_at",
|
||||||
|
field=models.DateTimeField(auto_now_add=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="game",
|
||||||
|
name="modified_at",
|
||||||
|
field=models.DateTimeField(auto_now=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="organization",
|
||||||
|
name="added_at",
|
||||||
|
field=models.DateTimeField(auto_now_add=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="organization",
|
||||||
|
name="modified_at",
|
||||||
|
field=models.DateTimeField(auto_now=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="timebaseddrop",
|
||||||
|
name="added_at",
|
||||||
|
field=models.DateTimeField(auto_now_add=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="timebaseddrop",
|
||||||
|
name="modified_at",
|
||||||
|
field=models.DateTimeField(auto_now=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="user",
|
||||||
|
name="added_at",
|
||||||
|
field=models.DateTimeField(auto_now_add=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="user",
|
||||||
|
name="modified_at",
|
||||||
|
field=models.DateTimeField(auto_now=True, null=True),
|
||||||
|
),
|
||||||
|
]
|
17
twitch/migrations/0004_alter_dropcampaign_options.py
Normal file
17
twitch/migrations/0004_alter_dropcampaign_options.py
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
# Generated by Django 5.1a1 on 2024-06-22 19:27
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
from django.db.migrations.operations.base import Operation
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies: list[tuple[str, str]] = [
|
||||||
|
("twitch", "0003_channel_added_at_channel_modified_at_and_more"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations: list[Operation] = [
|
||||||
|
migrations.AlterModelOptions(
|
||||||
|
name="dropcampaign",
|
||||||
|
options={},
|
||||||
|
),
|
||||||
|
]
|
26
twitch/migrations/0005_game_twitch_url.py
Normal file
26
twitch/migrations/0005_game_twitch_url.py
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
# Generated by Django 5.1a1 on 2024-06-22 20:05
|
||||||
|
|
||||||
|
import django.db.models.functions.text
|
||||||
|
from django.db import migrations, models
|
||||||
|
from django.db.migrations.operations.base import Operation
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies: list[tuple[str, str]] = [
|
||||||
|
("twitch", "0004_alter_dropcampaign_options"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations: list[Operation] = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="game",
|
||||||
|
name="twitch_url",
|
||||||
|
field=models.GeneratedField( # type: ignore # noqa: PGH003
|
||||||
|
db_persist=True,
|
||||||
|
expression=django.db.models.functions.text.Concat(
|
||||||
|
models.Value("https://www.twitch.tv/directory/category/"),
|
||||||
|
"slug",
|
||||||
|
),
|
||||||
|
output_field=models.TextField(),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
@ -1,9 +1,15 @@
|
|||||||
from django.db import models
|
from django.db import models
|
||||||
|
from django.db.models import Value
|
||||||
|
from django.db.models.functions import (
|
||||||
|
Concat,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Organization(models.Model):
|
class Organization(models.Model):
|
||||||
id = models.TextField(primary_key=True)
|
id = models.TextField(primary_key=True)
|
||||||
name = models.TextField(blank=True, null=True)
|
name = models.TextField(blank=True, null=True)
|
||||||
|
added_at = models.DateTimeField(blank=True, null=True, auto_now_add=True)
|
||||||
|
modified_at = models.DateTimeField(blank=True, null=True, auto_now=True)
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return self.name or self.id
|
return self.name or self.id
|
||||||
@ -12,7 +18,14 @@ class Organization(models.Model):
|
|||||||
class Game(models.Model):
|
class Game(models.Model):
|
||||||
id = models.TextField(primary_key=True)
|
id = models.TextField(primary_key=True)
|
||||||
slug = models.TextField(blank=True, null=True)
|
slug = models.TextField(blank=True, null=True)
|
||||||
|
twitch_url = models.GeneratedField( # type: ignore # noqa: PGH003
|
||||||
|
expression=Concat(Value("https://www.twitch.tv/directory/category/"), "slug"),
|
||||||
|
output_field=models.TextField(),
|
||||||
|
db_persist=True,
|
||||||
|
)
|
||||||
display_name = models.TextField(blank=True, null=True)
|
display_name = models.TextField(blank=True, null=True)
|
||||||
|
added_at = models.DateTimeField(blank=True, null=True, auto_now_add=True)
|
||||||
|
modified_at = models.DateTimeField(blank=True, null=True, auto_now=True)
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return self.display_name or self.slug or self.id
|
return self.display_name or self.slug or self.id
|
||||||
@ -22,6 +35,8 @@ class Channel(models.Model):
|
|||||||
id = models.TextField(primary_key=True)
|
id = models.TextField(primary_key=True)
|
||||||
display_name = models.TextField(blank=True, null=True)
|
display_name = models.TextField(blank=True, null=True)
|
||||||
name = models.TextField(blank=True, null=True)
|
name = models.TextField(blank=True, null=True)
|
||||||
|
added_at = models.DateTimeField(blank=True, null=True, auto_now_add=True)
|
||||||
|
modified_at = models.DateTimeField(blank=True, null=True, auto_now=True)
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return self.display_name or self.name or self.id
|
return self.display_name or self.name or self.id
|
||||||
@ -36,6 +51,8 @@ class DropBenefit(models.Model):
|
|||||||
name = models.TextField(blank=True, null=True)
|
name = models.TextField(blank=True, null=True)
|
||||||
owner_organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
|
owner_organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
|
||||||
game = models.ForeignKey(Game, on_delete=models.CASCADE)
|
game = models.ForeignKey(Game, on_delete=models.CASCADE)
|
||||||
|
added_at = models.DateTimeField(blank=True, null=True, auto_now_add=True)
|
||||||
|
modified_at = models.DateTimeField(blank=True, null=True, auto_now=True)
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return self.name or self.id
|
return self.name or self.id
|
||||||
@ -49,6 +66,8 @@ class TimeBasedDrop(models.Model):
|
|||||||
required_minutes_watched = models.IntegerField(blank=True, null=True)
|
required_minutes_watched = models.IntegerField(blank=True, null=True)
|
||||||
start_at = models.DateTimeField(blank=True, null=True)
|
start_at = models.DateTimeField(blank=True, null=True)
|
||||||
benefits = models.ManyToManyField(DropBenefit)
|
benefits = models.ManyToManyField(DropBenefit)
|
||||||
|
added_at = models.DateTimeField(blank=True, null=True, auto_now_add=True)
|
||||||
|
modified_at = models.DateTimeField(blank=True, null=True, auto_now=True)
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return self.name or self.id
|
return self.name or self.id
|
||||||
@ -76,9 +95,8 @@ class DropCampaign(models.Model):
|
|||||||
)
|
)
|
||||||
channels = models.ManyToManyField(Channel)
|
channels = models.ManyToManyField(Channel)
|
||||||
time_based_drops = models.ManyToManyField(TimeBasedDrop)
|
time_based_drops = models.ManyToManyField(TimeBasedDrop)
|
||||||
|
added_at = models.DateTimeField(blank=True, null=True, auto_now_add=True)
|
||||||
class Meta:
|
modified_at = models.DateTimeField(blank=True, null=True, auto_now=True)
|
||||||
verbose_name_plural = "Drop Campaigns"
|
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return self.name or self.id
|
return self.name or self.id
|
||||||
@ -87,6 +105,8 @@ class DropCampaign(models.Model):
|
|||||||
class User(models.Model):
|
class User(models.Model):
|
||||||
id = models.TextField(primary_key=True)
|
id = models.TextField(primary_key=True)
|
||||||
drop_campaigns = models.ManyToManyField(DropCampaign)
|
drop_campaigns = models.ManyToManyField(DropCampaign)
|
||||||
|
added_at = models.DateTimeField(blank=True, null=True, auto_now_add=True)
|
||||||
|
modified_at = models.DateTimeField(blank=True, null=True, auto_now=True)
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return self.id
|
return self.id
|
||||||
|
Reference in New Issue
Block a user