201 lines
7.5 KiB
Python
201 lines
7.5 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import httpx
|
|
from django.core.management.base import BaseCommand
|
|
from django.db.models.query import QuerySet
|
|
|
|
from twitch.models import Game
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Generator
|
|
|
|
from django.db.models import QuerySet
|
|
|
|
|
|
MAX_AMOUNT_GAMES_PER_REQUEST = 100
|
|
"""Maximum number of games that can be requested from Twitch API in a single request."""
|
|
|
|
|
|
class TwitchTooManyGamesError(Exception):
|
|
"""Raised when too many game IDs are requested from the Twitch API."""
|
|
|
|
def __init__(self, max_allowed: int, actual: int) -> None:
|
|
"""Initialize the exception with details about the limit breach.
|
|
|
|
Args:
|
|
max_allowed (int): The maximum number of game IDs allowed per request.
|
|
actual (int): The actual number of game IDs requested.
|
|
"""
|
|
self.max_allowed: int = max_allowed
|
|
self.actual: int = actual
|
|
super().__init__(f"Requested {actual} game IDs, but the maximum allowed is {max_allowed}.")
|
|
|
|
|
|
@dataclass
|
|
class AuthResponse:
|
|
"""Represents a response from the Twitch authentication endpoint.
|
|
|
|
{
|
|
"access_token": "access12345token",
|
|
"expires_in": 5587808,
|
|
"token_type": "bearer"
|
|
}
|
|
|
|
The expires_in shows you the number of seconds before the access_token will expire and must be refreshed.
|
|
"""
|
|
|
|
access_token: str
|
|
"""OAuth access token."""
|
|
|
|
expires_in: int
|
|
"""The number of seconds before the access_token expires."""
|
|
|
|
token_type: str
|
|
"""The type of the token. For example "bearer"."""
|
|
|
|
|
|
def download_images(twitch_client_id: str, client: httpx.Client, auth: AuthResponse, game_ids: list[int]) -> httpx.Response:
|
|
"""Downloads game data including images from the IGDB API.
|
|
|
|
This function sends a POST request to the IGDB API to fetch game data with all available fields.
|
|
The request is authorized using the Twitch client ID and access token.
|
|
|
|
Args:
|
|
twitch_client_id (str): The Twitch client ID used for authentication with IGDB API.
|
|
client (httpx.Client): An instance of httpx.Client for making HTTP requests.
|
|
auth (AuthResponse): The authentication response object containing the access token.
|
|
game_ids (list[int]): A list of game IDs to fetch data for.
|
|
|
|
Returns:
|
|
httpx.Response: The HTTP response from the IGDB API containing game data.
|
|
"""
|
|
url = "https://api.igdb.com/v4/games"
|
|
|
|
body: str = "fields *; where id = (" + ",".join(str(gid) for gid in game_ids) + ");"
|
|
|
|
response: httpx.Response = client.post(
|
|
url=url,
|
|
headers={"Client-ID": twitch_client_id, "Authorization": f"Bearer {auth.access_token}"},
|
|
content=body,
|
|
timeout=30.0,
|
|
)
|
|
return response
|
|
|
|
|
|
def batched_queryset(qs: QuerySet, batch_size: int = 500) -> Generator[list[Any], Any]:
|
|
"""Process a Django QuerySet in batches to avoid loading all records into memory at once.
|
|
|
|
This function yields lists of igdb_id values from the provided QuerySet in batches
|
|
of the specified size, allowing for efficient processing of large datasets.
|
|
|
|
Args:
|
|
qs (QuerySet): The Django QuerySet to process in batches
|
|
batch_size (int, optional): The number of records to include in each batch. Defaults to 500.
|
|
|
|
Yields:
|
|
list[Any]: A batch of igdb_id values from the QuerySet
|
|
|
|
Example:
|
|
>>> games = Game.objects.all()
|
|
>>> for batch in batched_queryset(games):
|
|
... process_game_batch(batch)
|
|
"""
|
|
start = 0
|
|
while True:
|
|
batch: list[Any] = list(qs.values_list("igdb_id", flat=True)[start : start + batch_size])
|
|
if not batch:
|
|
break
|
|
yield batch
|
|
start += batch_size
|
|
|
|
|
|
def get_twitch_data(twitch_client_id: str, game_ids: list[int]) -> httpx.Response:
|
|
"""Fetches game data from the Twitch API.
|
|
|
|
Args:
|
|
twitch_client_id (str): The Twitch client ID used for authentication.
|
|
game_ids (list[int]): A list of game IDs to fetch data for.
|
|
|
|
Raises:
|
|
TwitchTooManyGamesError: If the number of game IDs exceeds the maximum allowed per request.
|
|
ValueError: If the request parameters are invalid.
|
|
PermissionError: If the authentication fails due to invalid credentials.
|
|
|
|
References:
|
|
- Twitch API Documentation: https://dev.twitch.tv/docs/api/reference#get-games
|
|
|
|
Returns:
|
|
httpx.Response: The HTTP response from the Twitch API containing game data.
|
|
"""
|
|
if len(game_ids) > MAX_AMOUNT_GAMES_PER_REQUEST:
|
|
raise TwitchTooManyGamesError(MAX_AMOUNT_GAMES_PER_REQUEST, len(game_ids))
|
|
|
|
with httpx.Client() as client:
|
|
response: httpx.Response = client.get(
|
|
url="https://api.twitch.tv/helix/games",
|
|
headers={"Client-ID": twitch_client_id},
|
|
params=[("id", str(gid)) for gid in game_ids],
|
|
timeout=30.0,
|
|
)
|
|
|
|
if response.status_code == httpx.codes.OK:
|
|
# Successfully retrieved the specified games.
|
|
return response
|
|
|
|
if response.status_code == httpx.codes.BAD_REQUEST:
|
|
# The request must specify the id or name or igdb_id query parameter.
|
|
# The combined number of game IDs (id and igdb_id) and game names that you specify in the request must not exceed 100.
|
|
if len(game_ids) > MAX_AMOUNT_GAMES_PER_REQUEST:
|
|
raise TwitchTooManyGamesError(MAX_AMOUNT_GAMES_PER_REQUEST, len(game_ids))
|
|
|
|
msg: str = f"Bad Request: Check that the request parameters are correct. Response: {response.text}"
|
|
raise ValueError(msg)
|
|
|
|
if response.status_code == httpx.codes.UNAUTHORIZED:
|
|
# The Authorization header is required and must specify an app access token or user access token.
|
|
# The access token is not valid.
|
|
# The ID in the Client-Id header must match the client ID in the access token.
|
|
msg = "Unauthorized: Check that the access token and client ID are correct."
|
|
raise PermissionError(msg)
|
|
|
|
return response
|
|
|
|
|
|
class Command(BaseCommand):
|
|
"""Populate database with data from Twitch and IGDB APIs."""
|
|
|
|
help: str = __doc__ or ""
|
|
|
|
def handle(self, **options) -> None: # noqa: ARG002
|
|
"""Execute the image download process."""
|
|
twitch_client_id: str | None = os.getenv("TWITCH_CLIENT_ID")
|
|
twitch_client_secret: str | None = os.getenv("TWITCH_CLIENT_SECRET")
|
|
if not twitch_client_id or not twitch_client_secret:
|
|
self.stderr.write(self.style.ERROR("TWITCH_CLIENT_ID and TWITCH_CLIENT_SECRET must be set in environment"))
|
|
return
|
|
|
|
with httpx.Client() as client:
|
|
response: httpx.Response = client.post(
|
|
url="https://id.twitch.tv/oauth2/token",
|
|
params={
|
|
"client_id": twitch_client_id,
|
|
"client_secret": twitch_client_secret,
|
|
"grant_type": "client_credentials",
|
|
},
|
|
)
|
|
|
|
auth: AuthResponse = AuthResponse(**response.json())
|
|
|
|
self.stdout.write(self.style.SUCCESS(f"Auth response: {response.text}"))
|
|
|
|
# Get game_ids from the database
|
|
qs: QuerySet[Game, Game] = Game.objects.all()
|
|
for game_ids in batched_queryset(qs, batch_size=500):
|
|
with httpx.Client() as client:
|
|
download_images(twitch_client_id=twitch_client_id, client=client, auth=auth, game_ids=game_ids)
|
|
|
|
self.stdout.write(self.style.SUCCESS(f"Fetched {len(game_ids)} games"))
|