From 5658f88abf79fab131e426e003d424ca1a8d9d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Fri, 9 May 2025 05:02:43 +0200 Subject: [PATCH] Add articles to README --- README.md | 303 +++++++++++++++++++++++++++++++++++++++++++++++++++++- scrape.py | 210 +++++++++++++++++++++++++++---------- 2 files changed, 456 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index c52684b..e3cf02b 100644 --- a/README.md +++ b/README.md @@ -1 +1,302 @@ -# wutheringwaves \ No newline at end of file +# wutheringwaves + +## Articles + +- [[Chord Cleansing] Limited-Time Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2584) +- [V2.3 Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/2578) +- [Wuthering Waves Fan Creation Event "Fiery Arpeggio of Summer Reunion" is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/2577) +- [Wuthering Waves Fan Creation Event Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/2569) +- [[Tidal Photography] Now seeking top photographers for Solaris!](https://wutheringwaves.kurogames.com/en/main/news/detail/2564) +- [Version 2.3: Fiery Arpeggio of Summer Reunion Patch Notes](https://wutheringwaves.kurogames.com/en/main/news/detail/2561) +- [[Anniversary Weapon Convene: Phase I]](https://wutheringwaves.kurogames.com/en/main/news/detail/2535) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Blazing Justice](https://wutheringwaves.kurogames.com/en/main/news/detail/2534) +- [[Anniversary Resonator Convene: Phase I]](https://wutheringwaves.kurogames.com/en/main/news/detail/2533) +- [[Between Light and Shadow] Featured Resonator Convene: Boosted Drop Rate for Zani](https://wutheringwaves.kurogames.com/en/main/news/detail/2532) +- [Anniversary Resonator Convene: Phase I](https://wutheringwaves.kurogames.com/en/main/news/detail/2527) +- [Tactical Hologram Update, Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/2524) +- [Reminder Regarding Recent Complaints About Unauthorized Payment Platforms](https://wutheringwaves.kurogames.com/en/main/news/detail/2520) +- [Wuthering Waves Update Content | Version 2.3 "Fiery Arpeggio of Summer Reunion" Planned for Release on April 29 (UTC+8)](https://wutheringwaves.kurogames.com/en/main/news/detail/2517) +- [Event Preview | [Cubie Derby] Anniversary Events, Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/2513) +- [Profile Reveal | Scorched Radiance — Zani](https://wutheringwaves.kurogames.com/en/main/news/detail/2509) +- [Event Preview | [Cube, Cubic n Cubie] Anniversary Combat Event, Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/2504) +- [Post-Lament Anthropocene: Stars Intertwined | Zani](https://wutheringwaves.kurogames.com/en/main/news/detail/2500) +- [Wuthering Waves Version 2.3 Update Maintenance Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/2487) +- [Wuthering Waves Version Preview | Version 2.3](https://wutheringwaves.kurogames.com/en/main/news/detail/2485) +- [Resonator Review | Euphonic Chrysalis—Shorekeeper](https://wutheringwaves.kurogames.com/en/main/news/detail/2466) +- [Upcoming Events in Wuthering Waves Version 2.2](https://wutheringwaves.kurogames.com/en/main/news/detail/2465) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Stellar Symphony](https://wutheringwaves.kurogames.com/en/main/news/detail/2459) +- [[Till the Sea Turns Clear] Featured Resonator Convene: Boosted Drop Rate for Shorekeeper](https://wutheringwaves.kurogames.com/en/main/news/detail/2452) +- [Notice on the Upgrade and Maintenance of the [Reprise of Tides] Returnee Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2443) +- [[Wuthering Exploration] Featured Exploration Event: Whisperwind Haven](https://wutheringwaves.kurogames.com/en/main/news/detail/2432) +- [[Ragunna Weather Forecast] Featured Platforming Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2422) +- [[Lollo Campaign: Set Sail] Featured Commission Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2415) +- [[Chord Cleansing] Limited-Time Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2404) +- [[Virtual Crisis: Prototype Trials] Crisis Trial Challenge Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2396) +- [V2.2 Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/2383) +- [Wuthering Waves Fan Creation Event "Tangled Truth in Inverted Tower" is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/2382) +- [Version 2.2: Tangled Truth in Inverted Tower Patch Notes](https://wutheringwaves.kurogames.com/en/main/news/detail/2381) +- [Wuthering Waves Fan Creation Event Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/2369) +- [Post-Lament Anthropocene: Enemies — Fleurdelys](https://wutheringwaves.kurogames.com/en/main/news/detail/2364) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Red Spring](https://wutheringwaves.kurogames.com/en/main/news/detail/2343) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Whispers of Sirens](https://wutheringwaves.kurogames.com/en/main/news/detail/2342) +- [[End of Lost Trail] Featured Resonator Convene: Boosted Drop Rate for Camellya](https://wutheringwaves.kurogames.com/en/main/news/detail/2341) +- [[Neptune's Lullaby] Featured Resonator Convene: Boosted Drop Rate for Cantarella](https://wutheringwaves.kurogames.com/en/main/news/detail/2340) +- [Wuthering Waves Update Content | Version 2.2 "Tangled Truth in Inverted Tower" Planned for Release on March 27(UTC+8)](https://wutheringwaves.kurogames.com/en/main/news/detail/2329) +- [New Region Preview | Avinoleum Theological Seminary, Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/2317) +- [Post-Lament Anthropocene: Stars Intertwined | Cantarella](https://wutheringwaves.kurogames.com/en/main/news/detail/2311) +- [Profile Reveal | Sea of Dreams — Cantarella](https://wutheringwaves.kurogames.com/en/main/news/detail/2310) +- [Profile Reveal | Vessel of Sounds — Rover: Aero](https://wutheringwaves.kurogames.com/en/main/news/detail/2305) +- [Post-Lament Anthropocene: Enemies — Collection of Small & Medium Size Echoes in Rinascita](https://wutheringwaves.kurogames.com/en/main/news/detail/2303) +- [Wuthering Waves Version 2.2 Update Maintenance Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/2302) +- [Wuthering Waves Version Update Preview | Version 2.2 "Tangled Truth in Inverted Tower" Planned for Release on March 27, 2025 (UTC+8)](https://wutheringwaves.kurogames.com/en/main/news/detail/2297) +- [[Bountiful Crescendo] Limited-Time Material Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2273) +- [[Beyond the Waves] Featured Exploration Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2261) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Blazing Brilliance](https://wutheringwaves.kurogames.com/en/main/news/detail/2229) +- [[Vermillion's Ploy] Featured Resonator Convene: Boosted Drop Rate for Changli](https://wutheringwaves.kurogames.com/en/main/news/detail/2228) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Unflickering Valor](https://wutheringwaves.kurogames.com/en/main/news/detail/2227) +- [[Blaze Across the Deep] Featured Resonator Convene: Boosted Drop Rate for Brant](https://wutheringwaves.kurogames.com/en/main/news/detail/2226) +- [Upcoming Events in Wuthering Waves Version 2.1](https://wutheringwaves.kurogames.com/en/main/news/detail/2219) +- [Post-Lament Anthropocene: Stars Intertwined | Brant](https://wutheringwaves.kurogames.com/en/main/news/detail/2204) +- [[Tactical Simulacra Ⅲ] Featured Combat Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2193) +- [Profile Reveal | Flamebound Compass — Brant](https://wutheringwaves.kurogames.com/en/main/news/detail/2192) +- [[Chord Cleansing] Limited-Time Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2180) +- [[Apex Ragunna] Featured Co-op Overdash Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2173) +- [Wuthering Waves Fan Creation Event "Waves Sing, and the Cerulean Bird Calls" is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/2162) +- [Patch Notes for Version 2.1: Waves Sing, and the Cerulean Bird Calls](https://wutheringwaves.kurogames.com/en/main/news/detail/2154) +- [V2.1 Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/2153) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Luminous Hymn](https://wutheringwaves.kurogames.com/en/main/news/detail/2142) +- [[With Hushed Whispers] Featured Resonator Convene: Boosted Drop Rate for Phoebe](https://wutheringwaves.kurogames.com/en/main/news/detail/2141) +- [Wuthering Waves Fan Creation Event Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/2135) +- [Waves Sing, and the Cerulean Bird Calls — Version 2.1 Update Content](https://wutheringwaves.kurogames.com/en/main/news/detail/2123) +- [Fantasies of the Thousand Gateways Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/2115) +- [Special Combat Mechanics Explain | Negative Status](https://wutheringwaves.kurogames.com/en/main/news/detail/2112) +- [Profile Reveal | Graceful Luminescence — Phoebe](https://wutheringwaves.kurogames.com/en/main/news/detail/2106) +- [Post-Lament Anthropocene: Stars Intertwined | Phoebe](https://wutheringwaves.kurogames.com/en/main/news/detail/2101) +- [Event Preview | [Old Man and the Whale] Fishing Adventure Event Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/2100) +- [Whimpering Wastes Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/2094) +- [Post-Lament Anthropocene: Enemies — Collection of Small & Medium Size Echoes in Rinascita](https://wutheringwaves.kurogames.com/en/main/news/detail/2091) +- [Wuthering Waves Version 2.1 Update Maintenance Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/2083) +- [Wuthering Waves Version 2.1 "Waves Sing, and the Cerulean Bird Calls" is coming soon.](https://wutheringwaves.kurogames.com/en/main/news/detail/2072) +- [[Bountiful Crescendo] Limited-Time Material Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2069) +- [[Lollo Campaign: Comeback] Featured Commission Event](https://wutheringwaves.kurogames.com/en/main/news/detail/2066) +- [[Celebration Aftertunes] Gratitude for Companionship](https://wutheringwaves.kurogames.com/en/main/news/detail/2065) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Ages of Harvest](https://wutheringwaves.kurogames.com/en/main/news/detail/2047) +- [[Thawborn Renewal] Featured Resonator Convene: Boosted Drop Rate for Jinhsi](https://wutheringwaves.kurogames.com/en/main/news/detail/2046) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Tragicomedy](https://wutheringwaves.kurogames.com/en/main/news/detail/2045) +- [[Stage in the Box] Featured Resonator Convene: Boosted Drop Rate for Roccia](https://wutheringwaves.kurogames.com/en/main/news/detail/2044) +- [Upcoming Events in Wuthering Waves Version 2.0](https://wutheringwaves.kurogames.com/en/main/news/detail/2021) +- [Sanhua's Signature Outfit "Exorcistic Adjuration"](https://wutheringwaves.kurogames.com/en/main/news/detail/2015) +- [Post-Lament Anthropocene: Enemies — Dragon of Dirge](https://wutheringwaves.kurogames.com/en/main/news/detail/2009) +- [Profile Reveal | Stage in the Box — Roccia](https://wutheringwaves.kurogames.com/en/main/news/detail/2000) +- [Post-Lament Anthropocene: Stars Intertwined | Roccia](https://wutheringwaves.kurogames.com/en/main/news/detail/1989) +- [[Chord Cleansing] Limited-Time Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1985) +- [[Wuthering Exploration] Featured Exploration Event: Nimbus Sanctum](https://wutheringwaves.kurogames.com/en/main/news/detail/1984) +- [Post-Lament Anthropocene: Enemies — Hecate](https://wutheringwaves.kurogames.com/en/main/news/detail/1967) +- [How to Use Wuwa Bot?](https://wutheringwaves.kurogames.com/en/main/news/detail/1959) +- [[Although Colors Have No Form] Featured Co-op Challenge Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1954) +- [V2.0 WUWA Discord Quest Event Begins!](https://wutheringwaves.kurogames.com/en/main/news/detail/1945) +- [Wuthering Waves Fan Creation Event "All Silent Souls Can Sing" is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/1935) +- [V2.0 Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/1929) +- [Version 2.0: All Silent Souls Can Sing Patch Notes](https://wutheringwaves.kurogames.com/en/main/news/detail/1912) +- [Post-Lament Anthropocene: Enemies — Lorelei](https://wutheringwaves.kurogames.com/en/main/news/detail/1908) +- [Event Preview | Tactical Hologram: Phantom Pain Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1905) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Rime-Draped Sprouts](https://wutheringwaves.kurogames.com/en/main/news/detail/1901) +- [[Chromatic Prose] Featured Resonator Convene: Boosted Drop Rate for Zhezhi](https://wutheringwaves.kurogames.com/en/main/news/detail/1899) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for The Last Dance](https://wutheringwaves.kurogames.com/en/main/news/detail/1898) +- [[When Silence Tolls] Featured Resonator Convene: Boosted Drop Rate for Carlotta](https://wutheringwaves.kurogames.com/en/main/news/detail/1897) +- [Wuthering Waves Fan Creation Event Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/1883) +- [Post-Lament Anthropocene: Enemies — Sentry Construct](https://wutheringwaves.kurogames.com/en/main/news/detail/1881) +- [Retrospective | Enchanted Brush — Zhezhi](https://wutheringwaves.kurogames.com/en/main/news/detail/1871) +- [Wuthering Waves Update Preview | Version 2.0 "All Silent Souls Can Sing" to be released on Jan 2nd, 2025 (UTC+8)](https://wutheringwaves.kurogames.com/en/main/news/detail/1864) +- [FAQ – PlayStation®5](https://wutheringwaves.kurogames.com/en/main/news/detail/1858) +- [Post-Lament Anthropocene: Stars Intertwined | Carlotta](https://wutheringwaves.kurogames.com/en/main/news/detail/1852) +- [Profile Reveal | Crystal Remorph — Carlotta](https://wutheringwaves.kurogames.com/en/main/news/detail/1846) +- [Post-Lament Anthropocene: Weapons — Poetry and Paean Series](https://wutheringwaves.kurogames.com/en/main/news/detail/1840) +- [Wuthering Waves Version 2.0 Update Maintenance Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/1822) +- [Post-Lament Anthropocene: Enemies — Collection of Small & Medium Size Echoes in Rinascita](https://wutheringwaves.kurogames.com/en/main/news/detail/1821) +- [Jinhsi's Premium Outfit "Peach Blossom"](https://wutheringwaves.kurogames.com/en/main/news/detail/1810) +- [A Sneak Peek of Adventures in Rinascita | Collectibles and Challenges](https://wutheringwaves.kurogames.com/en/main/news/detail/1809) +- [[Chord Cleansing] Limited-Time Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1803) +- [A Sneak Peek of Adventures in Rinascita | Echo Gameplays](https://wutheringwaves.kurogames.com/en/main/news/detail/1797) +- [Wuthering Waves Version Update Preview | Version 2.0 All Silent Souls Can Sing Planned for Release on Jan 02, 2025](https://wutheringwaves.kurogames.com/en/main/news/detail/1795) +- [[Tactical Simulacra II] Featured Combat Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1788) +- [Update on Shipping Delay for Select Winter Invitation Merchandise](https://wutheringwaves.kurogames.com/en/main/news/detail/1772) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Verity's Handle](https://wutheringwaves.kurogames.com/en/main/news/detail/1771) +- [[Celestial Revelation] Featured Resonator Convene: Boosted Drop Rate for Xiangli Yao](https://wutheringwaves.kurogames.com/en/main/news/detail/1770) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Stringmaster](https://wutheringwaves.kurogames.com/en/main/news/detail/1769) +- [[When Thunder Pours] Featured Resonator Convene: Boosted Drop Rate for Yinlin](https://wutheringwaves.kurogames.com/en/main/news/detail/1768) +- [Upcoming Events in Wuthering Waves Version 1.4](https://wutheringwaves.kurogames.com/en/main/news/detail/1738) +- [Profile Reveal | Kaleido Refraction — Lumi](https://wutheringwaves.kurogames.com/en/main/news/detail/1734) +- [Post-Lament Anthropocene: Stars Intertwined | Lumi](https://wutheringwaves.kurogames.com/en/main/news/detail/1728) +- [[Chord Cleansing] Limited-Time Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1715) +- [[Beyond the Waves] Featured Exploration Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1693) +- [V1.4 Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/1678) +- [Wuthering Waves Fan Creation Event "When the night knocks" is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/1671) +- [Wuthering Waves Fan Creation Event Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/1661) +- [Version 1.4: When the Night Knocks Patch Notes](https://wutheringwaves.kurogames.com/en/main/news/detail/1647) +- [Event Preview | Depths of Illusive Realm: Phantasm Amass, Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1646) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Red Spring](https://wutheringwaves.kurogames.com/en/main/news/detail/1645) +- [[End of Lost Trail] Featured Resonator Convene: Boosted Drop Rate for Camellya](https://wutheringwaves.kurogames.com/en/main/news/detail/1644) +- [Event Preview | Somnium Labyrinth: Somnoire Adventure Event, Available Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1631) +- [Wuthering Waves Update Preview | Version 1.4 "When the Night Knocks" to be released on Nov 14th](https://wutheringwaves.kurogames.com/en/main/news/detail/1607) +- [Wuthering Waves Version 1.4 Update Maintenance Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/1606) +- [Profile Reveal | Crimson Blossom — Camellya](https://wutheringwaves.kurogames.com/en/main/news/detail/1605) +- [Post-Lament Anthropocene: Stars Intertwined | Camellya](https://wutheringwaves.kurogames.com/en/main/news/detail/1596) +- [[Chord Cleansing] Limited-Time Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1588) +- [Wuthering Waves Version Preview | Version 1.4 "When the Night Knocks" Coming November 14 (UTC+8)](https://wutheringwaves.kurogames.com/en/main/news/detail/1573) +- [Wuthering Waves Developer's Message](https://wutheringwaves.kurogames.com/en/main/news/detail/1567) +- [[Wuthering Exploration] Featured Exploration Event: Mt. Firmament](https://wutheringwaves.kurogames.com/en/main/news/detail/1562) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Verdant Summit](https://wutheringwaves.kurogames.com/en/main/news/detail/1542) +- [[Prevail the Lasting Night] Featured Resonator Convene: Boosted Drop Rate for Jiyan](https://wutheringwaves.kurogames.com/en/main/news/detail/1541) +- [Developer's Message](https://wutheringwaves.kurogames.com/en/main/news/detail/1535) +- [Profile Reveal | Cryogenic Wonders — Youhu](https://wutheringwaves.kurogames.com/en/main/news/detail/1525) +- [Post-Lament Anthropocene: Stars Intertwined | Youhu](https://wutheringwaves.kurogames.com/en/main/news/detail/1524) +- [A Compilation of Upcoming Events in Wuthering Waves Version 1.3](https://wutheringwaves.kurogames.com/en/main/news/detail/1509) +- [[Bountiful Crescendo] Limited-Time Material Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1502) +- [[Wuthering Exploration] Featured Exploration Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1492) +- [[Solaris Weather Forecast] Featured Platforming Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1491) +- [Wutherium Geographic: Lands and Waters — The Black Shores Vol. 2](https://wutheringwaves.kurogames.com/en/main/news/detail/1474) +- [[Fairy Tale's Finale] Featured Co-op Combat Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1473) +- [Version 1.3: To the Shore's End Patch Notes](https://wutheringwaves.kurogames.com/en/main/news/detail/1454) +- [Post-Lament Anthropocene: Enemies — Tactical Hologram: Inferno Rider](https://wutheringwaves.kurogames.com/en/main/news/detail/1422) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Stellar Symphony](https://wutheringwaves.kurogames.com/en/main/news/detail/1421) +- [[Till the Sea Turns Clear] Featured Resonator Convene: Boosted Drop Rate for Shorekeeper](https://wutheringwaves.kurogames.com/en/main/news/detail/1420) +- [Profile Reveal | Euphonic Chrysalis — The Shorekeeper](https://wutheringwaves.kurogames.com/en/main/news/detail/1419) +- [Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/1418) +- [Wuthering Waves Fan Creation Event "To the Shore's End" is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/1417) +- [Wuthering Waves Update Preview | Version 1.3 "To the Shore's End" to be released on Sep 29th.](https://wutheringwaves.kurogames.com/en/main/news/detail/1408) +- [Post-Lament Anthropocene: Stars Intertwined | The Shorekeeper](https://wutheringwaves.kurogames.com/en/main/news/detail/1398) +- [Version 1.3 [The Black Shores] Special Leveling Events](https://wutheringwaves.kurogames.com/en/main/news/detail/1386) +- [Wuthering Waves Fan Creation Event Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/1381) +- [Wuthering Waves Version 1.3 Update Maintenance Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/1374) +- [Wutherium Geographic: Lands and Waters — The Black Shores Vol. 1](https://wutheringwaves.kurogames.com/en/main/news/detail/1365) +- [New Map Preview | "The Black Shores" Opening Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1361) +- [Post-Lament Anthropocene: Weapons — Deep Sky Blazar](https://wutheringwaves.kurogames.com/en/main/news/detail/1354) +- [Post-Lament Anthropocene: Enemies — Fallacy of No Return](https://wutheringwaves.kurogames.com/en/main/news/detail/1345) +- [Post-Lament Anthropocene: Comprehensive Mirror for Historians — The Black Shores](https://wutheringwaves.kurogames.com/en/main/news/detail/1336) +- [Wuthering Waves Version Preview | Version 1.3 "To the Shore's End" Coming September 29](https://wutheringwaves.kurogames.com/en/main/news/detail/1328) +- [Updates for PC Launcher 1.6.2](https://wutheringwaves.kurogames.com/en/main/news/detail/1320) +- [[Chord Cleansing] Featured Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1316) +- [[Alloy Smelt II] Limited-Time Battle Challenge](https://wutheringwaves.kurogames.com/en/main/news/detail/1306) +- [[Wuthering Exploration] Limited-Time Exploration Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1296) +- [V1.2 WUWA Discord Quest Livestream Event Begins!](https://wutheringwaves.kurogames.com/en/main/news/detail/1288) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Verity's Handle](https://wutheringwaves.kurogames.com/en/main/news/detail/1280) +- [[Celestial Revelation] Featured Resonator Convene: Boosted Drop Rate for Xiangli Yao](https://wutheringwaves.kurogames.com/en/main/news/detail/1279) +- [[Do Echoids Dream of Electric Sheeps] Limited-Time Defense Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1268) +- [Profile Reveal | Matter Weaver — Xiangli Yao](https://wutheringwaves.kurogames.com/en/main/news/detail/1255) +- [[Converging Paths] The Moon-Chasing Festival Preparation](https://wutheringwaves.kurogames.com/en/main/news/detail/1250) +- [Wuthering Waves Developer's Message: The Black Shores Preview](https://wutheringwaves.kurogames.com/en/main/news/detail/1240) +- [[Bountiful Crescendo] Featured Material Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1233) +- [Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/1226) +- [Event Notice | Depths of Illusive Realm: Carnival in Slumberland Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1218) +- [Wuthering Waves Fan Creation Event Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/1203) +- [Wuthering Waves Fan Creation Event “In the Turquoise Moonglow” is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/1198) +- [Version 1.2: In the Turquoise Moonglow Patch Notes](https://wutheringwaves.kurogames.com/en/main/news/detail/1190) +- [Event Notice | "Reprise of Tides" Returnee Event Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1178) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Rime-Draped Sprouts](https://wutheringwaves.kurogames.com/en/main/news/detail/1171) +- [[Chromatic Prose] Featured Resonator Convene: Boosted Drop Rate for Zhezhi](https://wutheringwaves.kurogames.com/en/main/news/detail/1170) +- [Event Notice | "By Moon's Grace" Moon-Chasing Festival Themed Event Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1157) +- [Wuthering Waves Version Overview | Version 1.2 "In the Turquoise Moonglow" Coming August 15](https://wutheringwaves.kurogames.com/en/main/news/detail/1147) +- [Profile Reveal | Enchancted Brush — Zhezhi](https://wutheringwaves.kurogames.com/en/main/news/detail/1144) +- [Wuthering Waves Version 1.2 Update Maintenance Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/1137) +- [Wuthering Waves Official Launcher Pre-Download Feature Available Soon via Grayscale Release](https://wutheringwaves.kurogames.com/en/main/news/detail/1127) +- [Wutherium Geographic: Lands and Waters — Extra Volume: Culture and Customs](https://wutheringwaves.kurogames.com/en/main/news/detail/1121) +- [Wuthering Waves Version Preview | Version 1.2 "In the Turquoise Moonglow" Coming August 15](https://wutheringwaves.kurogames.com/en/main/news/detail/1114) +- [Wuthering Waves Developer's Message Vol. 4: Version 1.2 Optimization Preview With A Special Note From Developers](https://wutheringwaves.kurogames.com/en/main/news/detail/1109) +- [[Chord Cleansing] Featured Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1099) +- [Event Notice | "Lollo Campaign" Limited-time Commission Event Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1080) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Blazing Brilliance](https://wutheringwaves.kurogames.com/en/main/news/detail/1066) +- [[Vermillion's Ploy] Featured Resonator Convene: Boosted Drop Rate for Changli](https://wutheringwaves.kurogames.com/en/main/news/detail/1065) +- [Profile Reveal | Eternal Blaze — Changli](https://wutheringwaves.kurogames.com/en/main/news/detail/1061) +- [Event Notice | [Tactical Simulacra] Featured Combat Event Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1055) +- [[Bountiful Crescendo] Featured Material Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1038) +- [New Content "Dreams Ablaze in Darkness" Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/1025) +- [[Gifts of Gratitude] Featured Login Event](https://wutheringwaves.kurogames.com/en/main/news/detail/1018) +- [Wuthering Waves Fan Creation Event “Thaw of Eons” is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/1006) +- [Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/1005) +- [Wuthering Waves Fan Creation Event Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/1004) +- [Version 1.1: Thaw of Eons Patch Notes](https://wutheringwaves.kurogames.com/en/main/news/detail/996) +- [Wuthering Waves Version Overview | Version 1.1 "Thaw of Eons" Coming June 28](https://wutheringwaves.kurogames.com/en/main/news/detail/975) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for Ages of Harvest](https://wutheringwaves.kurogames.com/en/main/news/detail/974) +- [[Thawborn Renewal] Featured Resonator Convene: Boosted Drop Rate for Jinhsi](https://wutheringwaves.kurogames.com/en/main/news/detail/973) +- [Profile Reveal | Thawborn Renewal — Jinhsi](https://wutheringwaves.kurogames.com/en/main/news/detail/965) +- [Post-Lament Anthropocene: Enemies — Jué](https://wutheringwaves.kurogames.com/en/main/news/detail/947) +- [Post-Lament Anthropocene: Enemies — Collection of Small & Medium Size Echoes](https://wutheringwaves.kurogames.com/en/main/news/detail/946) +- [Wuthering Waves Version 1.1 Update Maintenance Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/945) +- [Post-Lament Anthropocene: Enemies — Tactical Hologram: Crownless](https://wutheringwaves.kurogames.com/en/main/news/detail/932) +- [New Map Preview | Huanglong "Mt. Firmament" Opening Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/923) +- [Wutherium Geographic: Lands and Waters — Huanglong: Jinzhou — Hongzhen](https://wutheringwaves.kurogames.com/en/main/news/detail/922) +- [Wutherium Geographic: Lands and Waters — Huanglong: Jinzhou — Mt. Firmament](https://wutheringwaves.kurogames.com/en/main/news/detail/921) +- [Rover's Notes | Version 1.1 "Thaw of Eons"](https://wutheringwaves.kurogames.com/en/main/news/detail/913) +- ["Chord Cleansing" LT Echo Double Drop Event](https://wutheringwaves.kurogames.com/en/main/news/detail/900) +- [Wuthering Waves Version Preview | Version 1.1 "Thaw of Eons" Coming June 28](https://wutheringwaves.kurogames.com/en/main/news/detail/897) +- [Wuthering Waves Developer's Message Vol. 3: Version 1.1 Preview](https://wutheringwaves.kurogames.com/en/main/news/detail/893) +- [Post-Lament Anthropocene: Enemies — Dreamless](https://wutheringwaves.kurogames.com/en/main/news/detail/881) +- [Limited-time Special Event Notice | Wuthering Exploration Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/874) +- [Wuthering Waves Battle Rush](https://wutheringwaves.kurogames.com/en/main/news/detail/866) +- [Profile Reveal | Evil-conquering Puppet — Yinlin](https://wutheringwaves.kurogames.com/en/main/news/detail/862) +- [[Absolute Pulsation] Featured Weapon Convene: Boosted Drop Rate for [Stringmaster]](https://wutheringwaves.kurogames.com/en/main/news/detail/847) +- [[When Thunder Pours] Featured Resonator Convene: Boosted Drop Rate for [Yinlin]](https://wutheringwaves.kurogames.com/en/main/news/detail/846) +- [Event Notice | Alloy Smelt — Limited-time Battle Challenge Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/836) +- [Announcement on "Wuthering Waves" 1.0 Version and Future Adjustment Directions](https://wutheringwaves.kurogames.com/en/main/news/detail/827) +- [Event Notice | Overdash Club — Limited-time Track Challenge Coming Soon!](https://wutheringwaves.kurogames.com/en/main/news/detail/813) +- [Wuthering Waves Fan Creation Event is now live!](https://wutheringwaves.kurogames.com/en/main/news/detail/799) +- [Profile Reveal: General of the Midnight Rangers — Jiyan](https://wutheringwaves.kurogames.com/en/main/news/detail/795) +- [Regarding Gameplay Issues and Our Future Improvement Plans](https://wutheringwaves.kurogames.com/en/main/news/detail/790) +- [Wuthering Waves List of Known Issues (Updated on May 23rd)](https://wutheringwaves.kurogames.com/en/main/news/detail/777) +- [Convene Details](https://wutheringwaves.kurogames.com/en/main/news/detail/758) +- [Wuthering Waves Pre-download Instructions](https://wutheringwaves.kurogames.com/en/main/news/detail/748) +- [Wuthering Waves Fair Play Policy](https://wutheringwaves.kurogames.com/en/main/news/detail/742) +- [Wuthering Waves Player Community Information](https://wutheringwaves.kurogames.com/en/main/news/detail/738) +- [Wuthering Waves x Twitch: Support a streamer](https://wutheringwaves.kurogames.com/en/main/news/detail/728) +- [Profile Reveal: Frosty Gusto — Lingyang](https://wutheringwaves.kurogames.com/en/main/news/detail/720) +- [Profile Reveal: Consultant from the Black Shores — Encore](https://wutheringwaves.kurogames.com/en/main/news/detail/714) +- [Profile Reveal: Phantom Hunters — Calcharo](https://wutheringwaves.kurogames.com/en/main/news/detail/710) +- [Developer's Message: CLOSED BETA TEST II Review](https://wutheringwaves.kurogames.com/en/main/news/detail/690) +- [Profile Reveal: Nature Calling — Verina](https://wutheringwaves.kurogames.com/en/main/news/detail/676) +- [End of Closed Beta II Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/675) +- [Reminder: End of Closed Beta II](https://wutheringwaves.kurogames.com/en/main/news/detail/674) +- [Profile Reveal: Cleansing Reflections — Jianxin](https://wutheringwaves.kurogames.com/en/main/news/detail/666) +- [Wuthering Waves Fan Creation Event](https://wutheringwaves.kurogames.com/en/main/news/detail/656) +- [Closed Beta II Server Opening Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/655) +- [End of CBT2 Qualifications Distribution Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/650) +- [Wuthering Waves Closed Beta II Pre-download Instructions](https://wutheringwaves.kurogames.com/en/main/news/detail/649) +- [Wuthering Waves: Developer's Message](https://wutheringwaves.kurogames.com/en/main/news/detail/630) +- [Start of CBT2 Qualifications Distribution Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/629) +- [Profile Reveal: Healing You'tan — Baizhi](https://wutheringwaves.kurogames.com/en/main/news/detail/623) +- [Profile Reveal: Gallant Blaze — Chixia](https://wutheringwaves.kurogames.com/en/main/news/detail/622) +- [Profile Reveal: Breath of Winds — Yangyang](https://wutheringwaves.kurogames.com/en/main/news/detail/621) +- [Wuthering Waves Closed Beta II Opens on Feb. 19](https://wutheringwaves.kurogames.com/en/main/news/detail/616) +- [Wuthering Waves CBT2 Recruitment FAQ](https://wutheringwaves.kurogames.com/en/main/news/detail/599) +- [Wuthering Waves Closed Beta II Recruitment Open!](https://wutheringwaves.kurogames.com/en/main/news/detail/598) +- [Developer's Notes —Taoqi](https://wutheringwaves.kurogames.com/en/main/news/detail/589) +- [Profile Reveal: Blossom of Slashes — Taoqi](https://wutheringwaves.kurogames.com/en/main/news/detail/583) +- [Developer's Notes — Yuanwu](https://wutheringwaves.kurogames.com/en/main/news/detail/573) +- [Profile Reveal: Fist of Thunder — Yuanwu](https://wutheringwaves.kurogames.com/en/main/news/detail/566) +- [Developer's Notes — Danjin](https://wutheringwaves.kurogames.com/en/main/news/detail/549) +- [Profile Reveal: Scarlet Shade — Danjin](https://wutheringwaves.kurogames.com/en/main/news/detail/542) +- [Developer's Note — Mortefi](https://wutheringwaves.kurogames.com/en/main/news/detail/526) +- [Profile Reveal: Dragon's Breath — Mortefi](https://wutheringwaves.kurogames.com/en/main/news/detail/512) +- [Developer's Notes — Aalto](https://wutheringwaves.kurogames.com/en/main/news/detail/497) +- [Profile Reveal: Mistcloak Strike — Aalto](https://wutheringwaves.kurogames.com/en/main/news/detail/475) +- [Developer's Notes — Sanhua](https://wutheringwaves.kurogames.com/en/main/news/detail/461) +- [Profile Reveal: Snow Waltz — Sanhua](https://wutheringwaves.kurogames.com/en/main/news/detail/451) +- [Wuthering Waves Fan Art Contest – Winners Reveal](https://wutheringwaves.kurogames.com/en/main/news/detail/403) +- [Wuthering Waves CLOSED BETA TEST Review](https://wutheringwaves.kurogames.com/en/main/news/detail/399) +- [Closed Beta Test End Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/396) +- [Developer's Notes - Combat System](https://wutheringwaves.kurogames.com/en/main/news/detail/393) +- [Closed Beta Test Ending Notice](https://wutheringwaves.kurogames.com/en/main/news/detail/389) +- [Closed Beta Test Server Opening Announcement](https://wutheringwaves.kurogames.com/en/main/news/detail/383) +- [Fair Gaming Policy](https://wutheringwaves.kurogames.com/en/main/news/detail/191) +- [Profile: Counting Sheep - An'ke](https://wutheringwaves.kurogames.com/en/main/news/detail/186) +- [Profile: Scarlet Apparition - Danjin](https://wutheringwaves.kurogames.com/en/main/news/detail/182) +- [Developer's Notes - Crownless: The Primal's Desire](https://wutheringwaves.kurogames.com/en/main/news/detail/179) +- [Profile: Qingloong Prevailing the Long Night - Jiyan](https://wutheringwaves.kurogames.com/en/main/news/detail/176) +- [Closed Beta Test FAQ](https://wutheringwaves.kurogames.com/en/main/news/detail/170) +- [Closed Beta Test Recruitment Begins!](https://wutheringwaves.kurogames.com/en/main/news/detail/169) +- [Developer's Notes - Resonators of First Encounter](https://wutheringwaves.kurogames.com/en/main/news/detail/162) + +## Articles Directory + +The articles are saved in the `articles` directory. +You can view them [here](articles). diff --git a/scrape.py b/scrape.py index df8d8ba..3c58d82 100644 --- a/scrape.py +++ b/scrape.py @@ -5,8 +5,6 @@ It retrieves the article menu and individual articles, prettifies the JSON outpu import asyncio import json - -# Set up logging import logging import os import shutil @@ -99,7 +97,27 @@ def set_file_timestamp(filepath: Path, timestamp_str: str) -> bool: return True -def commit_file_with_timestamp(filepath: Path) -> bool: +def get_file_timestamp(timestamp_str: str) -> float: + """Convert ISO timestamp string to Unix timestamp. + + Args: + timestamp_str (str): The ISO timestamp string. + + Returns: + float: The Unix timestamp, or 0 if conversion failed. + + """ + try: + # Parse the timestamp string + dt: datetime = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC) + # Convert to Unix timestamp + return dt.timestamp() + except ValueError: + logger.info("Error converting timestamp %s", timestamp_str) + return 0.0 + + +def commit_file_with_timestamp(filepath: Path) -> bool: # noqa: PLR0911 """Commit a file to Git with its modification time as the commit time. Args: @@ -122,7 +140,7 @@ def commit_file_with_timestamp(filepath: Path) -> bool: try: # Get the full path to the Git executable - git_executable = shutil.which("git") + git_executable: str | None = shutil.which("git") if not git_executable: logger.error("Git executable not found.") return False @@ -162,7 +180,107 @@ def commit_file_with_timestamp(filepath: Path) -> bool: return True -async def main() -> Literal[1, 0]: # noqa: C901, PLR0912, PLR0914, PLR0915 +def add_articles_to_readme(articles: dict[Any, Any] | None = None) -> None: + """Add the list of articles to the README.md file.""" + if articles is None: + logger.warning("No articles to add to README.md") + return + + readme_file: Path = Path("README.md") + if not readme_file.is_file(): + logger.error("README.md file not found.") + return + + with readme_file.open("r+", encoding="utf-8") as f: + # Read existing content + lines: list[str] = f.readlines() + + # Find "## Articles" section or add it + articles_section_index = -1 + for i, line in enumerate(lines): + if line.strip() == "## Articles": + articles_section_index: int = i + break + + # Create new content + new_lines: list[str] = [] + if articles_section_index >= 0: + new_lines = lines[: articles_section_index + 1] # Keep everything up to "## Articles" + else: + new_lines = lines + if new_lines and not new_lines[-1].endswith("\n"): + new_lines.append("\n") + new_lines.append("## Articles\n") + + # Add articles + new_lines.append("\n") # Add a blank line after the heading + for article in sorted(articles, key=lambda x: x.get("createTime", ""), reverse=True): + article_id: str = str(article.get("articleId", "")) + article_title: str = article.get("articleTitle", "No Title") + article_url: str = f"https://wutheringwaves.kurogames.com/en/main/news/detail/{article_id}" + new_lines.append(f"- [{article_title}]({article_url})\n") + + # Add articles directory section + new_lines.append("\n## Articles Directory\n") + new_lines.append("The articles are saved in the `articles` directory.\n") + new_lines.append("You can view them [here](articles).\n") + + # Write the updated content + f.seek(0) + f.truncate() + f.writelines(new_lines) + + logger.info("Articles added to README.md") + + +def batch_process_timestamps(menu_data: dict[Any, Any], output_dir: Path) -> None: + """Process all timestamps in batch for better performance. + + Args: + menu_data (list[dict[str, Any]]): The article menu data containing timestamps. + output_dir (Path): Directory containing the article files. + + """ + # Extract article IDs and timestamps + timestamp_map: dict[str, str] = {} + for item in menu_data: + article_id = str(item.get("articleId", "")) + create_time = item.get("createTime") + if article_id and create_time: + timestamp_map[article_id] = create_time + + logger.info("Collected %s timestamps from menu data", len(timestamp_map)) + + # Check which files need timestamp updates + files_to_update: list[tuple[Path, str]] = [] + for article_id, create_time in timestamp_map.items(): + file_path: Path = output_dir / f"{article_id}.json" + if not file_path.exists(): + continue + + expected_timestamp: float = get_file_timestamp(create_time) + if expected_timestamp == 0.0: + continue + + actual_timestamp: float = file_path.stat().st_mtime + + # Only update if timestamps don't match (with a small tolerance) + if abs(actual_timestamp - expected_timestamp) > 1.0: + files_to_update.append((file_path, create_time)) + + logger.info("Found %s files that need timestamp updates", len(files_to_update)) + + # Update timestamps and commit files + for file_path, create_time in files_to_update: + logger.info("Setting %s timestamp to %s", file_path, create_time) + if set_file_timestamp(file_path, create_time): + if not commit_file_with_timestamp(file_path): + logger.error("Failed to commit file %s to Git", file_path) + else: + logger.error("Failed to update timestamp for %s", file_path) + + +async def main() -> Literal[1, 0]: """Fetch and save articles from the Wuthering Waves website. Returns: @@ -205,67 +323,47 @@ async def main() -> Literal[1, 0]: # noqa: C901, PLR0912, PLR0914, PLR0915 existing_files: list[str] = [file.stem for file in output_dir.glob("*.json") if file.stem != "ArticleMenu"] # Filter out already downloaded articles - article_ids = [article_id for article_id in article_ids if article_id not in existing_files] + new_article_ids: list[str] = [article_id for article_id in article_ids if article_id not in existing_files] - # Download each article - download_tasks: list[Coroutine[Any, Any, dict[Any, Any] | None]] = [] - for article_id in article_ids: - article_url: str = f"{article_base_url}{article_id}.json?t={current_time}" - output_file: Path = output_dir / f"{article_id}.json" + if new_article_ids: + logger.info("Found %s new articles to download", len(new_article_ids)) - logger.info("Downloading article %s from %s", article_id, article_url) - download_tasks.append(fetch_json(article_url, client)) + # Download each new article + download_tasks: list[Coroutine[Any, Any, dict[Any, Any] | None]] = [] + for article_id in new_article_ids: + article_url: str = f"{article_base_url}{article_id}.json?t={current_time}" + output_file: Path = output_dir / f"{article_id}.json" - # Wait for all downloads to complete - results: list[dict[Any, Any] | BaseException | None] = await asyncio.gather(*download_tasks, return_exceptions=True) + logger.info("Downloading article %s from %s", article_id, article_url) + download_tasks.append(fetch_json(article_url, client)) - # Process the downloaded articles - for i, result in enumerate(results): - article_id: str = article_ids[i] - output_file = output_dir / f"{article_id}.json" + # Wait for all downloads to complete + results: list[dict[Any, Any] | BaseException | None] = await asyncio.gather(*download_tasks, return_exceptions=True) - if isinstance(result, Exception): - logger.error("Error downloading article %s: %s", article_id, result) - continue + # Process the downloaded articles + for i, result in enumerate(results): + article_id: str = new_article_ids[i] + output_file = output_dir / f"{article_id}.json" - if not result: - logger.warning("Downloaded article %s is empty or invalid", article_id) - continue + if isinstance(result, Exception): + logger.error("Error downloading article %s: %s", article_id, result) + continue - # Save the article JSON - if isinstance(result, dict) and await save_prettified_json(result, output_file): - logger.info("Successfully downloaded and prettified %s", output_file) + if not result: + logger.warning("Downloaded article %s is empty or invalid", article_id) + continue - json_files: list[Path] = list(output_dir.glob("*.json")) + # Save the article JSON + if isinstance(result, dict) and await save_prettified_json(result, output_file): + logger.info("Successfully downloaded and prettified %s", output_file) + else: + logger.info("No new articles to download") - # Reverse the JSON files so the youngest articles are at the top of the Git history - json_files.sort(key=lambda x: x.stat().st_mtime, reverse=True) + # Process timestamps in batch + batch_process_timestamps(menu_data, output_dir) - # Update file timestamps based on createTime - for file in json_files: - article_id = file.stem - if article_id == "ArticleMenu": - continue - - # Find the corresponding article in the menu - create_time = None - for item in menu_data: - if str(item.get("articleId", "")) == article_id and "createTime" in item: - create_time = item["createTime"] - break - - if not create_time: - logger.info("Warning: no createTime for %s in menu data - skipping", article_id) - continue - - logger.info("Setting %s timestamp to %s", file, create_time) - if not set_file_timestamp(file, create_time): - logger.error(" failed to update timestamp") - continue - - # Commit the file to Git with the correct timestamp - if not commit_file_with_timestamp(file): - logger.error(" failed to commit file %s to Git", file) + # Update the README + add_articles_to_readme(menu_data) logger.info("Script finished. Articles are in the '%s' directory.", output_dir) return 0