From a975d09e42118a458ceb8caf96d87f2ee16b2b54 Mon Sep 17 00:00:00 2001
From: dnns01 <git@dnns01.de>
Date: Tue, 4 Jul 2023 18:22:48 +0200
Subject: [PATCH] Scrape fernstudis.de for new issues of SprachRohr and
announce them
---
extensions/sprachrohr.py | 42 ++++++++++++++++++++++++++++++++++++++++
models.py | 7 ++++++-
requirements.txt | 21 ++++++++++----------
3 files changed, 58 insertions(+), 12 deletions(-)
create mode 100644 extensions/sprachrohr.py
diff --git a/extensions/sprachrohr.py b/extensions/sprachrohr.py
new file mode 100644
index 0000000..0419e58
--- /dev/null
+++ b/extensions/sprachrohr.py
@@ -0,0 +1,42 @@
+from aiohttp import ClientSession
+from bs4 import BeautifulSoup
+from discord.ext import commands, tasks
+import models
+
+
+class Sprachrohr(commands.Cog):
+ def __init__(self, bot):
+ self.bot = bot
+ self.config = bot.config["extensions"][__name__.split(".")[-1]]
+ self.url = self.config.get("url")
+
+ self.update_loop.start()
+
+ @tasks.loop(hours=3)
+ async def update_loop(self):
+ async with ClientSession() as session:
+ async with session.get(self.url) as r:
+ if r.status == 200:
+ content = await r.read()
+ soup = BeautifulSoup(content, "html.parser")
+
+ for issue in soup.find("ul", attrs={"class": "wp-block-latest-posts__list"}).find_all("li"):
+ link = issue.find("a")
+ url = link["href"]
+ year = link.text[-4:]
+ issue_in_year = link.text[len("SprachRohr "):-5]
+
+ (sprach_rohr, created) = models.SprachRohr.get_or_create(url=url, year=year,
+ issue=issue_in_year)
+ if created:
+ await self.announce_new_issue(sprach_rohr)
+
+ async def announce_new_issue(self, sprach_rohr: models.SprachRohr):
+ if channel := await self.bot.fetch_channel(self.config.get("channel_id")):
+ msg = await channel.send(f"{self.config.get('text')}\n{sprach_rohr.url}")
+ if channel.is_news():
+ await msg.publish()
+
+
+async def setup(bot: commands.Bot) -> None:
+ await bot.add_cog(Sprachrohr(bot))
diff --git a/models.py b/models.py
index a10b81d..732b816 100644
--- a/models.py
+++ b/models.py
@@ -205,6 +205,11 @@ class CommandText(BaseModel):
text = CharField()
command = ForeignKeyField(Command, backref="texts")
+class SprachRohr(BaseModel):
+ url = CharField()
+ year = IntegerField()
+ issue = CharField()
+
db.create_tables([Poll, PollChoice, PollChoiceChosen, Appointment, Attendee, Topic, Link, Timer, TimerAttendee, Command,
- CommandText], safe=True)
+ CommandText, SprachRohr], safe=True)
diff --git a/requirements.txt b/requirements.txt
index 273dd11..531c9fc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,18 +1,17 @@
-aiohttp==3.8.3
+aiohttp==3.8.4
aiosignal==1.3.1
async-timeout==4.0.2
-attrs==22.2.0
-certifi==2022.12.7
+attrs==23.1.0
+beautifulsoup4==4.12.2
cffi==1.15.1
-charset-normalizer==2.1.1
-discord.py==2.1.0
-emoji==2.0.0
+charset-normalizer==3.1.0
+discord.py==2.3.1
+emoji==2.6.0
frozenlist==1.3.3
idna==3.4
-multidict==6.0.3
-peewee==3.15.2
+multidict==6.0.4
+peewee==3.16.2
pycparser==2.21
PyNaCl==1.5.0
-requests==2.28.1
-urllib3==1.26.13
-yarl==1.8.2
+soupsieve==2.4.1
+yarl==1.9.2
--
GitLab