better SQL handling

kwaylon/data.py

@@ -42,8 +42,8 @@ class MsgData:
                 else:
                     LOGGER.info(f'read {self.reactions.shape[0]:,} reactions')
                     self.reactions['datetime'] = pd.to_datetime(self.reactions['datetime'])
-                    LOGGER.info(f"'datetime' dtype: {self.reactions['datetime'].dtype}")
+                    # LOGGER.info(f"'datetime' dtype: {self.reactions['datetime'].dtype}")
-                    LOGGER.info(f"{self.reactions['datetime'].values[:3]}...")
+                    # LOGGER.info(f"{self.reactions['datetime'].values[:3]}...")
 
                 # try:
                 #     self.reactions['datetime'] = pd.to_datetime(self.reactions['datetime']).dt.tz_convert(local_tz)
@@ -82,26 +82,34 @@ class MsgData:
         else:
             return self.reactions.loc[matching].sort_values('count', ascending=False).reset_index(drop=True)
 
-    async def fetch_message(self, client: Client, row: pd.Series):
+    async def get_emoji_info(self, emoji: str):
-        guild = await client.fetch_guild(row['guild_id'])
+        async with self.lock:
-        channel = await guild.fetch_channel(row['channel_id'])
+            try:
-        return await channel.fetch_message(row['msg_id'])
+                with self.sql_context as con:
+                    res = pd.read_sql(f"SELECT * FROM reactions WHERE emoji LIKE '{emoji}'", con=con, index_col=None)
+                    res['datetime'] = pd.to_datetime(res['datetime'])
+            except Exception as e:
+                LOGGER.exception(e)
+                res = None
+            else:
+                LOGGER.info(f'Read {res.shape[0]} reactions')
+            finally:
+                con.close()
+                return res
 
     async def update_reaction(self, msg: Message):
-        # Drop all the reactions for this message id, if there are any
+        async with self.lock:
-        try:
-            async with self.lock:
-                self.reactions = self.reactions.loc[self.reactions['msg_id'] != msg.id]
-        except KeyError as e:
-            pass
-
-        # If there are reactions on the message after the change
-        if len(msg.reactions) > 0:
-            new = pd.DataFrame([reaction_dict(r) for r in msg.reactions])
-            async with self.lock:
-                self.reactions = self.reactions.append(new)
             try:
-                await self.write_sql()
+                with self.sql_context as con:
+                    con.execute(f'DELETE FROM reactions WHERE msg_id = {msg.id}')
+                    data = [tuple(reaction_dict(reaction).values()) for reaction in msg.reactions]
+                    if len(data) > 0:
+                        query = f'INSERT INTO reactions VALUES({",".join("?" for _ in range(8))})'
+                        LOGGER.info(f'SQL: {query}')
+                        con.executemany(query, data)
             except:
-                LOGGER.info(self.reactions.columns)
+                raise
-                LOGGER.info(self.reactions.dtypes)
+            else:
+                LOGGER.info(f'Success')
+            finally:
+                con.close()

kwaylon/kwaylon.py

@@ -4,6 +4,7 @@ from datetime import timedelta, datetime
 from pathlib import Path
 
 import nextcord as discord
+import pandas as pd
 from nextcord import Client, Message, TextChannel
 from nextcord import RawReactionActionEvent
 
@@ -39,9 +40,6 @@ class Kwaylon(Client):
 
         self.data = MsgData(self.db_path)
 
-        await self.data.scan_messages(client=self, limit=100)
-        await self.data.write_sql()
-
         await self.data.load_sql()
         if not hasattr(self.data, 'reactions'):
             await self.data.scan_messages(client=self, limit=self.limit, days=self.days)
@@ -63,32 +61,32 @@ class Kwaylon(Client):
                 emoji = get_emoji_name(m.group('emoji'))
                 LOGGER.info(emoji)
 
-                if (most := self.data.most(emoji=emoji)) is not None:
+                if (df := await self.data.get_emoji_info(emoji)) is not None and df.shape[0] > 0:
-                    # LOGGER.info(f'\n{str(most)}')
+                    kwargs = {'days': 14}
-                    kwargs = {'emoji_name': emoji}
                     if (day_match := re.search('(?P<days>\d+) days', message.content)):
                         days = int(day_match.group('days'))
                         kwargs['days'] = days
 
                     if 'leaderboard' in message.content:
-                        await message.reply(await self.leaderboard(**kwargs))
+                        LOGGER.info(f'Building leaderboard')
+                        res = f'{m.group("emoji")} totals, past {kwargs["days"]} days\n'
+                        res += await self.leaderboard(df, **kwargs)
+                        await message.reply(res)
+                        LOGGER.info(f'Done')
                     else:
-                        if most.shape[0] > 0:
+                        LOGGER.info(f'Most {m.group("emoji")}')
-                            most = most.iloc[0]
+                        most = df.sort_values('count').iloc[-1]
-                            msg = await self.data.fetch_message(self, most)
+                        msg = await self.fetch_message(most)
-                            await message.reply(f'{msg.jump_url}')
+                        await message.reply(f'{msg.jump_url}')
-                            LOGGER.info(f'{msg.clean_content}')
+                else:
-                            LOGGER.info(f' - {msg.author}')
+                    await message.reply(f"NObody...gah, leave me alone!")
-                            LOGGER.info(f'{most["count"]}x {emoji}')
 
             for joke in self.jokes:
                 if (m := joke.scan(message)) is not None:
                     LOGGER.info(f'{joke.__class__.__name__} detected: {message.content}, {m.group()}')
                     await joke.respond(message, self, m)
 
-    async def leaderboard(self, emoji_name: str, days: int = 14) -> str:
+    async def leaderboard(self, df: pd.DataFrame, days: int = 14) -> str:
-        df = self.data.most(emoji=emoji_name)
-
         start = (datetime.today() - timedelta(days=days)).astimezone()
         valid_dates = df['datetime'] > start
         df = df.loc[valid_dates]
@@ -98,9 +96,11 @@ class Kwaylon(Client):
         counts.index = [(await self.fetch_user(idx)).display_name for idx in counts.index]
 
         width = max([len(str(s)) for s in counts.index])
-        res = f'{emoji_name} totals, past {days} days\n'
+
-        res += '\n'.join(f"`{str(name).ljust(width + 1)}with {cnt:<2.0f} total`"
+        res = '\n'.join(
-                         for name, cnt in counts.iteritems())
+            f"`{str(name).ljust(width + 1)}with {cnt:<2.0f} total`"
+            for name, cnt in counts.iteritems()
+        )
         return res
 
     async def handle_raw_reaction(self, payload: RawReactionActionEvent):
@@ -109,16 +109,24 @@ class Kwaylon(Client):
         channel = await guild.fetch_channel(payload.channel_id)
         message = await channel.fetch_message(payload.message_id)
 
-        if payload.event_type == 'REACTION_REMOVE':
+        if payload.event_type == 'REACTION_ADD':
-            LOGGER.info(f'{payload.emoji} removed from\n{message.author}: {message.content}')
-        elif payload.event_type == 'REACTION_ADD':
             LOGGER.info(
                 f'{payload.member.display_name} added {payload.emoji} to\n' + \
                 f'{message.author.display_name}: {message.content}')
+            # await self.data.add_reaction()
+
+        elif payload.event_type == 'REACTION_REMOVE':
+            LOGGER.info(f'{payload.emoji} removed from\n{message.author}: {message.content}')
+            # await self.data.remove_reaction(event=payload)
 
         if hasattr(self, 'data'):
             await self.data.update_reaction(msg=message)
 
+    async def fetch_message(self, row: pd.Series):
+        guild = await self.fetch_guild(row['guild_id'])
+        channel = await guild.fetch_channel(row['channel_id'])
+        return await channel.fetch_message(row['msg_id'])
+
 
 def get_emoji_name(string: str) -> str:
     if (m := re.search('<:(?P<name>\w+):(?P<id>\d+)>', string)):

kwaylon/msg.py

@@ -9,7 +9,7 @@ from nextcord.utils import AsyncIterator
 LOGGER = logging.getLogger(__name__)
 
 
-async def message_gen(client: Client, limit=20, days: int = 90, **kwargs) -> AsyncIterator[Message]:
+async def message_gen(client: Client, limit: int = None, days: int = 90, **kwargs) -> AsyncIterator[Message]:
     if 'after' not in kwargs:
         kwargs['after'] = (datetime.today() - timedelta(days=days))
     elif isinstance((after := kwargs.get('after', None)), datetime):

main.py

@@ -20,6 +20,11 @@ if __name__ == '__main__':
     @client.event
     async def on_ready():
         await client.handle_ready()
+        # await client.data.scan_messages(
+        #     client=client,
+        #     # limit=100,
+        #     days=60,
+        # )
 
 
     @client.event
@@ -35,6 +40,7 @@ if __name__ == '__main__':
     @client.event
     async def on_raw_reaction_remove(payload: RawReactionActionEvent):
         await client.handle_raw_reaction(payload)
+        # await client.data.remove_reaction(payload)
 
     load_dotenv()
     client.run(os.getenv('DISCORD_TOKEN'))