Merge pull request '⬆️ Update dependency yarl to v1.17.1' (#66) from renovate/yarl-1.x into main

Reviewed-on: #66

.gitea/workflows/docker.yml

@@ -0,0 +1,48 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - "*"
+    tags:
+      - "*"
+  pull_request:
+    branches:
+      - "main"
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    container:
+      image: catthehacker/ubuntu:act-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            git.sebclem.fr/${{ gitea.repository }}            
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}            
+
+      - name: Login to registry
+        if: github.ref_type == 'tag'
+        uses: docker/login-action@v3
+        with:
+          registry: git.sebclem.fr
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: ${{ github.ref_type == 'tag' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -1,5 +1,5 @@
 # For more information, please refer to https://aka.ms/vscode-docker-python
-FROM python:3.10-slim
+FROM python:3.13-alpine
 
 # Keeps Python from generating .pyc files in the container
 ENV PYTHONDONTWRITEBYTECODE=1
@@ -7,6 +7,8 @@ ENV PYTHONDONTWRITEBYTECODE=1
 # Turns off buffering for easier container logging
 ENV PYTHONUNBUFFERED=1
 
+RUN apk add build-base
+
 # Install pip requirements
 COPY requirements.txt .
 RUN python -m pip install -r requirements.txt

renovate.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ],
+  "commitMessagePrefix": ":arrow_up:"
+
+}

requirements.txt

@@ -1,2 +1,21 @@
-pixivpy3
-discord.py
+pixivpy3==3.7.5
+discord.py==2.4.0
+
+# Freeze
+aiohttp==3.10.10
+aiosignal==1.3.1
+async-timeout==5.0.1
+attrs==24.2.0
+certifi==2024.8.30
+charset-normalizer==3.4.0
+cloudscraper==1.2.71
+frozenlist==1.5.0
+idna==3.10
+isort==5.13.2
+multidict==6.1.0
+pyparsing==3.2.0
+requests==2.32.3
+requests-toolbelt==1.0.0
+typing_extensions==4.12.2
+urllib3==2.2.3
+yarl==1.17.1

src/discord_tools/message.py

@@ -1,33 +1,50 @@
-import math
 import os
+from typing import List
 
 import discord
+from idna import intranges_contain
 
 TIPS_MESSAGE = ":bulb: TIPS: You can disable Discord's default preview for link by wrapping it around `<` `>`"
+IMAGE_NBR_LIMIT = 10
+DEFAULT_UPLOAD_SIZE_LIMIT = 209715200
 
 
-async def send_message(message: discord.Message, files, tmp_dir: str, muted: bool):
+async def send_message(
+    message: discord.Message,
+    files,
+    tmp_dir: str,
+    muted: bool,
+    title: str,
+    art_author: str,
+):
     to_send_files = []
     for file in files:
         joined = os.path.join(tmp_dir, file)
         to_send_files.append(discord.File(joined, file))
-    if len(to_send_files) > 10:
-        first = True
-        while len(to_send_files) > 10:
-            splited_file = to_send_files[:10]
-            to_send_files = to_send_files[10:]
-            if first:
-                await message.reply(
-                    files=splited_file, content=(TIPS_MESSAGE if not muted else None)
-                )
-                first = False
-            else:
-                await message.channel.send(files=splited_file)
-        await message.channel.send(files=to_send_files)
-    else:
-        await message.reply(
-            files=to_send_files, content=(TIPS_MESSAGE if not muted else None)
-        )
+
+    upload_size_limit = (
+        message.guild.filesize_limit
+        if message.guild is not None
+        else DEFAULT_UPLOAD_SIZE_LIMIT
+    )
+    first = True
+    while len(to_send_files) > 0:
+        nbr_file_to_send = get_max_files_number(to_send_files, upload_size_limit)
+        splited_file = to_send_files[:nbr_file_to_send]
+        to_send_files = to_send_files[nbr_file_to_send:]
+        if first and nbr_file_to_send <= 0:
+            return
+        elif nbr_file_to_send <= 0:
+            # Next file is to big to be sent, skip it
+            to_send_files = to_send_files[1:]
+        elif first:
+            await message.reply(
+                files=splited_file,
+                content=f"**{title}** by `{art_author}` {chr(10) + TIPS_MESSAGE if not muted else ''}",
+            )
+            first = False
+        else:
+            await message.channel.send(files=splited_file)  # type: ignore
 
 
 async def send_command_reply(
@@ -36,6 +53,7 @@ async def send_command_reply(
     tmp_dir: str,
     muted: bool,
     title: str,
+    art_author: str,
     url: str,
     author: discord.User | discord.Member,
 ):
@@ -43,23 +61,48 @@ async def send_command_reply(
     for file in files:
         joined = os.path.join(tmp_dir, file)
         to_send_files.append(discord.File(joined, file))
-    if len(to_send_files) > 10:
-        if interaction.channel is not None:
-            first = True
-            while len(to_send_files) > 10:
-                splited_file = to_send_files[:10]
-                to_send_files = to_send_files[10:]
-                if first:
-                    await interaction.followup.send(
-                        files=splited_file,
-                        content=f"{title} - Submited by {author.mention} - <{url}> {chr(10) + TIPS_MESSAGE if not muted else ''}",
-                    )
-                    first = False
-                else:
-                    await interaction.channel.send(files=splited_file)  # type: ignore
-            await interaction.channel.send(files=to_send_files)  # type: ignore
-    else:
-        await interaction.followup.send(
-            files=to_send_files,
-            content=f"{title} - Submited by {author.mention} - <{url}> {chr(10) + TIPS_MESSAGE if not muted else ''}",
-        )
+
+    upload_size_limit = (
+        interaction.guild.filesize_limit
+        if interaction.guild is not None
+        else DEFAULT_UPLOAD_SIZE_LIMIT
+    )
+
+    first = True
+    while len(to_send_files) > 0:
+        nbr_file_to_send = get_max_files_number(to_send_files, upload_size_limit)
+        splited_file = to_send_files[:nbr_file_to_send]
+        to_send_files = to_send_files[nbr_file_to_send:]
+        if first and nbr_file_to_send <= 0:
+            await interaction.followup.send(
+                content=f"This image exceeds upload file size limit", ephemeral=True
+            )
+        elif nbr_file_to_send <= 0:
+            # Next file is to big to be sent, skip it
+            to_send_files = to_send_files[1:]
+        elif first:
+            await interaction.followup.send(
+                files=splited_file,
+                content=f"**{title}** by `{art_author}` - <{url}> - Submited by {author.mention} {chr(10) + TIPS_MESSAGE if not muted else ''}",
+            )
+            first = False
+        else:
+            await interaction.channel.send(files=splited_file)  # type: ignore
+
+
+def get_max_files_number(files: List[discord.File], size_limit: int):
+    size = 0
+    nbr = 0
+    for file in files:
+        size = size + os.path.getsize(file.fp.name)  # type: ignore
+        nbr = nbr + 1
+        if nbr >= IMAGE_NBR_LIMIT and size < size_limit:
+            return IMAGE_NBR_LIMIT
+        elif size > size_limit:
+            # We exceeds size limit, remove one image
+            if nbr >= IMAGE_NBR_LIMIT:
+                return IMAGE_NBR_LIMIT - 1
+            else:
+                return nbr - 1
+    # We didn't reach any limit
+    return nbr

src/main.py

@@ -1,3 +1,5 @@
+import asyncio
+import logging
 import os
 from shlex import join
 from tempfile import TemporaryDirectory
@@ -8,13 +10,18 @@ import discord_tools.message
 import discord_tools.pixiv_link_extractor
 import pixiv.pixiv_api as pixiv_api
 
+logging.basicConfig(
+    format="[%(asctime)s] [%(levelname)8s] %(name)s: %(message)s", level=logging.INFO
+)
+
+
 REFRESH_TOKEN = os.getenv("PIXIV_REFRESH_TOKEN")
 if REFRESH_TOKEN is None:
-    print("Pixiv refresh token is missing, please set PIXIV_REFRESH_TOKEN")
+    logging.fatal("Pixiv refresh token is missing, please set PIXIV_REFRESH_TOKEN")
     exit(1)
 DISCORD_TOKEN = os.getenv("DISCORD_TOKEN")
 if DISCORD_TOKEN is None:
-    print("Discord token is missing, please set DISCORD_TOKEN")
+    logging.fatal("Discord token is missing, please set DISCORD_TOKEN")
     exit(1)
 
 
@@ -36,7 +43,7 @@ client = MyClient(intents=intents)
 
 @client.event
 async def on_ready():
-    print(f"We have logged in as {client.user}")
+    logging.info(f"We have logged in as {client.user}")
 
 
 @client.tree.command()
@@ -49,15 +56,20 @@ async def pixivprev(interaction: discord.Interaction, url: str):
     if result and match is not None:
         await interaction.response.defer()
         with TemporaryDirectory() as tmp_dir:
-            title = pixiv_api.dowload_pixiv_images(
-                int(match.group("id")), tmp_dir, REFRESH_TOKEN
+            loop = asyncio.get_running_loop()
+            data = await loop.run_in_executor(
+                None,
+                pixiv_api.download_pixiv_images,
+                int(match.group("id")),
+                tmp_dir,
+                REFRESH_TOKEN,
             )
-            if title is None:
+            if data is None:
                 await interaction.response.send_message(
                     "Fail to extract pixiv image", ephemeral=True
                 )
                 return
-
+            (title, author) = data
             files = os.listdir(tmp_dir)
             files.sort()
             await discord_tools.message.send_command_reply(
@@ -66,6 +78,7 @@ async def pixivprev(interaction: discord.Interaction, url: str):
                 tmp_dir,
                 True,
                 title=title,
+                art_author=author,
                 url=url,
                 author=interaction.user,
             )
@@ -79,24 +92,31 @@ async def pixiv_preview(interaction: discord.Interaction, message: discord.Messa
     if result and match is not None:
         await interaction.response.defer()
         with TemporaryDirectory() as tmp_dir:
-            title = pixiv_api.dowload_pixiv_images(
-                int(match.group("id")), tmp_dir, REFRESH_TOKEN
+            loop = asyncio.get_running_loop()
+            data = await loop.run_in_executor(
+                None,
+                pixiv_api.download_pixiv_images,
+                int(match.group("id")),
+                tmp_dir,
+                REFRESH_TOKEN,
             )
-            if title is None:
+            if data is None:
                 await interaction.response.send_message(
                     "Fail to extract pixiv image", ephemeral=True
                 )
                 return
             files = os.listdir(tmp_dir)
             files.sort()
+            (title, author) = data
             await discord_tools.message.send_command_reply(
                 interaction=interaction,
                 files=files,
                 tmp_dir=tmp_dir,
                 muted=muted,
                 title=title,
+                art_author=author,
                 url=match.group("url"),
-                author=interaction.user,
+                author=message.author,
             )
     else:
         await interaction.response.send_message(
@@ -106,7 +126,7 @@ async def pixiv_preview(interaction: discord.Interaction, message: discord.Messa
 
 @client.event
 async def on_guild_join(guild: discord.Guild):
-    print(f"Joined new guild: {guild.name}")
+    logging.info(f"Joined new guild: {guild.name}")
 
 
 @client.event
@@ -117,17 +137,25 @@ async def on_message(message: discord.Message):
     if result and match is not None:
         with TemporaryDirectory() as tmp_dir:
             async with message.channel.typing():
-                title = pixiv_api.dowload_pixiv_images(
-                    int(match.group("id")), tmp_dir, REFRESH_TOKEN
+                loop = asyncio.get_running_loop()
+                data = await loop.run_in_executor(
+                    None,
+                    pixiv_api.download_pixiv_images,
+                    int(match.group("id")),
+                    tmp_dir,
+                    REFRESH_TOKEN,
                 )
-                if title is None:
+                if data is None:
                     return
+                (title, author) = data
                 files = os.listdir(tmp_dir)
                 files.sort()
                 # await send_message.send_message_with_embed(
                 #     message, files, title, tmp_dir, match.group("url")
                 # )
-                await discord_tools.message.send_message(message, files, tmp_dir, muted)
+                await discord_tools.message.send_message(
+                    message, files, tmp_dir, muted, title, author
+                )
 
 
 client.run(DISCORD_TOKEN)

src/pixiv/pixiv_api.py

@@ -1,7 +1,19 @@
+import logging
+import os
+import threading
+from queue import Queue
+from threading import Thread
+from typing import List, Tuple, Union
+
 import pixivpy3
 
+cpu_count = os.cpu_count()
+max_thread_num = cpu_count * 2 if cpu_count is not None else 4
+if max_thread_num > 10:
+    max_thread_num = 10
 
-def dowload_pixiv_images(
+
+def download_pixiv_images(
     illust_id: int,
     dest_folder: str,
     refresh_token: str,
@@ -12,17 +24,57 @@ def dowload_pixiv_images(
     json_result = api.illust_detail(illust_id)
     illust = json_result.illust
     if not illust["visible"]:
+        logging.info("Can't be viewed, return None")
         return None
     pages = illust["meta_pages"]
     if len(pages) > 0:
-        print(f"Dowloading mutiple images to {dest_folder} ({len(pages)})")
+        logging.info(f"Dowloading mutiple images to {dest_folder} ({len(pages)})")
+        if max_thread_num > len(pages):
+            num_threads = len(pages)
+        else:
+            num_threads = max_thread_num
+
+        logging.info(f"Starting {num_threads} thread(s)")
+        workers: List[Thread] = []
+        q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]] = Queue()
+        for i in range(num_threads):
+            worker = Thread(target=download_thread, args=(q,))
+            worker.setDaemon(True)
+            worker.start()
+            workers.append(worker)
+
         for page in pages:
             url = page["image_urls"]["original"]
-            api.download(url=url, path=dest_folder)
+            q.put((url, dest_folder, api))
+
+        q.join()
+
+        # Send end signal to all threads
+        for i in range(num_threads):
+            q.put(None)
+        # Wait for them to finish
+        for worker in workers:
+            worker.join()
+
     else:
-        print(f"Dowloading single image to {dest_folder}")
+        logging.info(f"Dowloading single image to {dest_folder}")
         api.download(
             url=illust["meta_single_page"]["original_image_url"], path=dest_folder
         )
-    print("Down4load finished !")
-    return illust["title"]
+    logging.info("Download finished !")
+    return (illust["title"], f"{illust['user']['name']} ({illust['user']['account']})")
+
+
+def download_thread(q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]]):
+    thread_name = threading.currentThread().getName()
+    while True:
+        data = q.get()
+        if data is None:
+            logging.debug(f"[{thread_name}] - Bye bye")
+            q.task_done()
+            break
+        (url, dest_folder, api) = data
+        logging.debug(f"[{thread_name}] - Downloading {url}...")
+        api.download(url=url, path=dest_folder)
+        print(f"[{thread_name}] - ... Done")
+        q.task_done()