Merge pull request '⬆️ Update dependency urllib3 to v2.2.2' (#35) from renovate/urllib3-2.x into main

Reviewed-on: #35

.gitea/workflows/docker.yml

@@ -0,0 +1,48 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - "*"
+    tags:
+      - "*"
+  pull_request:
+    branches:
+      - "main"
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    container:
+      image: catthehacker/ubuntu:act-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            git.sebclem.fr/${{ gitea.repository }}            
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}            
+
+      - name: Login to registry
+        if: github.ref_type == 'tag'
+        uses: docker/login-action@v3
+        with:
+          registry: git.sebclem.fr
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: ${{ github.ref_type == 'tag' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.vscode/launch.json

@@ -4,11 +4,21 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Python: Remote Attach",
+      "type": "python",
+      "request": "attach",
+      "connect": { "host": "docker.cloud.home", "port": 5678 },
+      "pathMappings": [
+        { "localRoot": "${workspaceFolder}", "remoteRoot": "." }
+      ],
+      "justMyCode": true
+    },
     {
       "name": "Python: Current File",
       "type": "python",
       "request": "launch",
-      "program": "main.py",
+      "program": "src/main.py",
       "envFile": "${workspaceFolder}/.env",
       "console": "integratedTerminal",
       "justMyCode": true

.vscode/settings.json

@@ -1,6 +1,9 @@
 {
   "[python]": {
-    "editor.defaultFormatter": "ms-python.black-formatter"
+    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.codeActionsOnSave": {
+      "source.organizeImports": true
+    }
   },
   "python.formatting.provider": "none",
   "python.analysis.autoImportCompletions": true,

Dockerfile

@@ -1,5 +1,5 @@
 # For more information, please refer to https://aka.ms/vscode-docker-python
-FROM python:3.10-slim
+FROM python:3.12-alpine
 
 # Keeps Python from generating .pyc files in the container
 ENV PYTHONDONTWRITEBYTECODE=1
@@ -7,6 +7,8 @@ ENV PYTHONDONTWRITEBYTECODE=1
 # Turns off buffering for easier container logging
 ENV PYTHONUNBUFFERED=1
 
+RUN apk add build-base
+
 # Install pip requirements
 COPY requirements.txt .
 RUN python -m pip install -r requirements.txt
@@ -19,4 +21,5 @@ COPY . /app
 RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
 USER appuser
 
-CMD ["python", "main.py"]
+# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
+CMD ["python", "src/main.py"]

docker-compose.debug.yml

@@ -6,6 +6,6 @@ services:
     build:
       context: .
       dockerfile: ./Dockerfile
-    command: ["sh", "-c", "pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 main.py "]
+    command: ["sh", "-c", "pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 src/main.py "]
     ports:
       - 5678:5678

main.py

@@ -1,55 +0,0 @@
-import math
-import os
-import re
-from shlex import join
-from tempfile import TemporaryDirectory
-import discord
-import pixiv
-import send_message
-
-REFRESH_TOKEN = os.getenv("PIXIV_REFRESH_TOKEN")
-if REFRESH_TOKEN is None:
-    print("Pixiv refresh token is missing, please set PIXIV_REFRESH_TOKEN")
-    exit(1)
-DISCORD_TOKEN = os.getenv("DISCORD_TOKEN")
-if DISCORD_TOKEN is None:
-    print("Discord token is missing, please set DISCORD_TOKEN")
-    exit(1)
-
-
-intents = discord.Intents.default()
-intents.message_content = True
-
-client = discord.Client(intents=intents)
-
-
-@client.event
-async def on_ready():
-    print(f"We have logged in as {client.user}")
-
-
-@client.event
-async def on_message(message: discord.Message):
-    if message.author == client.user:
-        return
-
-    if (
-        match := re.search(
-            r"(?P<url>https?://(?:www\.)?pixiv.net/\w+/artworks/(?P<id>\d+))",
-            message.content,
-        )
-    ) is not None:
-        with TemporaryDirectory() as tmp_dir:
-            async with message.channel.typing():
-                title = pixiv.dowload_pixiv_images(
-                    int(match.group("id")), tmp_dir, REFRESH_TOKEN
-                )
-                files = os.listdir(tmp_dir)
-                files.reverse()
-                await send_message.send_message_with_embed(
-                    message, files, title, tmp_dir, match.group("url")
-                )
-                await send_message.send_message(message, files, tmp_dir)
-
-
-client.run(DISCORD_TOKEN)

pixiv.py

@@ -1,27 +0,0 @@
-import pixivpy3
-
-
-def dowload_pixiv_images(
-    illust_id: int,
-    dest_folder: str,
-    refresh_token: str,
-):
-    api = pixivpy3.AppPixivAPI()
-    api.auth(refresh_token=refresh_token)
-    # get origin url
-    json_result = api.illust_detail(illust_id)
-    illust = json_result.illust
-    pages = illust["meta_pages"]
-    if len(pages) > 0:
-        print(f"Dowloading mutiple images to {dest_folder}")
-        for page in pages:
-            url = page["image_urls"]["original"]
-            print(url)
-            api.download(url=url, path=dest_folder)
-    else:
-        print(f"Dowloading single image to {dest_folder}")
-        api.download(
-            url=illust["meta_single_page"]["original_image_url"], path=dest_folder
-        )
-    print("Down4load finished !")
-    return illust["title"]

renovate.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ],
+  "commitMessagePrefix": ":arrow_up:"
+
+}

requirements.txt

@@ -1,2 +1,21 @@
-pixivpy3
-discord.py
+pixivpy3==3.7.5
+discord.py==2.3.2
+
+# Freeze
+aiohttp==3.9.5
+aiosignal==1.3.1
+async-timeout==4.0.3
+attrs==23.2.0
+certifi==2024.6.2
+charset-normalizer==3.3.2
+cloudscraper==1.2.71
+frozenlist==1.4.1
+idna==3.7
+isort==5.13.2
+multidict==6.0.5
+pyparsing==3.1.2
+requests==2.32.3
+requests-toolbelt==1.0.0
+typing_extensions==4.12.2
+urllib3==2.2.2
+yarl==1.9.4

send_message.py

@@ -1,59 +0,0 @@
-import math
-import os
-import discord
-
-
-async def send_message_with_embed(
-    message: discord.Message, files, title: str, tmp_dir: str, url: str
-):
-    to_send_files = []
-    to_send_embeds = []
-
-    for file in files:
-        joined = os.path.join(tmp_dir, file)
-        embed = discord.Embed(url=url)
-        embed.set_image(url=f"attachment://{file}")
-        to_send_files.append(discord.File(joined, file))
-        to_send_embeds.append(embed)
-    if len(to_send_embeds) > 4:
-        parts = math.ceil(len(to_send_embeds) / 4)
-        i = 1
-        while len(to_send_embeds) > 4:
-            splited_file = to_send_files[:4]
-            to_send_files = to_send_files[4:]
-            splited_embed = to_send_embeds[:4]
-            to_send_embeds = to_send_embeds[4:]
-            for embed in splited_embed:
-                embed.title = f"{title} - Part {i}/{parts}"
-            if i == 1:
-                await message.reply(files=splited_file, embeds=splited_embed)
-            else:
-                await message.channel.send(files=splited_file, embeds=splited_embed)
-            i = i + 1
-        for embed in to_send_embeds:
-            embed.title = f"{title} - Part {i}/{parts}"
-        await message.channel.send(files=to_send_files, embeds=to_send_embeds)
-    else:
-        for embed in to_send_embeds:
-            embed.title = title
-        await message.reply(files=to_send_files, embeds=to_send_embeds)
-
-
-async def send_message(message: discord.Message, files, tmp_dir: str):
-    to_send_files = []
-    for file in files:
-        joined = os.path.join(tmp_dir, file)
-        to_send_files.append(discord.File(joined, file))
-    if len(to_send_files) > 10:
-        first = True
-        while len(to_send_files) > 10:
-            splited_file = to_send_files[:4]
-            to_send_files = to_send_files[4:]
-            if first:
-                await message.reply(files=splited_file)
-                first = False
-            else:
-                await message.channel.send(files=splited_file)
-        await message.channel.send(files=to_send_files)
-    else:
-        await message.reply(files=to_send_files)

src/discord_tools/message.py

@@ -0,0 +1,108 @@
+import os
+from typing import List
+
+import discord
+from idna import intranges_contain
+
+TIPS_MESSAGE = ":bulb: TIPS: You can disable Discord's default preview for link by wrapping it around `<` `>`"
+IMAGE_NBR_LIMIT = 10
+DEFAULT_UPLOAD_SIZE_LIMIT = 209715200
+
+
+async def send_message(
+    message: discord.Message,
+    files,
+    tmp_dir: str,
+    muted: bool,
+    title: str,
+    art_author: str,
+):
+    to_send_files = []
+    for file in files:
+        joined = os.path.join(tmp_dir, file)
+        to_send_files.append(discord.File(joined, file))
+
+    upload_size_limit = (
+        message.guild.filesize_limit
+        if message.guild is not None
+        else DEFAULT_UPLOAD_SIZE_LIMIT
+    )
+    first = True
+    while len(to_send_files) > 0:
+        nbr_file_to_send = get_max_files_number(to_send_files, upload_size_limit)
+        splited_file = to_send_files[:nbr_file_to_send]
+        to_send_files = to_send_files[nbr_file_to_send:]
+        if first and nbr_file_to_send <= 0:
+            return
+        elif nbr_file_to_send <= 0:
+            # Next file is to big to be sent, skip it
+            to_send_files = to_send_files[1:]
+        elif first:
+            await message.reply(
+                files=splited_file,
+                content=f"**{title}** by `{art_author}` {chr(10) + TIPS_MESSAGE if not muted else ''}",
+            )
+            first = False
+        else:
+            await message.channel.send(files=splited_file)  # type: ignore
+
+
+async def send_command_reply(
+    interaction: discord.Interaction,
+    files,
+    tmp_dir: str,
+    muted: bool,
+    title: str,
+    art_author: str,
+    url: str,
+    author: discord.User | discord.Member,
+):
+    to_send_files = []
+    for file in files:
+        joined = os.path.join(tmp_dir, file)
+        to_send_files.append(discord.File(joined, file))
+
+    upload_size_limit = (
+        interaction.guild.filesize_limit
+        if interaction.guild is not None
+        else DEFAULT_UPLOAD_SIZE_LIMIT
+    )
+
+    first = True
+    while len(to_send_files) > 0:
+        nbr_file_to_send = get_max_files_number(to_send_files, upload_size_limit)
+        splited_file = to_send_files[:nbr_file_to_send]
+        to_send_files = to_send_files[nbr_file_to_send:]
+        if first and nbr_file_to_send <= 0:
+            await interaction.followup.send(
+                content=f"This image exceeds upload file size limit", ephemeral=True
+            )
+        elif nbr_file_to_send <= 0:
+            # Next file is to big to be sent, skip it
+            to_send_files = to_send_files[1:]
+        elif first:
+            await interaction.followup.send(
+                files=splited_file,
+                content=f"**{title}** by `{art_author}` - <{url}> - Submited by {author.mention} {chr(10) + TIPS_MESSAGE if not muted else ''}",
+            )
+            first = False
+        else:
+            await interaction.channel.send(files=splited_file)  # type: ignore
+
+
+def get_max_files_number(files: List[discord.File], size_limit: int):
+    size = 0
+    nbr = 0
+    for file in files:
+        size = size + os.path.getsize(file.fp.name)  # type: ignore
+        nbr = nbr + 1
+        if nbr >= IMAGE_NBR_LIMIT and size < size_limit:
+            return IMAGE_NBR_LIMIT
+        elif size > size_limit:
+            # We exceeds size limit, remove one image
+            if nbr >= IMAGE_NBR_LIMIT:
+                return IMAGE_NBR_LIMIT - 1
+            else:
+                return nbr - 1
+    # We didn't reach any limit
+    return nbr

src/discord_tools/pixiv_link_extractor.py

@@ -0,0 +1,25 @@
+import re
+from typing import Tuple, Union
+
+import discord
+
+link_regexp = r"(?P<muted1><)?(?P<url>https?://(?:www\.)?pixiv.net/(?:\w+/)?artworks/(?P<id>\d+))(?P<muted2>>)?"
+
+
+def should_process(
+    message: discord.Message,
+) -> Tuple[bool, bool, Union[re.Match, None]]:
+    if match := re.search(link_regexp, message.content):
+        muted = bool(match.group("muted1")) and bool(match.group("muted2"))
+        if len(message.attachments) == 0:
+            return (True, muted, match)
+        else:
+            return (False, False, match)
+    return (False, False, None)
+
+
+def check_pixiv_url(url: str):
+    if match := re.search(link_regexp, url):
+        return (True, match)
+    else:
+        return (False, None)

src/main.py

@@ -0,0 +1,161 @@
+import asyncio
+import logging
+import os
+from shlex import join
+from tempfile import TemporaryDirectory
+
+import discord
+
+import discord_tools.message
+import discord_tools.pixiv_link_extractor
+import pixiv.pixiv_api as pixiv_api
+
+logging.basicConfig(
+    format="[%(asctime)s] [%(levelname)8s] %(name)s: %(message)s", level=logging.INFO
+)
+
+
+REFRESH_TOKEN = os.getenv("PIXIV_REFRESH_TOKEN")
+if REFRESH_TOKEN is None:
+    logging.fatal("Pixiv refresh token is missing, please set PIXIV_REFRESH_TOKEN")
+    exit(1)
+DISCORD_TOKEN = os.getenv("DISCORD_TOKEN")
+if DISCORD_TOKEN is None:
+    logging.fatal("Discord token is missing, please set DISCORD_TOKEN")
+    exit(1)
+
+
+intents = discord.Intents.default()
+intents.message_content = True
+
+
+class MyClient(discord.Client):
+    def __init__(self, *, intents: discord.Intents):
+        super().__init__(intents=intents)
+        self.tree = discord.app_commands.CommandTree(self)
+
+    async def setup_hook(self):
+        await self.tree.sync()
+
+
+client = MyClient(intents=intents)
+
+
+@client.event
+async def on_ready():
+    logging.info(f"We have logged in as {client.user}")
+
+
+@client.tree.command()
+@discord.app_commands.describe(
+    url="Pixiv art url",
+)
+async def pixivprev(interaction: discord.Interaction, url: str):
+    """Create pixiv preview"""
+    (result, match) = discord_tools.pixiv_link_extractor.check_pixiv_url(url)
+    if result and match is not None:
+        await interaction.response.defer()
+        with TemporaryDirectory() as tmp_dir:
+            loop = asyncio.get_running_loop()
+            data = await loop.run_in_executor(
+                None,
+                pixiv_api.download_pixiv_images,
+                int(match.group("id")),
+                tmp_dir,
+                REFRESH_TOKEN,
+            )
+            if data is None:
+                await interaction.response.send_message(
+                    "Fail to extract pixiv image", ephemeral=True
+                )
+                return
+            (title, author) = data
+            files = os.listdir(tmp_dir)
+            files.sort()
+            await discord_tools.message.send_command_reply(
+                interaction,
+                files,
+                tmp_dir,
+                True,
+                title=title,
+                art_author=author,
+                url=url,
+                author=interaction.user,
+            )
+    else:
+        await interaction.response.send_message("Invalid pixiv url", ephemeral=True)
+
+
+@client.tree.context_menu(name="Create pixiv preview")
+async def pixiv_preview(interaction: discord.Interaction, message: discord.Message):
+    (result, muted, match) = discord_tools.pixiv_link_extractor.should_process(message)
+    if result and match is not None:
+        await interaction.response.defer()
+        with TemporaryDirectory() as tmp_dir:
+            loop = asyncio.get_running_loop()
+            data = await loop.run_in_executor(
+                None,
+                pixiv_api.download_pixiv_images,
+                int(match.group("id")),
+                tmp_dir,
+                REFRESH_TOKEN,
+            )
+            if data is None:
+                await interaction.response.send_message(
+                    "Fail to extract pixiv image", ephemeral=True
+                )
+                return
+            files = os.listdir(tmp_dir)
+            files.sort()
+            (title, author) = data
+            await discord_tools.message.send_command_reply(
+                interaction=interaction,
+                files=files,
+                tmp_dir=tmp_dir,
+                muted=muted,
+                title=title,
+                art_author=author,
+                url=match.group("url"),
+                author=message.author,
+            )
+    else:
+        await interaction.response.send_message(
+            "Can't find any valid pixiv link.", ephemeral=True
+        )
+
+
+@client.event
+async def on_guild_join(guild: discord.Guild):
+    logging.info(f"Joined new guild: {guild.name}")
+
+
+@client.event
+async def on_message(message: discord.Message):
+    if message.author == client.user:
+        return
+    (result, muted, match) = discord_tools.pixiv_link_extractor.should_process(message)
+    if result and match is not None:
+        with TemporaryDirectory() as tmp_dir:
+            async with message.channel.typing():
+                loop = asyncio.get_running_loop()
+                data = await loop.run_in_executor(
+                    None,
+                    pixiv_api.download_pixiv_images,
+                    int(match.group("id")),
+                    tmp_dir,
+                    REFRESH_TOKEN,
+                )
+                if data is None:
+                    return
+                (title, author) = data
+                files = os.listdir(tmp_dir)
+                files.sort()
+                # await send_message.send_message_with_embed(
+                #     message, files, title, tmp_dir, match.group("url")
+                # )
+                await discord_tools.message.send_message(
+                    message, files, tmp_dir, muted, title, author
+                )
+
+
+client.run(DISCORD_TOKEN)

src/pixiv/pixiv_api.py

@@ -0,0 +1,80 @@
+import logging
+import os
+import threading
+from queue import Queue
+from threading import Thread
+from typing import List, Tuple, Union
+
+import pixivpy3
+
+cpu_count = os.cpu_count()
+max_thread_num = cpu_count * 2 if cpu_count is not None else 4
+if max_thread_num > 10:
+    max_thread_num = 10
+
+
+def download_pixiv_images(
+    illust_id: int,
+    dest_folder: str,
+    refresh_token: str,
+):
+    api = pixivpy3.AppPixivAPI()
+    api.auth(refresh_token=refresh_token)
+    # get origin url
+    json_result = api.illust_detail(illust_id)
+    illust = json_result.illust
+    if not illust["visible"]:
+        logging.info("Can't be viewed, return None")
+        return None
+    pages = illust["meta_pages"]
+    if len(pages) > 0:
+        logging.info(f"Dowloading mutiple images to {dest_folder} ({len(pages)})")
+        if max_thread_num > len(pages):
+            num_threads = len(pages)
+        else:
+            num_threads = max_thread_num
+
+        logging.info(f"Starting {num_threads} thread(s)")
+        workers: List[Thread] = []
+        q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]] = Queue()
+        for i in range(num_threads):
+            worker = Thread(target=download_thread, args=(q,))
+            worker.setDaemon(True)
+            worker.start()
+            workers.append(worker)
+
+        for page in pages:
+            url = page["image_urls"]["original"]
+            q.put((url, dest_folder, api))
+
+        q.join()
+
+        # Send end signal to all threads
+        for i in range(num_threads):
+            q.put(None)
+        # Wait for them to finish
+        for worker in workers:
+            worker.join()
+
+    else:
+        logging.info(f"Dowloading single image to {dest_folder}")
+        api.download(
+            url=illust["meta_single_page"]["original_image_url"], path=dest_folder
+        )
+    logging.info("Download finished !")
+    return (illust["title"], f"{illust['user']['name']} ({illust['user']['account']})")
+
+
+def download_thread(q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]]):
+    thread_name = threading.currentThread().getName()
+    while True:
+        data = q.get()
+        if data is None:
+            logging.debug(f"[{thread_name}] - Bye bye")
+            q.task_done()
+            break
+        (url, dest_folder, api) = data
+        logging.debug(f"[{thread_name}] - Downloading {url}...")
+        api.download(url=url, path=dest_folder)
+        print(f"[{thread_name}] - ... Done")
+        q.task_done()