Compare commits

..

No commits in common. "main" and "0.0.6" have entirely different histories.
main ... 0.0.6

10 changed files with 60 additions and 305 deletions

View File

@ -1,48 +0,0 @@
name: ci
on:
push:
branches:
- "*"
tags:
- "*"
pull_request:
branches:
- "main"
jobs:
docker:
runs-on: ubuntu-latest
container:
image: catthehacker/ubuntu:act-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: |
git.sebclem.fr/${{ gitea.repository }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
if: github.ref_type == 'tag'
uses: docker/login-action@v3
with:
registry: git.sebclem.fr
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
push: ${{ github.ref_type == 'tag' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

10
.vscode/launch.json vendored
View File

@ -4,16 +4,6 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0", "version": "0.2.0",
"configurations": [ "configurations": [
{
"name": "Python: Remote Attach",
"type": "python",
"request": "attach",
"connect": { "host": "docker.cloud.home", "port": 5678 },
"pathMappings": [
{ "localRoot": "${workspaceFolder}", "remoteRoot": "." }
],
"justMyCode": true
},
{ {
"name": "Python: Current File", "name": "Python: Current File",
"type": "python", "type": "python",

View File

@ -1,5 +1,5 @@
# For more information, please refer to https://aka.ms/vscode-docker-python # For more information, please refer to https://aka.ms/vscode-docker-python
FROM python:3.13-alpine FROM python:3.10-slim
# Keeps Python from generating .pyc files in the container # Keeps Python from generating .pyc files in the container
ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONDONTWRITEBYTECODE=1
@ -7,8 +7,6 @@ ENV PYTHONDONTWRITEBYTECODE=1
# Turns off buffering for easier container logging # Turns off buffering for easier container logging
ENV PYTHONUNBUFFERED=1 ENV PYTHONUNBUFFERED=1
RUN apk add build-base
# Install pip requirements # Install pip requirements
COPY requirements.txt . COPY requirements.txt .
RUN python -m pip install -r requirements.txt RUN python -m pip install -r requirements.txt
@ -21,5 +19,4 @@ COPY . /app
RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
USER appuser USER appuser
# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
CMD ["python", "src/main.py"] CMD ["python", "src/main.py"]

View File

@ -1,11 +0,0 @@
version: '3.4'
services:
pixivcord:
image: pixivcord
build:
context: .
dockerfile: ./Dockerfile
command: ["sh", "-c", "pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 src/main.py "]
ports:
- 5678:5678

View File

@ -1,8 +0,0 @@
version: '3.4'
services:
pixivcord:
image: pixivcord
build:
context: .
dockerfile: ./Dockerfile

View File

@ -1,8 +0,0 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"config:recommended"
],
"commitMessagePrefix": ":arrow_up:"
}

View File

@ -1,21 +1,2 @@
pixivpy3==3.7.5 pixivpy3
discord.py==2.4.0 discord.py
# Freeze
aiohttp==3.10.10
aiosignal==1.3.1
async-timeout==4.0.3
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
cloudscraper==1.2.71
frozenlist==1.5.0
idna==3.10
isort==5.13.2
multidict==6.1.0
pyparsing==3.2.0
requests==2.32.3
requests-toolbelt==1.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.17.0

View File

@ -1,50 +1,33 @@
import math
import os import os
from typing import List
import discord import discord
from idna import intranges_contain
TIPS_MESSAGE = ":bulb: TIPS: You can disable Discord's default preview for link by wrapping it around `<` `>`" TIPS_MESSAGE = ":bulb: TIPS: You can disable Discord's default preview for link by wrapping it around `<` `>`"
IMAGE_NBR_LIMIT = 10
DEFAULT_UPLOAD_SIZE_LIMIT = 209715200
async def send_message( async def send_message(message: discord.Message, files, tmp_dir: str, muted: bool):
message: discord.Message,
files,
tmp_dir: str,
muted: bool,
title: str,
art_author: str,
):
to_send_files = [] to_send_files = []
for file in files: for file in files:
joined = os.path.join(tmp_dir, file) joined = os.path.join(tmp_dir, file)
to_send_files.append(discord.File(joined, file)) to_send_files.append(discord.File(joined, file))
if len(to_send_files) > 10:
upload_size_limit = ( first = True
message.guild.filesize_limit while len(to_send_files) > 10:
if message.guild is not None splited_file = to_send_files[:10]
else DEFAULT_UPLOAD_SIZE_LIMIT to_send_files = to_send_files[10:]
) if first:
first = True await message.reply(
while len(to_send_files) > 0: files=splited_file, content=(TIPS_MESSAGE if not muted else None)
nbr_file_to_send = get_max_files_number(to_send_files, upload_size_limit) )
splited_file = to_send_files[:nbr_file_to_send] first = False
to_send_files = to_send_files[nbr_file_to_send:] else:
if first and nbr_file_to_send <= 0: await message.channel.send(files=splited_file)
return await message.channel.send(files=to_send_files)
elif nbr_file_to_send <= 0: else:
# Next file is to big to be sent, skip it await message.reply(
to_send_files = to_send_files[1:] files=to_send_files, content=(TIPS_MESSAGE if not muted else None)
elif first: )
await message.reply(
files=splited_file,
content=f"**{title}** by `{art_author}` {chr(10) + TIPS_MESSAGE if not muted else ''}",
)
first = False
else:
await message.channel.send(files=splited_file) # type: ignore
async def send_command_reply( async def send_command_reply(
@ -53,7 +36,6 @@ async def send_command_reply(
tmp_dir: str, tmp_dir: str,
muted: bool, muted: bool,
title: str, title: str,
art_author: str,
url: str, url: str,
author: discord.User | discord.Member, author: discord.User | discord.Member,
): ):
@ -61,48 +43,23 @@ async def send_command_reply(
for file in files: for file in files:
joined = os.path.join(tmp_dir, file) joined = os.path.join(tmp_dir, file)
to_send_files.append(discord.File(joined, file)) to_send_files.append(discord.File(joined, file))
if len(to_send_files) > 10:
upload_size_limit = ( if interaction.channel is not None:
interaction.guild.filesize_limit first = True
if interaction.guild is not None while len(to_send_files) > 10:
else DEFAULT_UPLOAD_SIZE_LIMIT splited_file = to_send_files[:10]
) to_send_files = to_send_files[10:]
if first:
first = True await interaction.followup.send(
while len(to_send_files) > 0: files=splited_file,
nbr_file_to_send = get_max_files_number(to_send_files, upload_size_limit) content=f"{title} - Submited by {author.mention} - <{url}> {chr(10) + TIPS_MESSAGE if not muted else ''}",
splited_file = to_send_files[:nbr_file_to_send] )
to_send_files = to_send_files[nbr_file_to_send:] first = False
if first and nbr_file_to_send <= 0: else:
await interaction.followup.send( await interaction.channel.send(files=splited_file) # type: ignore
content=f"This image exceeds upload file size limit", ephemeral=True await interaction.channel.send(files=to_send_files) # type: ignore
) else:
elif nbr_file_to_send <= 0: await interaction.followup.send(
# Next file is to big to be sent, skip it files=to_send_files,
to_send_files = to_send_files[1:] content=f"{title} - Submited by {author.mention} - <{url}> {chr(10) + TIPS_MESSAGE if not muted else ''}",
elif first: )
await interaction.followup.send(
files=splited_file,
content=f"**{title}** by `{art_author}` - <{url}> - Submited by {author.mention} {chr(10) + TIPS_MESSAGE if not muted else ''}",
)
first = False
else:
await interaction.channel.send(files=splited_file) # type: ignore
def get_max_files_number(files: List[discord.File], size_limit: int):
size = 0
nbr = 0
for file in files:
size = size + os.path.getsize(file.fp.name) # type: ignore
nbr = nbr + 1
if nbr >= IMAGE_NBR_LIMIT and size < size_limit:
return IMAGE_NBR_LIMIT
elif size > size_limit:
# We exceeds size limit, remove one image
if nbr >= IMAGE_NBR_LIMIT:
return IMAGE_NBR_LIMIT - 1
else:
return nbr - 1
# We didn't reach any limit
return nbr

View File

@ -1,5 +1,3 @@
import asyncio
import logging
import os import os
from shlex import join from shlex import join
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
@ -10,18 +8,13 @@ import discord_tools.message
import discord_tools.pixiv_link_extractor import discord_tools.pixiv_link_extractor
import pixiv.pixiv_api as pixiv_api import pixiv.pixiv_api as pixiv_api
logging.basicConfig(
format="[%(asctime)s] [%(levelname)8s] %(name)s: %(message)s", level=logging.INFO
)
REFRESH_TOKEN = os.getenv("PIXIV_REFRESH_TOKEN") REFRESH_TOKEN = os.getenv("PIXIV_REFRESH_TOKEN")
if REFRESH_TOKEN is None: if REFRESH_TOKEN is None:
logging.fatal("Pixiv refresh token is missing, please set PIXIV_REFRESH_TOKEN") print("Pixiv refresh token is missing, please set PIXIV_REFRESH_TOKEN")
exit(1) exit(1)
DISCORD_TOKEN = os.getenv("DISCORD_TOKEN") DISCORD_TOKEN = os.getenv("DISCORD_TOKEN")
if DISCORD_TOKEN is None: if DISCORD_TOKEN is None:
logging.fatal("Discord token is missing, please set DISCORD_TOKEN") print("Discord token is missing, please set DISCORD_TOKEN")
exit(1) exit(1)
@ -43,7 +36,7 @@ client = MyClient(intents=intents)
@client.event @client.event
async def on_ready(): async def on_ready():
logging.info(f"We have logged in as {client.user}") print(f"We have logged in as {client.user}")
@client.tree.command() @client.tree.command()
@ -56,20 +49,9 @@ async def pixivprev(interaction: discord.Interaction, url: str):
if result and match is not None: if result and match is not None:
await interaction.response.defer() await interaction.response.defer()
with TemporaryDirectory() as tmp_dir: with TemporaryDirectory() as tmp_dir:
loop = asyncio.get_running_loop() title = pixiv_api.dowload_pixiv_images(
data = await loop.run_in_executor( int(match.group("id")), tmp_dir, REFRESH_TOKEN
None,
pixiv_api.download_pixiv_images,
int(match.group("id")),
tmp_dir,
REFRESH_TOKEN,
) )
if data is None:
await interaction.response.send_message(
"Fail to extract pixiv image", ephemeral=True
)
return
(title, author) = data
files = os.listdir(tmp_dir) files = os.listdir(tmp_dir)
files.sort() files.sort()
await discord_tools.message.send_command_reply( await discord_tools.message.send_command_reply(
@ -78,7 +60,6 @@ async def pixivprev(interaction: discord.Interaction, url: str):
tmp_dir, tmp_dir,
True, True,
title=title, title=title,
art_author=author,
url=url, url=url,
author=interaction.user, author=interaction.user,
) )
@ -92,31 +73,19 @@ async def pixiv_preview(interaction: discord.Interaction, message: discord.Messa
if result and match is not None: if result and match is not None:
await interaction.response.defer() await interaction.response.defer()
with TemporaryDirectory() as tmp_dir: with TemporaryDirectory() as tmp_dir:
loop = asyncio.get_running_loop() title = pixiv_api.dowload_pixiv_images(
data = await loop.run_in_executor( int(match.group("id")), tmp_dir, REFRESH_TOKEN
None,
pixiv_api.download_pixiv_images,
int(match.group("id")),
tmp_dir,
REFRESH_TOKEN,
) )
if data is None:
await interaction.response.send_message(
"Fail to extract pixiv image", ephemeral=True
)
return
files = os.listdir(tmp_dir) files = os.listdir(tmp_dir)
files.sort() files.sort()
(title, author) = data
await discord_tools.message.send_command_reply( await discord_tools.message.send_command_reply(
interaction=interaction, interaction=interaction,
files=files, files=files,
tmp_dir=tmp_dir, tmp_dir=tmp_dir,
muted=muted, muted=muted,
title=title, title=title,
art_author=author,
url=match.group("url"), url=match.group("url"),
author=message.author, author=interaction.user,
) )
else: else:
await interaction.response.send_message( await interaction.response.send_message(
@ -126,7 +95,7 @@ async def pixiv_preview(interaction: discord.Interaction, message: discord.Messa
@client.event @client.event
async def on_guild_join(guild: discord.Guild): async def on_guild_join(guild: discord.Guild):
logging.info(f"Joined new guild: {guild.name}") print(f"Joined new guild: {guild.name}")
@client.event @client.event
@ -137,25 +106,15 @@ async def on_message(message: discord.Message):
if result and match is not None: if result and match is not None:
with TemporaryDirectory() as tmp_dir: with TemporaryDirectory() as tmp_dir:
async with message.channel.typing(): async with message.channel.typing():
loop = asyncio.get_running_loop() title = pixiv_api.dowload_pixiv_images(
data = await loop.run_in_executor( int(match.group("id")), tmp_dir, REFRESH_TOKEN
None,
pixiv_api.download_pixiv_images,
int(match.group("id")),
tmp_dir,
REFRESH_TOKEN,
) )
if data is None:
return
(title, author) = data
files = os.listdir(tmp_dir) files = os.listdir(tmp_dir)
files.sort() files.sort()
# await send_message.send_message_with_embed( # await send_message.send_message_with_embed(
# message, files, title, tmp_dir, match.group("url") # message, files, title, tmp_dir, match.group("url")
# ) # )
await discord_tools.message.send_message( await discord_tools.message.send_message(message, files, tmp_dir, muted)
message, files, tmp_dir, muted, title, author
)
client.run(DISCORD_TOKEN) client.run(DISCORD_TOKEN)

View File

@ -1,19 +1,7 @@
import logging
import os
import threading
from queue import Queue
from threading import Thread
from typing import List, Tuple, Union
import pixivpy3 import pixivpy3
cpu_count = os.cpu_count()
max_thread_num = cpu_count * 2 if cpu_count is not None else 4
if max_thread_num > 10:
max_thread_num = 10
def dowload_pixiv_images(
def download_pixiv_images(
illust_id: int, illust_id: int,
dest_folder: str, dest_folder: str,
refresh_token: str, refresh_token: str,
@ -23,58 +11,16 @@ def download_pixiv_images(
# get origin url # get origin url
json_result = api.illust_detail(illust_id) json_result = api.illust_detail(illust_id)
illust = json_result.illust illust = json_result.illust
if not illust["visible"]:
logging.info("Can't be viewed, return None")
return None
pages = illust["meta_pages"] pages = illust["meta_pages"]
if len(pages) > 0: if len(pages) > 0:
logging.info(f"Dowloading mutiple images to {dest_folder} ({len(pages)})") print(f"Dowloading mutiple images to {dest_folder} ({len(pages)})")
if max_thread_num > len(pages):
num_threads = len(pages)
else:
num_threads = max_thread_num
logging.info(f"Starting {num_threads} thread(s)")
workers: List[Thread] = []
q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]] = Queue()
for i in range(num_threads):
worker = Thread(target=download_thread, args=(q,))
worker.setDaemon(True)
worker.start()
workers.append(worker)
for page in pages: for page in pages:
url = page["image_urls"]["original"] url = page["image_urls"]["original"]
q.put((url, dest_folder, api)) api.download(url=url, path=dest_folder)
q.join()
# Send end signal to all threads
for i in range(num_threads):
q.put(None)
# Wait for them to finish
for worker in workers:
worker.join()
else: else:
logging.info(f"Dowloading single image to {dest_folder}") print(f"Dowloading single image to {dest_folder}")
api.download( api.download(
url=illust["meta_single_page"]["original_image_url"], path=dest_folder url=illust["meta_single_page"]["original_image_url"], path=dest_folder
) )
logging.info("Download finished !") print("Down4load finished !")
return (illust["title"], f"{illust['user']['name']} ({illust['user']['account']})") return illust["title"]
def download_thread(q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]]):
thread_name = threading.currentThread().getName()
while True:
data = q.get()
if data is None:
logging.debug(f"[{thread_name}] - Bye bye")
q.task_done()
break
(url, dest_folder, api) = data
logging.debug(f"[{thread_name}] - Downloading {url}...")
api.download(url=url, path=dest_folder)
print(f"[{thread_name}] - ... Done")
q.task_done()