Add thread for multiple image
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

This commit is contained in:
SebClem 2023-11-02 11:43:33 +01:00
parent 61ef3bfd30
commit 5912a8527f

View File

@ -1,5 +1,14 @@
import os
import threading
from queue import Queue
from threading import Thread
from typing import List, Tuple, Union
import pixivpy3 import pixivpy3
cpu_count = os.cpu_count()
max_thread_num = cpu_count * 4 if cpu_count is not None else 4
def dowload_pixiv_images( def dowload_pixiv_images(
illust_id: int, illust_id: int,
@ -16,13 +25,52 @@ def dowload_pixiv_images(
pages = illust["meta_pages"] pages = illust["meta_pages"]
if len(pages) > 0: if len(pages) > 0:
print(f"Dowloading mutiple images to {dest_folder} ({len(pages)})") print(f"Dowloading mutiple images to {dest_folder} ({len(pages)})")
if max_thread_num > len(pages):
num_threads = len(pages)
else:
num_threads = max_thread_num
print(f"Starting {num_threads} thread(s)")
workers: List[Thread] = []
q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]] = Queue()
for i in range(num_threads):
worker = Thread(target=download_thread, args=(q,))
worker.setDaemon(True)
worker.start()
workers.append(worker)
for page in pages: for page in pages:
url = page["image_urls"]["original"] url = page["image_urls"]["original"]
api.download(url=url, path=dest_folder) q.put((url, dest_folder, api))
q.join()
# Send end signal to all threads
for i in range(num_threads):
q.put(None)
# Wait for them to finish
for worker in workers:
worker.join()
else: else:
print(f"Dowloading single image to {dest_folder}") print(f"Dowloading single image to {dest_folder}")
api.download( api.download(
url=illust["meta_single_page"]["original_image_url"], path=dest_folder url=illust["meta_single_page"]["original_image_url"], path=dest_folder
) )
print("Down4load finished !") print("Download finished !")
return illust["title"] return illust["title"]
def download_thread(q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]]):
thread_name = threading.currentThread().getName()
while True:
data = q.get()
if data is None:
print(f"[{thread_name}] - Bye bye")
q.task_done()
break
(url, dest_folder, api) = data
print(f"[{thread_name}] - Downloading {url}...")
api.download(url=url, path=dest_folder)
print(f"[{thread_name}] - ... Done")
q.task_done()