Add thread for multiple image
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
This commit is contained in:
parent
61ef3bfd30
commit
5912a8527f
@ -1,5 +1,14 @@
|
|||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from queue import Queue
|
||||||
|
from threading import Thread
|
||||||
|
from typing import List, Tuple, Union
|
||||||
|
|
||||||
import pixivpy3
|
import pixivpy3
|
||||||
|
|
||||||
|
cpu_count = os.cpu_count()
|
||||||
|
max_thread_num = cpu_count * 4 if cpu_count is not None else 4
|
||||||
|
|
||||||
|
|
||||||
def dowload_pixiv_images(
|
def dowload_pixiv_images(
|
||||||
illust_id: int,
|
illust_id: int,
|
||||||
@ -16,13 +25,52 @@ def dowload_pixiv_images(
|
|||||||
pages = illust["meta_pages"]
|
pages = illust["meta_pages"]
|
||||||
if len(pages) > 0:
|
if len(pages) > 0:
|
||||||
print(f"Dowloading mutiple images to {dest_folder} ({len(pages)})")
|
print(f"Dowloading mutiple images to {dest_folder} ({len(pages)})")
|
||||||
|
if max_thread_num > len(pages):
|
||||||
|
num_threads = len(pages)
|
||||||
|
else:
|
||||||
|
num_threads = max_thread_num
|
||||||
|
|
||||||
|
print(f"Starting {num_threads} thread(s)")
|
||||||
|
workers: List[Thread] = []
|
||||||
|
q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]] = Queue()
|
||||||
|
for i in range(num_threads):
|
||||||
|
worker = Thread(target=download_thread, args=(q,))
|
||||||
|
worker.setDaemon(True)
|
||||||
|
worker.start()
|
||||||
|
workers.append(worker)
|
||||||
|
|
||||||
for page in pages:
|
for page in pages:
|
||||||
url = page["image_urls"]["original"]
|
url = page["image_urls"]["original"]
|
||||||
api.download(url=url, path=dest_folder)
|
q.put((url, dest_folder, api))
|
||||||
|
|
||||||
|
q.join()
|
||||||
|
|
||||||
|
# Send end signal to all threads
|
||||||
|
for i in range(num_threads):
|
||||||
|
q.put(None)
|
||||||
|
# Wait for them to finish
|
||||||
|
for worker in workers:
|
||||||
|
worker.join()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
print(f"Dowloading single image to {dest_folder}")
|
print(f"Dowloading single image to {dest_folder}")
|
||||||
api.download(
|
api.download(
|
||||||
url=illust["meta_single_page"]["original_image_url"], path=dest_folder
|
url=illust["meta_single_page"]["original_image_url"], path=dest_folder
|
||||||
)
|
)
|
||||||
print("Down4load finished !")
|
print("Download finished !")
|
||||||
return illust["title"]
|
return illust["title"]
|
||||||
|
|
||||||
|
|
||||||
|
def download_thread(q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]]):
|
||||||
|
thread_name = threading.currentThread().getName()
|
||||||
|
while True:
|
||||||
|
data = q.get()
|
||||||
|
if data is None:
|
||||||
|
print(f"[{thread_name}] - Bye bye")
|
||||||
|
q.task_done()
|
||||||
|
break
|
||||||
|
(url, dest_folder, api) = data
|
||||||
|
print(f"[{thread_name}] - Downloading {url}...")
|
||||||
|
api.download(url=url, path=dest_folder)
|
||||||
|
print(f"[{thread_name}] - ... Done")
|
||||||
|
q.task_done()
|
||||||
|
Loading…
Reference in New Issue
Block a user