Add thread for multiple image
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
This commit is contained in:
parent
61ef3bfd30
commit
5912a8527f
@ -1,5 +1,14 @@
|
||||
import os
|
||||
import threading
|
||||
from queue import Queue
|
||||
from threading import Thread
|
||||
from typing import List, Tuple, Union
|
||||
|
||||
import pixivpy3
|
||||
|
||||
cpu_count = os.cpu_count()
|
||||
max_thread_num = cpu_count * 4 if cpu_count is not None else 4
|
||||
|
||||
|
||||
def dowload_pixiv_images(
|
||||
illust_id: int,
|
||||
@ -16,13 +25,52 @@ def dowload_pixiv_images(
|
||||
pages = illust["meta_pages"]
|
||||
if len(pages) > 0:
|
||||
print(f"Dowloading mutiple images to {dest_folder} ({len(pages)})")
|
||||
if max_thread_num > len(pages):
|
||||
num_threads = len(pages)
|
||||
else:
|
||||
num_threads = max_thread_num
|
||||
|
||||
print(f"Starting {num_threads} thread(s)")
|
||||
workers: List[Thread] = []
|
||||
q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]] = Queue()
|
||||
for i in range(num_threads):
|
||||
worker = Thread(target=download_thread, args=(q,))
|
||||
worker.setDaemon(True)
|
||||
worker.start()
|
||||
workers.append(worker)
|
||||
|
||||
for page in pages:
|
||||
url = page["image_urls"]["original"]
|
||||
api.download(url=url, path=dest_folder)
|
||||
q.put((url, dest_folder, api))
|
||||
|
||||
q.join()
|
||||
|
||||
# Send end signal to all threads
|
||||
for i in range(num_threads):
|
||||
q.put(None)
|
||||
# Wait for them to finish
|
||||
for worker in workers:
|
||||
worker.join()
|
||||
|
||||
else:
|
||||
print(f"Dowloading single image to {dest_folder}")
|
||||
api.download(
|
||||
url=illust["meta_single_page"]["original_image_url"], path=dest_folder
|
||||
)
|
||||
print("Down4load finished !")
|
||||
print("Download finished !")
|
||||
return illust["title"]
|
||||
|
||||
|
||||
def download_thread(q: Queue[Union[Tuple[str, str, pixivpy3.AppPixivAPI], None]]):
|
||||
thread_name = threading.currentThread().getName()
|
||||
while True:
|
||||
data = q.get()
|
||||
if data is None:
|
||||
print(f"[{thread_name}] - Bye bye")
|
||||
q.task_done()
|
||||
break
|
||||
(url, dest_folder, api) = data
|
||||
print(f"[{thread_name}] - Downloading {url}...")
|
||||
api.download(url=url, path=dest_folder)
|
||||
print(f"[{thread_name}] - ... Done")
|
||||
q.task_done()
|
||||
|
Loading…
Reference in New Issue
Block a user