diff --git a/assets/docs/sources/tutorial/0_common_usage.md b/assets/docs/sources/tutorial/0_common_usage.md index 614dd3cb..7ca8472e 100644 --- a/assets/docs/sources/tutorial/0_common_usage.md +++ b/assets/docs/sources/tutorial/0_common_usage.md @@ -82,25 +82,38 @@ from jmcomic import * # 客户端 client = JmOption.default().new_jm_client() -# 捕获jmcomic可能出现的异常 +# 捕获获取本子/章节详情时可能出现的异常 try: # 请求本子实体类 album: JmAlbumDetail = client.get_album_detail('427413') except MissingAlbumPhotoException as e: print(f'id={e.error_jmid}的本子不存在') - + except JsonResolveFailException as e: print(f'解析json失败') # 响应对象 resp = e.resp print(f'resp.text: {resp.text}, resp.status_code: {resp.status_code}') - + except RequestRetryAllFailException as e: print(f'请求失败，重试次数耗尽') - + except JmcomicException as e: # 捕获所有异常，用作兜底 print(f'jmcomic遇到异常: {e}') + +# 多线程下载时，可能出现非当前线程下载失败，抛出异常， +# 而JmDownloader有对应字段记录了这些线程发生的异常 +# 使用check_exception=True参数可以使downloader主动检查是否存在下载异常 +# 如果有，则当前线程会主动上抛一个PartialDownloadFailedException异常 +# 该参数主要用于主动检查部分下载失败的情况， +# 因为非当前线程抛出的异常（比如下载章节的线程和下载图片的线程），这些线程如果抛出异常， +# 当前线程是感知不到的，try-catch下载方法download_album不能捕获到其他线程发生的异常。 +try: + album, downloader = download_album(123, check_exception=True) +except PartialDownloadFailedException as e: + downloader: JmDownloader = e.downloader + print(f'下载出现部分失败, 下载失败的章节: {downloader.download_failed_photo}, 下载失败的图片: {downloader.download_failed_image}') ``` diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index 93308a35..df2750cf 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.5.34' +__version__ = '2.5.35' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py index 5f657a90..47141450 100644 --- a/src/jmcomic/api.py +++ b/src/jmcomic/api.py @@ -48,6 +48,7 @@ def download_album(jm_album_id, option=None, downloader=None, callback=None, + check_exception=True, ) -> Union[__DOWNLOAD_API_RET, Set[__DOWNLOAD_API_RET]]: """ 下载一个本子（album），包含其所有的章节（photo） @@ -58,6 +59,7 @@ def download_album(jm_album_id, :param option: 下载选项 :param downloader: 下载器类 :param callback: 返回值回调函数，可以拿到 album 和 downloader + :param check_exception: 是否检查异常, 如果为True，会检查downloader是否有下载异常，并上抛PartialDownloadFailedException :return: 对于的本子实体类，下载器（如果是上述的批量情况，返回值为download_batch的返回值） """ @@ -69,14 +71,17 @@ def download_album(jm_album_id, if callback is not None: callback(album, dler) - + if check_exception: + dler.raise_if_has_exception() return album, dler def download_photo(jm_photo_id, option=None, downloader=None, - callback=None): + callback=None, + check_exception=True, + ): """ 下载一个章节（photo），参数同 download_album """ @@ -88,7 +93,8 @@ def download_photo(jm_photo_id, if callback is not None: callback(photo, dler) - + if check_exception: + dler.raise_if_has_exception() return photo, dler diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index f06bb784..a9ac3463 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -566,7 +566,7 @@ def check_special_text(cls, resp): cls.raise_request_error( resp, - f'{reason}' + f'{reason}({content})' + (f': {url}' if url is not None else '') ) diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py index 40df64ec..1518454a 100644 --- a/src/jmcomic/jm_downloader.py +++ b/src/jmcomic/jm_downloader.py @@ -1,6 +1,31 @@ from .jm_option import * +def catch_exception(func): + from functools import wraps + + @wraps(func) + def wrapper(self, *args, **kwargs): + self: JmDownloader + try: + return func(self, *args, **kwargs) + except Exception as e: + detail: JmBaseEntity = args[0] + if detail.is_image(): + detail: JmImageDetail + jm_log('image.failed', f'图片下载失败: [{detail.download_url}], 异常: [{e}]') + self.download_failed_image.append((detail, e)) + + elif detail.is_photo(): + detail: JmPhotoDetail + jm_log('photo.failed', f'章节下载失败: [{detail.id}], 异常: [{e}]') + self.download_failed_photo.append((detail, e)) + + raise e + + return wrapper + + # noinspection PyMethodMayBeStatic class DownloadCallback: @@ -50,48 +75,50 @@ class JmDownloader(DownloadCallback): def __init__(self, option: JmOption) -> None: self.option = option + self.client = option.build_jm_client() # 下载成功的记录dict self.download_success_dict: Dict[JmAlbumDetail, Dict[JmPhotoDetail, List[Tuple[str, JmImageDetail]]]] = {} # 下载失败的记录list - self.download_failed_list: List[Tuple[JmImageDetail, BaseException]] = [] + self.download_failed_image: List[Tuple[JmImageDetail, BaseException]] = [] + self.download_failed_photo: List[Tuple[JmPhotoDetail, BaseException]] = [] def download_album(self, album_id): - client = self.client_for_album(album_id) - album = client.get_album_detail(album_id) - self.download_by_album_detail(album, client) + album = self.client.get_album_detail(album_id) + self.download_by_album_detail(album) return album - def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient): + def download_by_album_detail(self, album: JmAlbumDetail): self.before_album(album) if album.skip: return - self.execute_by_condition( + self.execute_on_condition( iter_objs=album, - apply=lambda photo: self.download_by_photo_detail(photo, client), + apply=self.download_by_photo_detail, count_batch=self.option.decide_photo_batch_count(album) ) self.after_album(album) def download_photo(self, photo_id): - client = self.client_for_photo(photo_id) - photo = client.get_photo_detail(photo_id) - self.download_by_photo_detail(photo, client) + photo = self.client.get_photo_detail(photo_id) + self.download_by_photo_detail(photo) return photo - def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient): - client.check_photo(photo) + @catch_exception + def download_by_photo_detail(self, photo: JmPhotoDetail): + self.client.check_photo(photo) self.before_photo(photo) if photo.skip: return - self.execute_by_condition( + self.execute_on_condition( iter_objs=photo, - apply=lambda image: self.download_by_image_detail(image, client), + apply=self.download_by_image_detail, count_batch=self.option.decide_image_batch_count(photo) ) self.after_photo(photo) - def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient): + @catch_exception + def download_by_image_detail(self, image: JmImageDetail): img_save_path = self.option.decide_image_filepath(image) image.save_path = img_save_path @@ -110,22 +137,15 @@ def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient): if use_cache is True and image.exists: return - try: - client.download_by_image_detail( - image, - img_save_path, - decode_image=decode_image, - ) - except BaseException as e: - jm_log('image.failed', f'图片下载失败: [{image.download_url}], 异常: {e}') - # 保存失败记录 - self.download_failed_list.append((image, e)) - raise + self.client.download_by_image_detail( + image, + img_save_path, + decode_image=decode_image, + ) self.after_image(image, img_save_path) - # noinspection PyMethodMayBeStatic - def execute_by_condition(self, + def execute_on_condition(self, iter_objs: DetailEntity, apply: Callable, count_batch: int, @@ -166,20 +186,6 @@ def do_filter(self, detail: DetailEntity): """ return detail - # noinspection PyUnusedLocal - def client_for_album(self, jm_album_id) -> JmcomicClient: - """ - 默认情况下，所有的JmDownloader共用一个JmcomicClient - """ - return self.option.build_jm_client() - - # noinspection PyUnusedLocal - def client_for_photo(self, jm_photo_id) -> JmcomicClient: - """ - 默认情况下，所有的JmDownloader共用一个JmcomicClient - """ - return self.option.build_jm_client() - @property def all_success(self) -> bool: """ @@ -189,7 +195,7 @@ def all_success(self) -> bool: 注意！如果使用了filter机制，例如通过filter只下载3张图片，那么all_success也会为False """ - if len(self.download_failed_list) != 0: + if self.has_download_failures: return False for album, photo_dict in self.download_success_dict.items(): @@ -202,6 +208,10 @@ def all_success(self) -> bool: return True + @property + def has_download_failures(self): + return len(self.download_failed_image) != 0 or len(self.download_failed_photo) != 0 + # 下面是回调方法 def before_album(self, album: JmAlbumDetail): @@ -259,6 +269,23 @@ def after_image(self, image: JmImageDetail, img_save_path): downloader=self, ) + def raise_if_has_exception(self): + if not self.has_download_failures: + return + msg_ls = ['部分下载失败', '', ''] + + if len(self.download_failed_photo) != 0: + msg_ls[1] = f'共{len(self.download_failed_photo)}个章节下载失败: {self.download_failed_photo}' + + if len(self.download_failed_image) != 0: + msg_ls[2] = f'共{len(self.download_failed_image)}个图片下载失败: {self.download_failed_image}' + + ExceptionTool.raises( + '\n'.join(msg_ls), + {'downloader': self}, + PartialDownloadFailedException, + ) + # 下面是对with语法的支持 def __enter__(self): @@ -283,7 +310,7 @@ class DoNotDownloadImage(JmDownloader): 不会下载任何图片的Downloader，用作测试 """ - def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient): + def download_by_image_detail(self, image: JmImageDetail): # ensure make dir self.option.decide_image_filepath(image) @@ -297,12 +324,13 @@ class JustDownloadSpecificCountImage(JmDownloader): count_lock = Lock() count = 0 - def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient): + @catch_exception + def download_by_image_detail(self, image: JmImageDetail): # ensure make dir self.option.decide_image_filepath(image) if self.try_countdown(): - return super().download_by_image_detail(image, client) + return super().download_by_image_detail(image) def try_countdown(self): if self.count < 0: diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py index 16d99d7e..4d799b84 100644 --- a/src/jmcomic/jm_entity.py +++ b/src/jmcomic/jm_entity.py @@ -125,10 +125,9 @@ def idoname(self): return f'[{self.id}] {self.oname}' def __str__(self): - return f'{self.__class__.__name__}' \ - '{' \ - f'{self.id}: {self.title}' \ - '}' + return f'''{self.__class__.__name__}({self.__alias__()}-{self.id}: "{self.title}")''' + + __repr__ = __str__ @classmethod def __alias__(cls): @@ -258,6 +257,11 @@ def tag(self) -> str: def is_image(cls): return True + def __str__(self): + return f'''{self.__class__.__name__}(image-[{self.download_url}])''' + + __repr__ = __str__ + class JmPhotoDetail(DetailEntity, Downloadable): diff --git a/src/jmcomic/jm_exception.py b/src/jmcomic/jm_exception.py index a6040298..8305277b 100644 --- a/src/jmcomic/jm_exception.py +++ b/src/jmcomic/jm_exception.py @@ -15,6 +15,7 @@ def from_context(self, key): def __str__(self): return self.msg + class ResponseUnexpectedException(JmcomicException): description = '响应不符合预期异常' @@ -44,7 +45,6 @@ def pattern(self): class JsonResolveFailException(ResponseUnexpectedException): description = 'Json解析异常' - pass class MissingAlbumPhotoException(ResponseUnexpectedException): @@ -57,9 +57,15 @@ def error_jmid(self) -> str: class RequestRetryAllFailException(JmcomicException): description = '请求重试全部失败异常' - pass +class PartialDownloadFailedException(JmcomicException): + description = '部分章节或图片下载失败异常' + + @property + def downloader(self): + return self.from_context(ExceptionTool.CONTEXT_KEY_DOWNLOADER) + class ExceptionTool: """ 抛异常的工具 @@ -71,6 +77,7 @@ class ExceptionTool: CONTEXT_KEY_HTML = 'html' CONTEXT_KEY_RE_PATTERN = 'pattern' CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id' + CONTEXT_KEY_DOWNLOADER = 'downloader' @classmethod def raises(cls, diff --git a/tests/test_jmcomic/test_jm_api.py b/tests/test_jmcomic/test_jm_api.py index 226d5aad..ef401185 100644 --- a/tests/test_jmcomic/test_jm_api.py +++ b/tests/test_jmcomic/test_jm_api.py @@ -76,3 +76,24 @@ def run_func_async(func): print(e) raise AssertionError(exception_list) + + def test_partial_exception(self): + class TestDownloader(JmDownloader): + @catch_exception + def download_by_image_detail(self, image: JmImageDetail): + raise Exception('test_partial_exception') + + @catch_exception + def download_by_photo_detail(self, photo: JmPhotoDetail): + if photo.index != 2: + raise Exception('test_partial_exception') + return super().download_by_photo_detail(photo) + + self.assertRaises( + PartialDownloadFailedException, + lambda: download_album(182150, downloader=TestDownloader, check_exception=True) + ) + self.assertRaises( + PartialDownloadFailedException, + lambda: download_photo(182151, downloader=TestDownloader, check_exception=True) + )