爬取结果异常处理

@@ -69,9 +69,13 @@ def get_job_info(taskId: int):
 
 
 
 
 async def scrawl_and_save(taskParam: SpiderParams):
 async def scrawl_and_save(taskParam: SpiderParams):
-    # 执行爬虫获取结果
-    results = await startBrowser(taskParam)
-    logger.info(f"爬虫重试情况:{startBrowser.statistics}")
+    try:
+        # 执行爬虫获取结果 给下面一行代码添加 try  cache try 捕获异常
+        results = await startBrowser(taskParam)
+    except Exception as e:
+        logger.info(f"爬虫重试情况:{startBrowser.statistics}")
+        logger.error(f"爬虫任务执行失败,失败原因:{e}")
+        return
     asyncTasks = (save_or_update(item) for item in results)
     asyncTasks = (save_or_update(item) for item in results)
     await asyncio.gather(*asyncTasks)
     await asyncio.gather(*asyncTasks)
     logger.info(f"爬虫任务执行完成,爬取到数据{len(results)}条 保存到数据库完成")
     logger.info(f"爬虫任务执行完成,爬取到数据{len(results)}条 保存到数据库完成")

@@ -12,8 +12,7 @@ from tenacity import (
     before_sleep_log,
     before_sleep_log,
     retry,
     retry,
     stop_after_attempt,
     stop_after_attempt,
-    wait_exponential,
-    wait_fixed,
+    wait_exponential
 )
 )
 
 
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)