Skip to content

文鑫 / guduo_spider

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 0c600de3 0c600de3ecd57a33c86696558cc14c9cbbf5664d by wenxin
Options

bug修复

1 parent a43dadc3
Inline Side-by-side
Showing 1 changed file with 6 additions and 3 deletions
...@@ -15,7 +15,7 @@ async def startBrowser(spiderParam: SpiderParams) -> List[GuoDuoSpiderResult]: ...@@ -15,7 +15,7 @@ async def startBrowser(spiderParam: SpiderParams) -> List[GuoDuoSpiderResult]:
15 url = spiderParam.url 15 url = spiderParam.url
16 scrawl_types = spiderParam.target_type 16 scrawl_types = spiderParam.target_type
17 async with async_playwright() as p: 17 async with async_playwright() as p:
18 browser = await p.firefox.launch(headless=False) 18 browser = await p.firefox.launch(headless=True)
19 # 创建指定个数的浏览器页面 19 # 创建指定个数的浏览器页面
20 pages = await asyncio.gather(*(browser.new_page() 20 pages = await asyncio.gather(*(browser.new_page()
21 for _ in range(len(time_range)))) 21 for _ in range(len(time_range))))
...@@ -46,6 +46,9 @@ async def hand_one_data(targetDate: str, ...@@ -46,6 +46,9 @@ async def hand_one_data(targetDate: str,
46 f'[data-pika-month="{month}"]' 46 f'[data-pika-month="{month}"]'
47 f'[data-pika-day="{day}"]') 47 f'[data-pika-day="{day}"]')
48 doc = page.locator(data_css_selector) 48 doc = page.locator(data_css_selector)
49 # 判断指定元素是否存在如果不存在就返回空的[]
50 if not await doc.is_visible():
51 return []
49 # 点击指定日期 52 # 点击指定日期
50 await doc.click() 53 await doc.click()
51 # 最后一步修正定位 54 # 最后一步修正定位
...@@ -91,8 +94,8 @@ async def get_data(targetPage: Page, scrawl_type: int, target_time: str) -> Opti ...@@ -91,8 +94,8 @@ async def get_data(targetPage: Page, scrawl_type: int, target_time: str) -> Opti
91 if __name__ == '__main__': 94 if __name__ == '__main__':
92 # 获取程序开始执行的时间 95 # 获取程序开始执行的时间
93 start_time = time.time() 96 start_time = time.time()
94 param = SpiderParams(startDate='2024-01-02', 97 param = SpiderParams(startDate='1991-01-02',
95 endDate='2024-01-05', 98 endDate='1991-01-05',
96 target_type=[TypeEnum.ANIME, TypeEnum.EPISODES, TypeEnum.MOVIE] 99 target_type=[TypeEnum.ANIME, TypeEnum.EPISODES, TypeEnum.MOVIE]
97 , url='http://guduodata.com/') 100 , url='http://guduodata.com/')
98 res = asyncio.run(startBrowser(param)) 101 res = asyncio.run(startBrowser(param))
......