Python使用Selenium抓取动态网页的方法步骤

pip install selenium

pip install pyppeteer

pip install requests beautifulsoup4

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

# 设置 ChromeDriver 的路径
service = Service(executable_path='path/to/chromedriver')

# 初始化 Chrome 浏览器
driver = webdriver.Chrome(service=service)

# 打开目标网页
url = 'https://example.com/dynamic-page'
driver.get(url)

# 等待页面加载完成
try:
    # 等待特定元素加载，假设我们等待一个商品列表元素加载
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, "product-list"))
    )

    # 抓取页面的 HTML 内容
    page_content = driver.page_source
    print(page_content)

finally:
    # 关闭浏览器
    driver.quit()

# 查找商品名称的元素（假设 class 为 product-name）
products = driver.find_elements(By.CLASS_NAME, 'product-name')

for product in products:
    print(product.text)

# 模拟滚动，加载更多内容
SCROLL_PAUSE_TIME = 2

# 获取当前页面的高度
last_height = driver.execute_script("return document.body.scrollHeight")

while True:
    # 滚动到底部
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

    # 等待加载新内容
    time.sleep(SCROLL_PAUSE_TIME)

    # 获取新页面的高度
    new_height = driver.execute_script("return document.body.scrollHeight")
    if new_height == last_height:
        break  # 如果高度不变，说明已经加载完毕

    last_height = new_height

import asyncio
from pyppeteer import launch

async def fetch_dynamic_content(url):
    # 启动浏览器
    browser = await launch(headless=True)
    page = await browser.newPage()
    
    # 打开网页
    await page.goto(url)
    
    # 等待特定元素加载完成
    await page.waitForSelector('.product-list')

    # 获取页面内容
    content = await page.content()
    print(content)

    # 关闭浏览器
    await browser.close()

# 启动异步任务
url = 'https://example.com/dynamic-page'
asyncio.get_event_loop().run_until_complete(fetch_dynamic_content(url))

await page.screenshot({'path': 'screenshot.png'})

import requests

# 目标 URL（在浏览器网络面板中找到的 API 请求 URL）
url = 'https://example.com/api/products'

# 发送请求并获取数据
response = requests.get(url)
data = response.json()

# 打印数据
print(data)

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
response = requests.get(url, headers=headers)