python并发执行request请求的示例

pip install requests

import concurrent.futures  
import requests  
# 假设我们有一个URL列表  
urls = [  
    'http://example.com/api/data1',  
    'http://example.com/api/data2',  
    'http://example.com/api/data3',  
    # ... 添加更多URL  
]  
# 定义一个函数，该函数接收一个URL，发送GET请求，并打印响应内容  
def fetch_data(url):  
    try:  
        response = requests.get(url)  
        response.raise_for_status()  # 如果请求失败（例如，4xx、5xx），则抛出HTTPError异常  
        print(f"URL: {url}, Status Code: {response.status_code}, Content: {response.text[:100]}...")  
    except requests.RequestException as e:  
        print(f"Error fetching {url}: {e}")  
# 使用ThreadPoolExecutor并发地执行fetch_data函数  
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:  # 你可以根据需要调整max_workers的值  
    future_to_url = {executor.submit(fetch_data, url): url for url in urls}  
    for future in concurrent.futures.as_completed(future_to_url):  
        url = future_to_url[future]  
        try:  
            # 通过调用future.result()来获取函数的返回值，这会阻塞，直到结果可用  
            # 但是请注意，这里我们只是打印结果，没有返回值，所以调用future.result()只是为了等待函数完成  
            future.result()  
        except Exception as exc:  
            print(f'Generated an exception for {url}: {exc}')

import concurrent.futures  
import requests  
# 假设我们有一个URL列表  
urls = [  
    'https://www.example.com',  
    'https://httpbin.org/get',  
    'https://api.example.com/some/endpoint',  
    # ... 添加更多URL  
]  
# 定义一个函数来发送GET请求并处理响应  
def fetch_url(url):  
    try:  
        response = requests.get(url, timeout=5)  # 设置超时为5秒  
        response.raise_for_status()  # 如果请求失败，抛出HTTPError异常  
        return response.text  # 返回响应内容，这里只是作为示例，实际使用中可能不需要返回  
    except requests.RequestException as e:  
        print(f"Error fetching {url}: {e}")  
        return None  
# 使用ThreadPoolExecutor并发地发送请求  
def fetch_all_urls(urls):  
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:  
        # 使用executor.map来自动处理迭代和Future的获取  
        results = executor.map(fetch_url, urls)  
    # 处理结果（这里只是简单地打印出来）  
    for result in results:  
        if result is not None:  
            print(f"Fetched content from a URL (truncated): {result[:100]}...")  
# 调用函数  
fetch_all_urls(urls)

import threading  
import requests  
def fetch_url(url):  
    try:  
        response = requests.get(url)  
        response.raise_for_status()  
        print(f"URL: {url}, Status Code: {response.status_code}")  
    except requests.RequestException as e:  
        print(f"Error fetching {url}: {e}")  
threads = []  
for url in urls:  
    t = threading.Thread(target=fetch_url, args=(url,))  
    threads.append(t)  
    t.start()  
# 等待所有线程完成  
for t in threads:  
    t.join()

from multiprocessing import Pool  
import requests  
def fetch_url(url):  
    try:  
        response = requests.get(url)  
        response.raise_for_status()  
        return f"URL: {url}, Status Code: {response.status_code}"  
    except requests.RequestException as e:  
        return f"Error fetching {url}: {e}"  
with Pool(processes=4) as pool:  # 设定进程池的大小  
    results = pool.map(fetch_url, urls)  
for result in results:  
    print(result)

import asyncio  
import aiohttp  
async def fetch_url(url, session):  
    async with session.get(url) as response:  
        return await response.text()  
async def main():  
    async with aiohttp.ClientSession() as session:  
        tasks = []  
        for url in urls:  
            task = asyncio.create_task(fetch_url(url, session))  
            tasks.append(task)  
        results = await asyncio.gather(*tasks)  
        for result, url in zip(results, urls):  
            print(f"URL: {url}, Content: {result[:100]}...")  
# Python 3.7+ 可以使用下面的方式运行主协程  
asyncio.run(main())