python使用bs4爬取boss直聘静态页面
思路:
1、将需要查询城市列表,通过城市接口转换成相应的code码
2、遍历城市、职位生成url
3、通过url获取列表页面信息,遍历列表页面信息
4、再根据列表页面信息的job_link获取详情页面信息,将需要的信息以字典data的形式存在列表datas里
5、判断列表页面是否有下一页,重复步骤3、4;同时将列表datas一直传递下去
6、一个城市、职位url爬取完后,将列表datas接在列表datas_list后面,重复3、4、5
7、最后将列表datas_list的数据,遍历写在Excel里面
知识点:
1、将response内容以json形式输出,解析json并取值
2、soup 的select()和find_all()和find()方法使用
3、异常Exception的使用
4、wldt创建编辑Excel的使用
import requests, time, xlwt from bs4 import BeautifulSoup class MyJob(): def __init__(self, mycity, myquery): self.city = mycity self.query = myquery self.list_url = "https://www.zhipin.com/job_detail/?query=%s&city=%s&industry=&position="%(self.query, self.city) self.datas = [] self.header = { 'authority': 'www.zhipin.com', 'method': 'GET', 'scheme': 'https', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'max-age=0', 'cookie': 'lastCity=101210100;uab_collina=154408714637849548916323;toUrl=/;c=1558272251;g=-;l=l=%2Fwww.zhipin.com%2Fuser%2Flogin.html&r=; Hm_lvt_194df3105ad7148dcf2b98a91b5e727a=1555852331,1556985726,1558169427,1558272251; __a=40505844.1544087205.1558169426.1558272251.41.14.4.31; Hm_lpvt_194df3105ad7148dcf2b98a91b5e727a=1558272385', 'referer': 'https://www.zhipin.com/?ka=header-logo', 'upgrade-insecure-requests': '1', 'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' } #将城市转化为code码 def get_city(self,city_list): city_url = "https://www.zhipin.com/wapi/zpCommon/data/city.json" #获取城市 json = requests.get(city_url).json() zpData = json["zpData"]["cityList"] list = [] for city in city_list : for data_sf in zpData: for data_dq in data_sf["subLevelModelList"]: if city == data_dq["name"]: list.append(data_dq["code"]) return list #获取所有页内容 def get_job_list(self, url, datas): print(url) html = requests.get(url, headers=self.header).text soup = BeautifulSoup(html, 'html.parser') jobs = soup.select(".job-primary") for job in jobs: data = {} # 招聘id data["job_id"] = job.find_all("div", attrs={"class": "info-primary"})[0].find("a").get("data-jobid") # 招聘链接 data["job_link"] = "https://www.zhipin.com" + job.find_all("div", attrs={"class": "info-primary"})[0].find("a").get("href") # 招聘岗位 data["job_name"] = job.find_all("div", attrs={"class": "info-primary"})[0].find("div", attrs={"class": "job-title"}).get_text() # 薪资 data["job_red"] = job.find_all("div", attrs={"class": "info-primary"})[0].find("span", attrs={"class": "red"}).get_text() # 地址 #工作年限 #学历 data["job_address"] = job.find_all("div", attrs={"class": "info-primary"})[0].find("p").get_text().split(" ") # 企业链接 data["job_company_link"] = job.find_all("div", attrs={"class": "info-company"})[0].find("a").get("href") # 企业信息 data["job_company"] = job.find_all("div", attrs={"class": "info-company"})[0].find("p").get_text().split(" ") # boss链接 data["job_publis_link"] = job.find_all("div", attrs={"class": "info-publis"})[0].find("img").get("src") # boos信息 data["job_publis"] = job.find_all("div", attrs={"class": "info-publis"})[0].find("h3").get_text().split(" ") time.sleep(5) self.get_job_detail(data) # 获取job详情页内容 print(data) datas.append(data) # 将某条job添加到datas中,直到将当前页添加完 try: next_url = soup.find("div", attrs={"class": "page"}).find("a", attrs={"class": "next"}).get("href") #if next_url[-1] =="3": # 第二页自动抛异常 if next_url in "javascript:;": # 最后一页自动抛异常 raise Exception() except Exception as e: print("最后一页了;%s" % e) return datas # 返回所有页内容 else: time.sleep(5) next_url = "https://www.zhipin.com" + next_url self.get_job_list(next_url, datas) return datas # 返回所有页内容 #获取详情页内容 def get_job_detail(self, data): print(data["job_link"]) html = requests.get(data["job_link"], headers=self.header).text soup = BeautifulSoup(html, 'html.parser') # 招聘公司 data["detail_content_name"] = soup.find_all("div", attrs={"class": "detail-content"})[0].find("div", attrs={"class": "name"}).get_text() # 福利 data["detail_primary_tags"] = soup.find_all("div", attrs={"class": "info-primary"})[0].find("div", attrs={"class": "job-tags"}).get_text().strip() # 招聘岗位 data["detail_primary_name"] = soup.find_all("div", attrs={"class": "info-primary"})[0].find("h1").get_text() # 招聘状态 data["detail_primary_status"] = soup.find_all("div", attrs={"class": "info-primary"})[0].find("div", attrs={"class": "job-status"}).get_text() # 薪资 data["detail_primary_salary"] = soup.find_all("div", attrs={"class": "info-primary"})[0].find("span", attrs={"class": "salary"}).get_text() # 地址 #工作年限 #学历 data["detail_primary_address"] = soup.find_all("div", attrs={"class": "info-primary"})[0].find("p").get_text() # 工作地址 data["detail_content_address"] = soup.find_all("div", attrs={"class": "detail-content"})[0].find("div", attrs={"class": "location-address"}).get_text() # 职位描述 data["detail_content_text"] = soup.find_all("div", attrs={"class": "detail-content"})[0].find("div", attrs={"class": "text"}).get_text().strip().replace(";", "\n") # boss名字 data["detail_op_name"] = soup.find_all("div", attrs={"class": "detail-op"})[1].find("h2", attrs={"class": "name"}).get_text() # boss职位 data["detail_op_job"] = soup.find_all("div", attrs={"class": "detail-op"})[1].find("p", attrs={"class": "gray"}).get_text().split("·")[0] # boss状态 data["detail_op_status"] = soup.find_all("div", attrs={"class": "detail-op"})[1].find("p", attrs={"class": "gray"}).get_text().split("·")[1] #将获取的数据写入Excel def setExcel(self, datas_list): book = xlwt.Workbook(encoding='utf-8') table = book.add_sheet("boss软件测试") table.write(0, 0, "编号") table.write(0, 1, "招聘链接") table.write(0, 2, "招聘岗位") table.write(0, 3, "薪资") table.write(0, 4, "地址") table.write(0, 5, "企业链接") table.write(0, 6, "企业信息") table.write(0, 7, "boss链接") table.write(0, 8, "boss信息") table.write(0, 9, "detail详情") i = 1 for data in datas_list: table.write(i, 0, data["job_id"]) table.write(i, 1, data["job_link"]) table.write(i, 2, data["job_name"]) table.write(i, 3, data["job_red"]) table.write(i, 4, data["job_address"]) table.write(i, 5, data["job_company_link"]) table.write(i, 6, data["job_company"]) table.write(i, 7, data["job_publis_link"]) table.write(i, 8, data["job_publis"]) table.write(i, 10, data["detail_content_name"]) table.write(i, 11, data["detail_primary_name"]) table.write(i, 12, data["detail_primary_status"]) table.write(i, 13, data["detail_primary_salary"]) table.write(i, 14, data["detail_primary_address"]) table.write(i, 15, data["detail_content_text"]) table.write(i, 16, data["detail_op_name"]) table.write(i, 17, data["detail_op_job"]) table.write(i, 18, data["detail_op_status"]) table.write(i, 19, data["detail_primary_tags"]) table.write(i, 20, data["detail_content_address"]) i += 1 book.save(r'C:\%s_boss软件测试.xls' % time.strftime('%Y%m%d%H%M%S')) print("Excel保存成功") if __name__ == '__main__': city_list = MyJob("","").get_city(["杭州"]) query_list = ["软件测试", "测试工程师"] datas_list = [] for city in city_list: for query in query_list: myjob = MyJob(city, query) datas = myjob.get_job_list(myjob.list_url, myjob.datas) datas_list.extend(datas) myjob.setExcel(datas_list)
以上就是python使用bs4爬取boss直聘静态页面的详细内容,更多关于python 爬取boss直聘的资料请关注脚本之家其它相关文章!
相关文章
pyqt5 QScrollArea设置在自定义侧(任何位置)
这篇文章主要介绍了pyqt5 QScrollArea设置在自定义侧(任何位置),文中通过示例代码介绍的非常详细,对大家的学习或者工作具有一定的参考学习价值,需要的朋友们下面随着小编来一起学习学习吧2019-09-09Python Flask中Cookie和Session区别详解
Flask是一个使用 Python 编写的轻量级 Web 应用框架。其 WSGI 工具箱采用 Werkzeug ,模板引擎则使用 Jinja2 。Flask使用 BSD 授权。Flask也被称为 “microframework” ,因为它使用简单的核心,用 extension 增加其他功能,Flask中Cookie和Session有什么区别呢2022-07-07
最新评论