Python3获取拉勾网招聘信息的方法实例

'''
@author: Max_Lyu
Create time: 2019/4/1
url: https://github.com/MaxLyu/Lagou_Analyze
'''
 # 请求起始 url 返回 cookies
 def get_start_url(self):
 session = requests.session()
 session.get(self.start_url, headers=self.headers, timeout=3)
 cookies = session.cookies
 return cookies

 # 将返回的 cookies 一起 post 给 target_url 并获取数据
 def post_target_url(self):
 cookies = self.get_start_url()
 pn = 1
 for pg in range(30):
  formdata = {
  'first': 'false',
  'pn': pn,
  'kd': 'python数据分析'
  }
  pn += 1

  response = requests.post(self.target_url, data=formdata, cookies=cookies, headers=self.headers, timeout=3)
  self.parse(response)
  time.sleep(60) # 拉勾的反扒技术比较强，短睡眠时间会被封

 # 解析 response，获取 items
 def parse(self, response):
 print(response)
 items = []
 print(response.text)
 data = json.loads(response.text)['content']['positionResult']['result']

 if len(data):
  for i in range(len(data)):
  positionId = data[i]['positionId']
  education = data[i]['education']
  workYear = data[i]['workYear']
  salary = data[i]['salary']
  list = [positionId, education, workYear, salary]
  items.append(list)
 self.save_data(items)
 time.sleep(1.3)

'''
@author: Max_Lyu
Create time: 2019/4/1
url: https://github.com/MaxLyu/Lagou_Analyze
'''
def get_url():
 urls = []
 with open("analyst.csv", 'r', newline='') as file:
 # 读取文件
 reader = csv.reader(file)
 for row in reader:
  # 根据 positionID 补全 url
  if row[0] != "ID":
  url = "https://www.lagou.com/jobs/{}.html".format(row[0])
  urls.append(url)

 file.close()
 return urls

# 获取详细信息
def get_info():
 urls = get_url()
 length = len(urls)
 for url in urls:
 print(url)
 description = ''
 print(length)
 response = requests.get(url, headers=headers)
 response.encoding = 'utf-8'
 content = etree.HTML(response.text)
 detail = content.xpath('//*[@id="job_detail"]/dd[2]/div/p/text()')
 print(detail)

 for i in range(1, len(detail)):

  if '要求' in detail[i-1]:
  for j in range(i, len(detail)):
   detail[j] = detail[j].replace('\xa0', '')
   detail[j] = re.sub('[、;；.0-9。]', '', detail[j])
   description = description + detail[j] + '/'
  print(description)
 write_file(description)
 length -= 1
 time.sleep(3)