python3之Splash的具体使用

#安装所需要的包：
yum install -y yum-utils device-mapper-persistent-data lvm2
#设置稳定存储库：
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
#开始安装DOCKER CE：
yum install docker-ce
#启动dockers：
systemctl start docker
#测试安装是否正确：
docker run hello-world

docker pull scrapinghub/splash
docker run -d -p 8050:8050 scrapinghub/splash
#通过浏览器访问8050端口验证安装是否成功

pip3 install scrapy-splash

function main(splash, args)
  assert(splash:go(args.url))
  assert(splash:wait(0.5))
  return {
    html = splash:html(),
    png = splash:png(),
    har = splash:har(),
  }
end

function main(splash, args)
assert(splash:go("https://www.baidu.com"))
assert(splash:wait(0.5))
local title = splash:evaljs("document.title")  
return {
title = title
}
end

#返回结果：
Splash Response: Object

title: "百度一下，你就知道"

function main(splash)
  return {
    hello="world"
  }
end

#返回结果
Splash Response: Object
hello: "world"


function main(splash)
  return "world"
end

#返回结果
Splash Response: "world"

function main(splash,args)
    local url = args.url
end

#上面的第二个参数args就相当于splash.args属性，如下代码与上面是等价的

function main(splash)
   local url=splash.args.url
end

function main(splash, args)
  assert(splash:go('https://www.toutiao.com'))
  splash.scroll_position={y=400}
  return {
    png = splash:png()
  }
end

#它会向下滚动400像素来获取图片

ok, reason = splash:go{url, baseurl=nil, headers=nil, http_method="GET", body=nil, formdata=nil}

function main(splash, args)
  local ok, reason = splash:go{"http://httpbin.org/post", http_method="POST", body="name=Germey"}
  if ok then
        return splash:html()
  end
end

function main(splash, args)
  splash:go("https://www.toutiao.com")
  local ok reason = splash:wait(1)
  return {
    ok=ok,
    reason=reason
  }
end

#返回true说明返回页面成功

function main(splash, args)
  local get_div_count = splash:jsfunc([[
  function () {
    var body = document.body;
    var divs = body.getElementsByTagName('div');
    return divs.length;
  }
  ]])
  splash:go("https://www.baidu.com")
  return ("There are %s DIVs"):format(
    get_div_count())
end

#
Splash Response: "There are 21 DIVs"

local title = splash:evaljs("document.title")

#返回页面标题

function main(splash, args)
  splash:go("https://www.baidu.com")
  splash:runjs("foo = function() { return 'bar' }")
  local result = splash:evaljs("foo()")
  return result
end

function main(splash, args)
  splash:autoload([[
    function get_document_title(){
      return document.title;
    }
  ]])
  splash:go("https://www.baidu.com")
  return splash:evaljs("get_document_title()")
end


#加载JS库文件
function main(splash, args)
  assert(splash:autoload("https://code.jquery.com/jquery-2.1.3.min.js"))
  assert(splash:go("https://www.taobao.com"))
  local version = splash:evaljs("$.fn.jquery")
  return 'JQuery version: ' .. version
end

function main(splash, args)
  local snapshots = {}
  local timer = splash:call_later(function()
    snapshots["a"] = splash:png()
    splash.scroll_position={y=500}
    splash:wait(1.0)
    snapshots["b"] = splash:png()
  end, 2)
  splash:go("https://www.toutiao.com")
  splash:wait(3.0)
  return snapshots
end

#等待2秒后执行截图然后再等待3秒后执行截图

local reply = splash:http_get("http://example.com")

#返回一个响应对象，不会讲结果返回到浏览器

function main(splash, args)
  local treat = require("treat")
  local json = require("json")
  local response = splash:http_post{"http://httpbin.org/post",     
      body=json.encode({name="Germey"}),
      headers={["content-type"]="application/json"}
    }
    return {
    html=treat.as_string(response.body),
    url=response.url,
    status=response.status
    }
end

#
html:{"args":{},"data":"{\"name\": \"Germey\"}","files":{},"form":{},"headers":{"Accept-Encoding":"gzip, deflate","Accept-Language":"en,*","Connection":"close","Content-Length":"18","Content-Type":"application/json","Host":"httpbin.org","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/9.0 Safari/602.1"},"json":{"name":"Germey"},"origin":"221.218.181.223","url":"http://httpbin.org/post"}
status: 200
url: http://httpbin.org/post

function main(splash)
    assert(splash:set_content("<html><body><h1>hello</h1></body></html>"))
    return splash:png()
end

function main(splash, args)
  splash:go("https://httpbin.org/get")
  return splash:html()
end

function main(splash)
    splash:add_cookie{"sessionid", "237465ghgfsd", "/", domain="http://example.com"}
    splash:go("http://example.com/")
    return splash:get_cookies()
end

#
Splash Response: Array[1]
0: Object
domain: "http://example.com"
httpOnly: false
name: "sessionid"
path: "/"
secure: false
value: "237465ghgfsd"

function main(splash)
  splash:set_custom_headers({
     ["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
     ["Site"] = "httpbin.org",
  })
  splash:go("http://httpbin.org/get")
  return splash:html()
end

function main(splash)
  splash:go("https://www.baidu.com/")
  input = splash:select("#kw")
  input:send_text('Splash')
  splash:wait(3)
  return splash:png()
end

function main(splash)
  local treat = require('treat')
  assert(splash:go("https://www.zhihu.com"))
  assert(splash:wait(1))
  local texts = splash:select_all('.ContentLayout-mainColumn .ContentItem-title')
  local results = {}
  for index, text in ipairs(texts) do
    results[index] = text.node.textContent
  end
  return treat.as_array(results)
end

#返回所有节点下的文本内容

function main(splash)
  splash:go("https://www.baidu.com/")
  input = splash:select("#kw")
  input:send_text('Splash')
  submit = splash:select('#su')
  submit:mouse_click()
  splash:wait(3)
  return splash:png()
end

import requests
url='http://172.16.32.136:8050/'
response=requests.get(url+'render.html?url=https://www.baidu.com&wait=3&images=0')
print(response.text)  #返回网页源代码

import requests
url='http://172.16.32.136:8050/'
#指定图像宽和高
response=requests.get(url+'render.png?url=https://www.taobao.com&wait=5&width=1000&height=700&render_all=1')
with open('taobao.png','wb') as f:
    f.write(response.content)

import requests
url='http://172.16.32.136:8050/'

response=requests.get(url+'render.jpeg?url=https://www.taobao.com&wait=5&width=1000&height=700&render_all=1')
with open('taobao.jpeg','wb') as f:
    f.write(response.content)

import requests
url='http://172.16.32.136:8050/'
response=requests.get(url+'render.har?url=https://www.jd.com&wait=5')

print(response.text)

import requests
url='http://172.16.32.136:8050/'
response=requests.get(url+'render.json?url=https://httpbin.org&html=1&png=1&history=1&har=1')

print(response.text)

import requests
from urllib.parse import quote
lua='''
function main(splash)
    return 'hello'
end
'''
url='http://172.16.32.136:8050/execute?lua_source='+quote(lua)
response=requests.get(url)
print(response.text)

import requests
from urllib.parse import quote
lua='''
function main(splash,args)
    local treat=require("treat")
    local response=splash:http_get("http://httpbin.org/get")
    return {
        html=treat.as_string(response.body),
        url=response.url,
        status=response.status
    }
end
'''
url='http://172.16.32.136:8050/execute?lua_source='+quote(lua)
response=requests.get(url)
print(response.text)

#
{"status": 200, "html": "{\"args\":{},\"headers\":{\"Accept-Encoding\":\"gzip, deflate\",\"Accept-Language\":\"en,*\",\"Connection\":\"close\",\"Host\":\"httpbin.org\",\"User-Agent\":\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/9.0 Safari/602.1\"},\"origin\":\"221.218.181.223\",\"url\":\"http://httpbin.org/get\"}\n", "url": http://httpbin.org/get}

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/9 13:33
# @Author  : Py.qi
# @File    : JD.py
# @Software: PyCharm
import re

import requests
import pymongo
from pyquery import PyQuery as pq

client=pymongo.MongoClient('localhost',port=27017)
db=client['JD']

def page_parse(html):
    doc=pq(html,parser='html')
    items=doc('#J_goodsList .gl-item').items()
    for item in items:
        if item('.p-img img').attr('src'):
            image=item('.p-img img').attr('src')
        else:
            image=item('.p-img img').attr('data-lazy-img')
        texts={
            'image':'https:'+image,
            'price':item('.p-price').text()[:6],
            'title':re.sub('\n','',item('.p-name').text()),
            'commit':item('.p-commit').text()[:-3],

        }
        yield texts

def save_to_mongo(data):
    if db['jd_collection'].insert(data):
        print('保存到MongoDB成功',data)
    else:
        print('MongoDB存储错误',data)

def main(number):
    url='http://192.168.146.140:8050/render.html?url=https://search.jd.com/Search?keyword=python&page={}&wait=1&images=0'.format(number)
    response=requests.get(url)
    data=page_parse(response.text)
    for i in data:
        save_to_mongo(i)
        #print(i)

if __name__ == '__main__':
    for number in range(1,200,2):
        print('开始抓取第{}页'.format(number))
        main(number)