Java 网络爬虫新手入门详解

<dependency>
 <groupId>org.jsoup</groupId>
 <artifactId>jsoup</artifactId>
 <version>1.12.1</version>
</dependency>

/**
 * jsoup方式 获取虎扑新闻列表页
 * @param url 虎扑新闻列表页url
 */
public void jsoupList(String url){
 try {
  Document document = Jsoup.connect(url).get();
  // 使用 css选择器 提取列表新闻 a 标签
  // <a href="https://voice.hupu.com/nba/2484553.html" rel="external nofollow" target="_blank">霍华德：夏休期内曾节食30天，这考验了我的身心</a>
  Elements elements = document.select("div.news-list > ul > li > div.list-hd > h4 > a");
  for (Element element:elements){
//    System.out.println(element);
   // 获取详情页链接
   String d_url = element.attr("href");
   // 获取标题
   String title = element.ownText();

   System.out.println("详情页链接："+d_url+" ,详情页标题："+title);

  }
 } catch (IOException e) {
  e.printStackTrace();
 }
}

public static void main(String[] args) {
 String url = "https://voice.hupu.com/nba";
 CrawlerBase crawlerBase = new CrawlerBase();
 crawlerBase.jsoupList(url);
}

<dependency>
 <groupId>org.apache.httpcomponents</groupId>
 <artifactId>httpclient</artifactId>
 <version>4.5.10</version>
</dependency>
<dependency>
 <groupId>org.apache.httpcomponents</groupId>
 <artifactId>httpcore</artifactId>
 <version>4.4.10</version>
</dependency>
<dependency>
 <groupId>org.apache.httpcomponents</groupId>
 <artifactId>httpmime</artifactId>
 <version>4.5.10</version>
</dependency>

/**
 * httpclient + 正则表达式 获取虎扑新闻列表页
 * @param url 虎扑新闻列表页url
 */
public void httpClientList(String url){
 try {
  CloseableHttpClient httpclient = HttpClients.createDefault();
  HttpGet httpGet = new HttpGet(url);
  CloseableHttpResponse response = httpclient.execute(httpGet);
  if (response.getStatusLine().getStatusCode() == 200) {
   HttpEntity entity = response.getEntity();
   String body = EntityUtils.toString(entity,"utf-8");
 
   if (body!=null) {
     /*
     * 替换掉换行符、制表符、回车符，去掉这些符号，正则表示写起来更简单一些
     * 只有空格符号和其他正常字体
     */
    Pattern p = Pattern.compile("\t|\r|\n");
    Matcher m = p.matcher(body);
    body = m.replaceAll("");
    /*
     * 提取列表页的正则表达式
     * 去除换行符之后的 li
     * <div class="list-hd">         <h4>          <a href="https://voice.hupu.com/nba/2485167.html" rel="external nofollow" target="_blank">与球迷亲切互动！凯尔特人官方晒球队开放训练日照片</a>         </h4>        </div>
     */
    Pattern pattern = Pattern
      .compile("<div class=\"list-hd\">\\s* <h4>\\s* <a href=\"(.*?)\"\\s* target=\"_blank\">(.*?)</a>\\s* </h4>\\s* </div>" );

    Matcher matcher = pattern.matcher(body);
    // 匹配出所有符合正则表达式的数据
    while (matcher.find()){
//      String info = matcher.group(0);
//      System.out.println(info);
     // 提取出链接和标题
     System.out.println("详情页链接："+matcher.group(1)+" ,详情页标题："+matcher.group(2));
    }
   }else {
    System.out.println("处理失败！！！获取正文内容为空");
   }
  } else {
   System.out.println("处理失败！！！返回状态码：" + response.getStatusLine().getStatusCode());
  }
 }catch (Exception e){
  e.printStackTrace();
 }

}

public static void main(String[] args) {
 String url = "https://voice.hupu.com/nba";
 CrawlerBase crawlerBase = new CrawlerBase();
//  crawlerBase.jsoupList(url);
 crawlerBase.httpClientList(url);
}