1 package us.codecraft.webmagic.processor.example;
2
3 import us.codecraft.webmagic.Page;
4 import us.codecraft.webmagic.Site;
5 import us.codecraft.webmagic.Spider;
6 import us.codecraft.webmagic.processor.PageProcessor;
7
8
9
10
11
12 public class ZhihuPageProcessor implements PageProcessor {
13
14 private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
15
16 @Override
17 public void process(Page page) {
18 page.addTargetRequests(page.getHtml().links().regex("https://www\\.zhihu\\.com/question/\\d+/answer/\\d+.*").all());
19 page.putField("title", page.getHtml().xpath("//h1[@class='QuestionHeader-title']/text()").toString());
20 page.putField("question", page.getHtml().xpath("//div[@class='QuestionRichText']//tidyText()").toString());
21 page.putField("answer", page.getHtml().xpath("//div[@class='QuestionAnswer-content']/tidyText()").toString());
22 if (page.getResultItems().get("title")==null){
23
24 page.setSkip(true);
25 }
26 }
27
28 @Override
29 public Site getSite() {
30 return site;
31 }
32
33 public static void main(String[] args) {
34 Spider.create(new ZhihuPageProcessor()).addUrl("https://www.zhihu.com/explore").run();
35 }
36 }