1 package us.codecraft.webmagic.samples;
2
3 import us.codecraft.webmagic.Page;
4 import us.codecraft.webmagic.Site;
5 import us.codecraft.webmagic.Spider;
6 import us.codecraft.webmagic.processor.PageProcessor;
7 import us.codecraft.webmagic.scheduler.PriorityScheduler;
8
9
10
11
12 public class AlexanderMcqueenGoodsProcessor implements PageProcessor {
13
14 private Site site = Site.me().setRetryTimes(3).setSleepTime(0);
15
16
17 public static final String URL_LIST = "http://www\\.alexandermcqueen\\.cn/.*";
18
19 public static final String URL_POST = "http://www\\.alexandermcqueen\\.cn/cn/\\w+/.*\\.html";
20
21 @Override
22 public void process(Page page) {
23 if (page.getUrl().regex(URL_POST).match()) {
24 page.putField("goodsName", page.getHtml().xpath("//div[@id='description']/h1/tidyText()"));
25 if (page.getResultItems().get("goodsName") == null) {
26 page.setSkip(true);
27 }
28 page.putField("currency", page.getHtml().xpath("//div[@id='description']//div[@class='itemBoxPrice']/span//span[@class='currency']/tidyText()"));
29 page.putField("goodsPrice", page.getHtml().xpath("//div[@id='description']//div[@class='itemBoxPrice']/span//span[@class='priceValue']/tidyText()"));
30 page.putField("description", page.getHtml()
31 .xpath("//div[@id='tabbedDescription']//div[@class='tabbedDescription']//ul[@id='tabs']//li[@id='tab_description']/div[@id='description_pane']/tidyText()"));
32 page.putField("material", page.getHtml()
33 .xpath("//div[@id='tabbedDescription']" +
34 "//div[@class='tabbedDescription']" +
35 "//ul[@id='tabs']" +
36 "//li[@id='tab_description']" +
37 "//div[@class='productProperty']" +
38 "//div[@class='productPropertyRow']/span[2]/tidyText()"));
39 page.putField("goodsCode", page.getHtml()
40 .xpath("//div[@id='tabbedDescription']" +
41 "//div[@class='tabbedDescription']" +
42 "//ul[@id='tabs']" +
43 "//li[@id='tab_description']" +
44 "//div[@class='productProperty']" +
45 "//div[@class='productPropertyRow']//span[@id='modelFabricColorContainer']/tidyText()"));
46 page.putField("goodsSize", page.getHtml()
47 .xpath("//div[@id='sizesContainer']//div[@id='sizes']//ul[@class='SizeW']"));
48 page.putField("goodsColors", page.getHtml()
49 .xpath("//div[@id='colors']/ul/html()"));
50 } else {
51 page.addTargetRequests(page.getHtml().links().regex(URL_POST).all(), 1000);
52 page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all(), 1);
53 }
54 }
55
56 @Override
57 public Site getSite() {
58 return site;
59 }
60
61 public static void main(String[] args) {
62 Spider.create(new AlexanderMcqueenGoodsProcessor()).setScheduler(new PriorityScheduler())
63 .addUrl("http://www.alexandermcqueen.cn/sitemap.asp?tskay=E2F1A848").thread(5).run();
64 }
65 }