1 package us.codecraft.webmagic.monitor;
2
3 import java.lang.management.ManagementFactory;
4 import java.util.ArrayList;
5 import java.util.Collections;
6 import java.util.List;
7 import java.util.concurrent.atomic.AtomicInteger;
8
9 import javax.management.InstanceAlreadyExistsException;
10 import javax.management.JMException;
11 import javax.management.MBeanRegistrationException;
12 import javax.management.MBeanServer;
13 import javax.management.MalformedObjectNameException;
14 import javax.management.NotCompliantMBeanException;
15 import javax.management.ObjectName;
16
17 import us.codecraft.webmagic.Request;
18 import us.codecraft.webmagic.Spider;
19 import us.codecraft.webmagic.SpiderListener;
20 import us.codecraft.webmagic.utils.Experimental;
21 import us.codecraft.webmagic.utils.UrlUtils;
22
23
24
25
26
27 @Experimental
28 public class SpiderMonitor {
29
30 private static final SpiderMonitor INSTANCE = new SpiderMonitor();
31
32 private MBeanServer mbeanServer;
33
34 private String jmxServerName;
35
36 private List<SpiderStatusMXBean> spiderStatuses = new ArrayList<>();
37
38 protected SpiderMonitor() {
39 jmxServerName = "WebMagic";
40 mbeanServer = ManagementFactory.getPlatformMBeanServer();
41 }
42
43
44
45
46
47
48
49
50 public synchronized SpiderMonitor register(Spider... spiders) throws JMException {
51 for (Spider spider : spiders) {
52 MonitorSpiderListener monitorSpiderListener = new MonitorSpiderListener();
53 if (spider.getSpiderListeners() == null) {
54 List<SpiderListener> spiderListeners = new ArrayList<>();
55 spiderListeners.add(monitorSpiderListener);
56 spider.setSpiderListeners(spiderListeners);
57 } else {
58 spider.getSpiderListeners().add(monitorSpiderListener);
59 }
60 SpiderStatusMXBean spiderStatusMBean = getSpiderStatusMBean(spider, monitorSpiderListener);
61 registerMBean(spiderStatusMBean);
62 spiderStatuses.add(spiderStatusMBean);
63 }
64 return this;
65 }
66
67 protected SpiderStatusMXBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) {
68 return new SpiderStatus(spider, monitorSpiderListener);
69 }
70
71 protected List<SpiderStatusMXBean> getSpiderStatuses() {
72 return this.spiderStatuses;
73 }
74
75 public static SpiderMonitor instance() {
76 return INSTANCE;
77 }
78
79 public class MonitorSpiderListener implements SpiderListener {
80
81 private final AtomicInteger successCount = new AtomicInteger(0);
82
83 private final AtomicInteger errorCount = new AtomicInteger(0);
84
85 private List<String> errorUrls = Collections.synchronizedList(new ArrayList<String>());
86
87 @Override
88 public void onSuccess(Request request) {
89 successCount.incrementAndGet();
90 }
91
92 @Override
93 public void onError(Request request, Exception e) {
94 errorUrls.add(request.getUrl());
95 errorCount.incrementAndGet();
96 }
97
98 public AtomicInteger getSuccessCount() {
99 return successCount;
100 }
101
102 public AtomicInteger getErrorCount() {
103 return errorCount;
104 }
105
106 public List<String> getErrorUrls() {
107 return errorUrls;
108 }
109 }
110
111 protected void registerMBean(SpiderStatusMXBean spiderStatus) throws MalformedObjectNameException, InstanceAlreadyExistsException, MBeanRegistrationException, NotCompliantMBeanException {
112 ObjectName objName = new ObjectName(jmxServerName + ":name=" + UrlUtils.removePort(spiderStatus.getName()));
113 mbeanServer.registerMBean(spiderStatus, objName);
114 }
115
116 }