View Javadoc
1   package us.codecraft.webmagic;
2   
3   import java.util.concurrent.TimeUnit;
4   import java.util.concurrent.locks.Condition;
5   import java.util.concurrent.locks.ReentrantLock;
6   
7   import us.codecraft.webmagic.scheduler.Scheduler;
8   import us.codecraft.webmagic.thread.CountableThreadPool;
9   
10  public class SpiderScheduler {
11      private Scheduler scheduler;
12      private final ReentrantLock newUrlLock = new ReentrantLock();
13      private final Condition newUrlCondition = newUrlLock.newCondition();
14  
15      public SpiderScheduler(Scheduler scheduler) {
16          this.scheduler = scheduler;
17      }
18  
19      public Scheduler getScheduler() {
20          return scheduler;
21      }
22  
23      public void setScheduler(Scheduler scheduler) {
24          this.scheduler = scheduler;
25      }
26  
27      public Request poll(Spider spider) {
28          return scheduler.poll(spider);
29      }
30  
31      public void push(Request request, Spider spider) {
32          scheduler.push(request, spider);
33      }
34  
35      public boolean waitNewUrl(CountableThreadPool threadPool, long emptySleepTime) {
36          newUrlLock.lock();
37          try {
38              if (threadPool.getThreadAlive() == 0) {
39                  return false;
40              }
41              newUrlCondition.await(emptySleepTime, TimeUnit.MILLISECONDS);
42              return false;
43          } catch (InterruptedException e) {
44              return true;
45          } finally {
46              newUrlLock.unlock();
47          }
48      }
49  
50      public void signalNewUrl() {
51          try {
52              newUrlLock.lock();
53              newUrlCondition.signalAll();
54          } finally {
55              newUrlLock.unlock();
56          }
57      }
58  
59  }