View Javadoc
1   package us.codecraft.webmagic.scheduler;
2   
3   import java.util.concurrent.BlockingQueue;
4   import java.util.concurrent.LinkedBlockingQueue;
5   
6   import us.codecraft.webmagic.Request;
7   import us.codecraft.webmagic.Site;
8   import us.codecraft.webmagic.Task;
9   
10  /**
11   * Basic Scheduler implementation.<br>
12   * Store urls to fetch in LinkedBlockingQueue and remove duplicate urls by HashMap.
13   *
14   * Note: if you use this {@link QueueScheduler}
15   * with {@link Site#getCycleRetryTimes()} enabled, you may encountered dead-lock
16   * when the queue is full.
17   *
18   * @author code4crafter@gmail.com <br>
19   * @since 0.1.0
20   */
21  public class QueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {
22  
23      private final BlockingQueue<Request> queue;
24  
25      public QueueScheduler() {
26          this.queue = new LinkedBlockingQueue<>();
27      }
28  
29      /**
30       * Creates a {@code QueueScheduler} with the given (fixed) capacity.
31       *
32       * @param capacity the capacity of this queue,
33       * see {@link LinkedBlockingQueue#LinkedBlockingQueue(int)}
34       * @since 0.8.0
35       */
36      public QueueScheduler(int capacity) {
37          this.queue = new LinkedBlockingQueue<>(capacity);
38      }
39  
40      @Override
41      public void pushWhenNoDuplicate(Request request, Task task) {
42          logger.trace("Remaining capacity: {}", this.queue.remainingCapacity());
43  
44          try {
45              queue.put(request);
46          } catch (InterruptedException e) {
47              Thread.currentThread().interrupt();
48          }
49      }
50  
51      @Override
52      public Request poll(Task task) {
53          return queue.poll();
54      }
55  
56      @Override
57      public int getLeftRequestsCount(Task task) {
58          return queue.size();
59      }
60  
61      @Override
62      public int getTotalRequestsCount(Task task) {
63          return getDuplicateRemover().getTotalRequestsCount(task);
64      }
65  }