View Javadoc
1   package us.codecraft.webmagic;
2   
3   import java.io.Serializable;
4   import java.util.Collections;
5   import java.util.HashMap;
6   import java.util.Map;
7   
8   import us.codecraft.webmagic.downloader.Downloader;
9   import us.codecraft.webmagic.model.HttpRequestBody;
10  import us.codecraft.webmagic.utils.Experimental;
11  
12  /**
13   * Object contains url to crawl.<br>
14   * It contains some additional information.<br>
15   *
16   * @author code4crafter@gmail.com <br>
17   * @since 0.1.0
18   */
19  public class Request implements Serializable {
20  
21      private static final long serialVersionUID = 2062192774891352043L;
22  
23      public static final String CYCLE_TRIED_TIMES = "_cycle_tried_times";
24  
25      private String url;
26  
27      private String method;
28  
29      private HttpRequestBody requestBody;
30  
31      /**
32       * this req use this downloader
33       */
34      private Downloader downloader;
35  
36      /**
37       * Store additional information in extras.
38       */
39      private Map<String, Object> extras = new HashMap<>();
40  
41      /**
42       * cookies for current url, if not set use Site's cookies
43       */
44      private Map<String, String> cookies = new HashMap<String, String>();
45  
46      private Map<String, String> headers = new HashMap<String, String>();
47  
48      /**
49       * Priority of the request.<br>
50       * The bigger will be processed earlier. <br>
51       * @see us.codecraft.webmagic.scheduler.PriorityScheduler
52       */
53      private long priority;
54  
55      /**
56       * When it is set to TRUE, the downloader will not try to parse response body to text.
57       *
58       */
59      private boolean binaryContent = false;
60  
61      private String charset;
62  
63      public Request() {
64      }
65  
66      public Request(String url) {
67          this.url = url;
68      }
69  
70      public long getPriority() {
71          return priority;
72      }
73  
74      /**
75       * Set the priority of request for sorting.<br>
76       * Need a scheduler supporting priority.<br>
77       * @see us.codecraft.webmagic.scheduler.PriorityScheduler
78       *
79       * @param priority priority
80       * @return this
81       */
82      @Experimental
83      public Request setPriority(long priority) {
84          this.priority = priority;
85          return this;
86      }
87  
88      @SuppressWarnings("unchecked")
89      public <T> T getExtra(String key) {
90          if (extras == null) {
91              return null;
92          }
93          return (T) extras.get(key);
94      }
95  
96      public <T> Request putExtra(String key, T value) {
97          extras.put(key, value);
98          return this;
99      }
100 
101     public String getUrl() {
102         return url;
103     }
104 
105     public Map<String, Object> getExtras() {
106         return Collections.unmodifiableMap(extras);
107     }
108 
109     public Request setExtras(Map<String, Object> extras) {
110         this.extras.putAll(extras);
111         return this;
112     }
113 
114     public Request setUrl(String url) {
115         this.url = url;
116         return this;
117     }
118 
119     /**
120      * The http method of the request. Get for default.
121      * @return httpMethod
122      * @see us.codecraft.webmagic.utils.HttpConstant.Method
123      * @since 0.5.0
124      */
125     public String getMethod() {
126         return method;
127     }
128 
129     public Request setMethod(String method) {
130         this.method = method;
131         return this;
132     }
133 
134     @Override
135     public int hashCode() {
136         int result = url != null ? url.hashCode() : 0;
137         result = 31 * result + (method != null ? method.hashCode() : 0);
138         return result;
139     }
140 
141     @Override
142     public boolean equals(Object o) {
143         if (this == o) return true;
144         if (o == null || getClass() != o.getClass()) return false;
145 
146         Request request = (Request) o;
147 
148         if (url != null ? !url.equals(request.url) : request.url != null) return false;
149         return method != null ? method.equals(request.method) : request.method == null;
150     }
151 
152     public Request addCookie(String name, String value) {
153         cookies.put(name, value);
154         return this;
155     }
156 
157     public Request addHeader(String name, String value) {
158         headers.put(name, value);
159         return this;
160     }
161 
162     public Map<String, String> getCookies() {
163         return cookies;
164     }
165 
166     public Map<String, String> getHeaders() {
167         return headers;
168     }
169 
170     public HttpRequestBody getRequestBody() {
171         return requestBody;
172     }
173 
174     public void setRequestBody(HttpRequestBody requestBody) {
175         this.requestBody = requestBody;
176     }
177 
178     public boolean isBinaryContent() {
179         return binaryContent;
180     }
181 
182     public Downloader getDownloader() {
183         return downloader;
184     }
185 
186     public void setDownloader(Downloader downloader) {
187         this.downloader = downloader;
188     }
189 
190     public Request setBinaryContent(boolean binaryContent) {
191         this.binaryContent = binaryContent;
192         return this;
193     }
194 
195     public String getCharset() {
196         return charset;
197     }
198 
199     public Request setCharset(String charset) {
200         this.charset = charset;
201         return this;
202     }
203 
204     @Override
205     public String toString() {
206         return "Request{" +
207                 "url='" + url + '\'' +
208                 ", method='" + method + '\'' +
209                 ", extras=" + extras +
210                 ", priority=" + priority +
211                 ", headers=" + headers +
212                 ", cookies="+ cookies+
213                 '}';
214     }
215 
216 }