1 package us.codecraft.webmagic.downloader;
2
3 import org.apache.http.HttpHost;
4 import org.apache.http.auth.AuthState;
5 import org.apache.http.auth.ChallengeState;
6 import org.apache.http.auth.UsernamePasswordCredentials;
7 import org.apache.http.client.CookieStore;
8 import org.apache.http.client.config.CookieSpecs;
9 import org.apache.http.client.config.RequestConfig;
10 import org.apache.http.client.methods.HttpUriRequest;
11 import org.apache.http.client.methods.RequestBuilder;
12 import org.apache.http.client.protocol.HttpClientContext;
13 import org.apache.http.entity.ByteArrayEntity;
14 import org.apache.http.impl.auth.BasicScheme;
15 import org.apache.http.impl.client.BasicCookieStore;
16 import org.apache.http.impl.cookie.BasicClientCookie;
17 import us.codecraft.webmagic.Request;
18 import us.codecraft.webmagic.Site;
19 import us.codecraft.webmagic.proxy.Proxy;
20 import us.codecraft.webmagic.utils.HttpConstant;
21 import us.codecraft.webmagic.utils.UrlUtils;
22
23 import java.util.Map;
24
25
26
27
28
29
30
31
32 public class HttpUriRequestConverter {
33
34 public HttpClientRequestContext convert(Request request, Site site, Proxy proxy) {
35 HttpClientRequestContext httpClientRequestContext = new HttpClientRequestContext();
36 httpClientRequestContext.setHttpUriRequest(convertHttpUriRequest(request, site, proxy));
37 httpClientRequestContext.setHttpClientContext(convertHttpClientContext(request, site, proxy));
38 return httpClientRequestContext;
39 }
40
41 private HttpClientContext convertHttpClientContext(Request request, Site site, Proxy proxy) {
42 HttpClientContext httpContext = new HttpClientContext();
43 if (proxy != null && proxy.getUsername() != null) {
44 AuthState authState = new AuthState();
45 BasicScheme proxyAuthScheme = new BasicScheme(ChallengeState.PROXY);
46 UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword());
47 authState.update(proxyAuthScheme, proxyCredentials);
48 httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState);
49 }
50 if (request.getCookies() != null && !request.getCookies().isEmpty()) {
51 CookieStore cookieStore = new BasicCookieStore();
52 for (Map.Entry<String, String> cookieEntry : request.getCookies().entrySet()) {
53 BasicClientCookie cookie1 = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());
54 cookie1.setDomain(UrlUtils.removePort(UrlUtils.getDomain(request.getUrl())));
55 cookieStore.addCookie(cookie1);
56 }
57 httpContext.setCookieStore(cookieStore);
58 }
59 return httpContext;
60 }
61
62 private HttpUriRequest convertHttpUriRequest(Request request, Site site, Proxy proxy) {
63 RequestBuilder requestBuilder = selectRequestMethod(request).setUri(UrlUtils.fixIllegalCharacterInUrl(request.getUrl()));
64 if (site.getHeaders() != null) {
65 for (Map.Entry<String, String> headerEntry : site.getHeaders().entrySet()) {
66 requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
67 }
68 }
69
70 RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
71 if (site != null) {
72 requestConfigBuilder.setConnectionRequestTimeout(site.getTimeOut())
73 .setSocketTimeout(site.getTimeOut())
74 .setConnectTimeout(site.getTimeOut())
75 .setCookieSpec(CookieSpecs.STANDARD);
76 }
77
78 if (proxy != null) {
79 requestConfigBuilder.setProxy(new HttpHost(proxy.getHost(), proxy.getPort(), proxy.getScheme()));
80 }
81 requestBuilder.setConfig(requestConfigBuilder.build());
82 HttpUriRequest httpUriRequest = requestBuilder.build();
83 if (request.getHeaders() != null && !request.getHeaders().isEmpty()) {
84 for (Map.Entry<String, String> header : request.getHeaders().entrySet()) {
85 httpUriRequest.addHeader(header.getKey(), header.getValue());
86 }
87 }
88 return httpUriRequest;
89 }
90
91 private RequestBuilder selectRequestMethod(Request request) {
92 String method = request.getMethod();
93 if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) {
94
95 return RequestBuilder.get();
96 } else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) {
97 return addFormParams(RequestBuilder.post(),request);
98 } else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) {
99 return RequestBuilder.head();
100 } else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) {
101 return addFormParams(RequestBuilder.put(), request);
102 } else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) {
103 return RequestBuilder.delete();
104 } else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) {
105 return RequestBuilder.trace();
106 }
107 throw new IllegalArgumentException("Illegal HTTP Method " + method);
108 }
109
110 private RequestBuilder addFormParams(RequestBuilder requestBuilder, Request request) {
111 if (request.getRequestBody() != null) {
112 ByteArrayEntity entity = new ByteArrayEntity(request.getRequestBody().getBody());
113 entity.setContentType(request.getRequestBody().getContentType());
114 requestBuilder.setEntity(entity);
115 }
116 return requestBuilder;
117 }
118
119 }