2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler;
19 import java.io.ByteArrayOutputStream;
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.List;
25 import javax.xml.transform.OutputKeys;
26 import javax.xml.transform.Transformer;
27 import javax.xml.transform.TransformerConfigurationException;
28 import javax.xml.transform.TransformerException;
29 import javax.xml.transform.TransformerFactory;
30 import javax.xml.transform.dom.DOMSource;
31 import javax.xml.transform.stream.StreamResult;
33 import org.apache.commons.httpclient.Header;
34 import org.apache.commons.httpclient.HttpClient;
35 import org.apache.commons.httpclient.HttpMethod;
36 import org.apache.commons.httpclient.HttpStatus;
37 import org.apache.commons.httpclient.NameValuePair;
38 import org.apache.commons.httpclient.methods.GetMethod;
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.apache.xml.serialize.OutputFormat;
42 import org.apache.xml.serialize.XMLSerializer;
43 import org.w3c.dom.Document;
44 import org.w3c.tidy.Tidy;
45 import org.wamblee.xml.DomUtils;
46 import org.wamblee.xml.XslTransformer;
49 * General support claas for all kinds of requests.
51 * @author Erik Brakkee
53 public abstract class AbstractPageRequest implements PageRequest {
55 private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class);
57 private static final String REDIRECT_HEADER = "Location";
59 private int _maxTries;
61 private int _maxDelay;
63 private NameValuePair[] _params;
65 private NameValuePair[] _headers;
69 private XslTransformer _transformer;
72 * Constructs the request.
75 * Maximum retries to perform.
77 * Maximum delay before executing a request.
79 * Request parameters to use.
81 * Request headers to use.
83 * XSLT used to convert the response.
85 protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
86 NameValuePair[] aParams, NameValuePair[] aHeaders, String aXslt, XslTransformer aTransformer) {
87 if (aParams == null) {
88 throw new IllegalArgumentException("aParams is null");
90 if (aHeaders == null) {
91 throw new IllegalArgumentException("aHeaders is null");
94 throw new IllegalArgumentException("aXslt is null");
96 _maxTries = aMaxTries;
97 _maxDelay = aMaxDelay;
101 _transformer = aTransformer;
107 * @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String)
109 public void overrideXslt(String aXslt) {
114 * Gets the parameters for the request.
116 * @param aParams Additional parameters to use, obtained from another page, most likely as
117 * hidden form fields.
118 * @return Request parameters.
120 protected NameValuePair[] getParameters(NameValuePair[] aParams) {
121 List<NameValuePair> params = new ArrayList<NameValuePair>();
122 params.addAll(Arrays.asList(_params));
123 params.addAll(Arrays.asList(aParams));
124 return params.toArray(new NameValuePair[0]);
128 * Gets the headers for the request.
129 * @return Request headers.
131 protected NameValuePair[] getHeaders() {
136 * Executes the request with a random delay and with a maximum number of
140 * HTTP client to use.
142 * Method representing the request.
143 * @return XML document describing the response.
144 * @throws IOException
145 * In case of IO problems.
146 * @throws TransformerException
147 * In case transformation of the HTML to XML fails.
149 protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
150 throws IOException, TransformerException {
152 for (NameValuePair header: getHeaders()) {
153 aMethod.setRequestHeader(header.getName(), header.getValue());
156 int triesLeft = _maxTries;
157 while (triesLeft > 0) {
160 return executeMethodWithoutRetries(aClient, aMethod);
161 } catch (TransformerException e) {
162 if (triesLeft == 0) {
167 throw new RuntimeException("Code should never reach this point");
171 * Executes the request without doing any retries in case XSLT
172 * transformation fails.
175 * HTTP client to use.
178 * @return XML document containing the result.
179 * @throws IOException
180 * In case of IO problems.
181 * @throws TransformerException
182 * In case transformation of the result to XML fails.
184 protected Document executeMethodWithoutRetries(HttpClient aClient,
185 HttpMethod aMethod) throws IOException, TransformerException {
187 aMethod = executeWithRedirects(aClient, aMethod);
188 byte[] xhtmlData = getXhtml(aMethod);
191 Document transformed = _transformer.transform(xhtmlData,
192 _transformer.resolve(_xslt));
193 ByteArrayOutputStream os = new ByteArrayOutputStream();
194 Transformer transformer = TransformerFactory.newInstance()
196 transformer.setParameter(OutputKeys.INDENT, "yes");
197 transformer.setParameter(OutputKeys.METHOD, "xml");
198 transformer.transform(new DOMSource(transformed), new StreamResult(
200 LOG.debug("Transformed result is \n" + os.toString());
202 } catch (TransformerConfigurationException e) {
203 throw new TransformerException("Transformer configuration problem", e);
205 // Release the connection.
206 aMethod.releaseConnection();
211 * Gets the result of the HTTP method as an XHTML document.
215 * @return XHTML as a byte array.
216 * @throws IOException
217 * In case of problems obtaining the XHTML.
219 private byte[] getXhtml(HttpMethod aMethod) throws IOException {
220 // Transform the HTML into wellformed XML.
221 Tidy tidy = new Tidy();
224 tidy.setShowWarnings(false);
226 // We write the jtidy output to XML since the DOM tree it produces is
227 // not namespace aware and namespace awareness is required by XSLT.
228 // An alternative is to configure namespace awareness of the XML parser
229 // in a system wide way.
230 ByteArrayOutputStream os = new ByteArrayOutputStream();
231 Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), os);
232 DomUtils.removeDuplicateAttributes(w3cDoc);
233 LOG.debug("Content of response is \n" + os.toString());
235 ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
236 XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
237 serializer.serialize(w3cDoc);
240 return xhtml.toByteArray();
244 * Sleeps for a random time but no more than the maximum delay.
247 private void delay() {
249 Thread.sleep((long) ((float) _maxDelay * Math.random()));
250 } catch (InterruptedException e) {
251 return; // to satisfy checkstyle
256 * Executes the request and follows redirects if needed.
259 * HTTP client to use.
262 * @return Final HTTP method used (differs from the parameter passed in in
263 * case of redirection).
264 * @throws IOException
265 * In case of network problems.
267 private HttpMethod executeWithRedirects(HttpClient aClient,
268 HttpMethod aMethod) throws IOException {
270 int statusCode = aClient.executeMethod(aMethod);
272 switch (statusCode) {
273 case HttpStatus.SC_OK: {
276 case HttpStatus.SC_MOVED_PERMANENTLY:
277 case HttpStatus.SC_MOVED_TEMPORARILY:
278 case HttpStatus.SC_SEE_OTHER: {
279 aMethod.releaseConnection();
280 Header header = aMethod.getResponseHeader(REDIRECT_HEADER);
281 aMethod = new GetMethod(header.getValue());
282 return executeWithRedirects(aClient, aMethod); // TODO protect
287 throw new IOException("Method failed: "
288 + aMethod.getStatusLine());