2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler;
19 import java.io.ByteArrayOutputStream;
20 import java.io.IOException;
22 import javax.xml.transform.OutputKeys;
23 import javax.xml.transform.Transformer;
24 import javax.xml.transform.TransformerConfigurationException;
25 import javax.xml.transform.TransformerException;
26 import javax.xml.transform.TransformerFactory;
27 import javax.xml.transform.dom.DOMSource;
28 import javax.xml.transform.stream.StreamResult;
30 import org.apache.commons.httpclient.Header;
31 import org.apache.commons.httpclient.HttpClient;
32 import org.apache.commons.httpclient.HttpMethod;
33 import org.apache.commons.httpclient.HttpStatus;
34 import org.apache.commons.httpclient.NameValuePair;
35 import org.apache.commons.httpclient.methods.GetMethod;
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.xml.serialize.OutputFormat;
39 import org.apache.xml.serialize.XMLSerializer;
40 import org.w3c.dom.Document;
41 import org.w3c.tidy.Tidy;
42 import org.wamblee.xml.DomUtils;
43 import org.wamblee.xml.XslTransformer;
46 * General support claas for all kinds of requests.
48 public abstract class AbstractPageRequest implements PageRequest {
50 private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class);
52 private static final String REDIRECT_HEADER = "Location";
54 private int _maxTries;
56 private int _maxDelay;
58 private NameValuePair[] _params;
60 private NameValuePair[] _headers;
64 private XslTransformer _transformer;
67 * Constructs the request.
70 * Maximum retries to perform.
72 * Maximum delay before executing a request.
74 * Request parameters to use.
76 * Request headers to use.
78 * XSLT used to convert the response.
80 protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
81 NameValuePair[] aParams, NameValuePair[] aHeaders, String aXslt, XslTransformer aTransformer) {
82 if (aParams == null) {
83 throw new IllegalArgumentException("aParams is null");
85 if (aHeaders == null) {
86 throw new IllegalArgumentException("aHeaders is null");
89 throw new IllegalArgumentException("aXslt is null");
91 _maxTries = aMaxTries;
92 _maxDelay = aMaxDelay;
96 _transformer = aTransformer;
102 * @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String)
104 public void overrideXslt(String aXslt) {
109 * Gets the parameters for the request.
111 * @return Request parameters.
113 protected NameValuePair[] getParameters() {
118 * Gets the headers for the request.
119 * @return Request headers.
121 protected NameValuePair[] getHeaders() {
126 * Executes the request with a random delay and with a maximum number of
130 * HTTP client to use.
132 * Method representing the request.
133 * @return XML document describing the response.
134 * @throws IOException
135 * In case of IO problems.
136 * @throws TransformerException
137 * In case transformation of the HTML to XML fails.
139 protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
140 throws IOException, TransformerException {
142 for (NameValuePair header: getHeaders()) {
143 aMethod.setRequestHeader(header.getName(), header.getValue());
146 int triesLeft = _maxTries;
147 while (triesLeft > 0) {
150 return executeMethodWithoutRetries(aClient, aMethod);
151 } catch (TransformerException e) {
152 if (triesLeft == 0) {
157 throw new RuntimeException("Code should never reach this point");
161 * Executes the request without doing any retries in case XSLT
162 * transformation fails.
165 * HTTP client to use.
168 * @return XML document containing the result.
169 * @throws IOException
170 * In case of IO problems.
171 * @throws TransformerException
172 * In case transformation of the result to XML fails.
174 protected Document executeMethodWithoutRetries(HttpClient aClient,
175 HttpMethod aMethod) throws IOException, TransformerException {
177 aMethod = executeWithRedirects(aClient, aMethod);
178 byte[] xhtmlData = getXhtml(aMethod);
180 Document transformed = _transformer.transform(xhtmlData,
181 _transformer.resolve(_xslt));
182 ByteArrayOutputStream os = new ByteArrayOutputStream();
183 Transformer transformer = TransformerFactory.newInstance()
185 transformer.setParameter(OutputKeys.INDENT, "yes");
186 transformer.setParameter(OutputKeys.METHOD, "xml");
187 transformer.transform(new DOMSource(transformed), new StreamResult(
189 LOG.debug("Transformed result is \n" + os.toString());
191 } catch (TransformerConfigurationException e) {
192 throw new TransformerException("Transformer configuration problem", e);
194 // Release the connection.
195 aMethod.releaseConnection();
200 * Gets the result of the HTTP method as an XHTML document.
204 * @return XHTML as a byte array.
205 * @throws IOException
206 * In case of problems obtaining the XHTML.
208 private byte[] getXhtml(HttpMethod aMethod) throws IOException {
209 // Transform the HTML into wellformed XML.
210 Tidy tidy = new Tidy();
213 tidy.setShowWarnings(false);
215 // We write the jtidy output to XML since the DOM tree it produces is
216 // not namespace aware and namespace awareness is required by XSLT.
217 // An alternative is to configure namespace awareness of the XML parser
218 // in a system wide way.
219 ByteArrayOutputStream os = new ByteArrayOutputStream();
220 Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), os);
221 DomUtils.removeDuplicateAttributes(w3cDoc);
222 LOG.debug("Content of response is \n" + os.toString());
224 ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
225 XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
226 serializer.serialize(w3cDoc);
229 return xhtml.toByteArray();
233 * Sleeps for a random time but no more than the maximum delay.
236 private void delay() {
238 Thread.sleep((long) ((float) _maxDelay * Math.random()));
239 } catch (InterruptedException e) {
240 return; // to satisfy checkstyle
245 * Executes the request and follows redirects if needed.
248 * HTTP client to use.
251 * @return Final HTTP method used (differs from the parameter passed in in
252 * case of redirection).
253 * @throws IOException
254 * In case of network problems.
256 private HttpMethod executeWithRedirects(HttpClient aClient,
257 HttpMethod aMethod) throws IOException {
259 int statusCode = aClient.executeMethod(aMethod);
261 switch (statusCode) {
262 case HttpStatus.SC_OK: {
265 case HttpStatus.SC_MOVED_PERMANENTLY:
266 case HttpStatus.SC_MOVED_TEMPORARILY:
267 case HttpStatus.SC_SEE_OTHER: {
268 aMethod.releaseConnection();
269 Header header = aMethod.getResponseHeader(REDIRECT_HEADER);
270 aMethod = new GetMethod(header.getValue());
271 return executeWithRedirects(aClient, aMethod); // TODO protect
276 throw new IOException("Method failed: "
277 + aMethod.getStatusLine());