2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler;
19 import java.io.ByteArrayOutputStream;
20 import java.io.IOException;
22 import javax.xml.transform.OutputKeys;
23 import javax.xml.transform.Transformer;
24 import javax.xml.transform.TransformerConfigurationException;
25 import javax.xml.transform.TransformerException;
26 import javax.xml.transform.TransformerFactory;
27 import javax.xml.transform.dom.DOMSource;
28 import javax.xml.transform.stream.StreamResult;
30 import org.apache.commons.httpclient.Header;
31 import org.apache.commons.httpclient.HttpClient;
32 import org.apache.commons.httpclient.HttpMethod;
33 import org.apache.commons.httpclient.HttpStatus;
34 import org.apache.commons.httpclient.NameValuePair;
35 import org.apache.commons.httpclient.methods.GetMethod;
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.xml.serialize.OutputFormat;
39 import org.apache.xml.serialize.XMLSerializer;
40 import org.w3c.dom.Document;
41 import org.w3c.tidy.Tidy;
42 import org.wamblee.xml.DomUtils;
43 import org.wamblee.xml.XslTransformer;
46 * General support claas for all kinds of requests.
48 public abstract class AbstractPageRequest implements PageRequest {
50 private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class);
52 private static final String REDIRECT_HEADER = "Location";
54 private int _maxTries;
56 private int _maxDelay;
58 private NameValuePair[] _params;
62 private XslTransformer _transformer;
65 * Constructs the request.
68 * Maximum retries to perform.
70 * Maximum delay before executing a request.
72 * Request parameters to use.
74 * XSLT used to convert the response.
76 protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
77 NameValuePair[] aParams, String aXslt, XslTransformer aTransformer) {
78 if (aParams == null) {
79 throw new IllegalArgumentException("aParams is null");
82 throw new IllegalArgumentException("aXslt is null");
84 _maxTries = aMaxTries;
85 _maxDelay = aMaxDelay;
88 _transformer = aTransformer;
94 * @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String)
96 public void overrideXslt(String aXslt) {
101 * Gets the parameters for the request.
103 * @return Request parameters.
105 protected NameValuePair[] getParameters() {
110 * Executes the request with a random delay and with a maximum number of
114 * HTTP client to use.
116 * Method representing the request.
117 * @return XML document describing the response.
118 * @throws IOException
119 * In case of IO problems.
120 * @throws TransformerException
121 * In case transformation of the HTML to XML fails.
123 protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
124 throws IOException, TransformerException {
125 int triesLeft = _maxTries;
126 while (triesLeft > 0) {
129 return executeMethodWithoutRetries(aClient, aMethod);
130 } catch (TransformerException e) {
131 if (triesLeft == 0) {
136 throw new RuntimeException("Code should never reach this point");
140 * Executes the request without doing any retries in case XSLT
141 * transformation fails.
144 * HTTP client to use.
147 * @return XML document containing the result.
148 * @throws IOException
149 * In case of IO problems.
150 * @throws TransformerException
151 * In case transformation of the result to XML fails.
153 protected Document executeMethodWithoutRetries(HttpClient aClient,
154 HttpMethod aMethod) throws IOException, TransformerException {
156 aMethod = executeWithRedirects(aClient, aMethod);
157 byte[] xhtmlData = getXhtml(aMethod);
159 Document transformed = _transformer.transform(xhtmlData,
160 _transformer.resolve(_xslt));
161 ByteArrayOutputStream os = new ByteArrayOutputStream();
162 Transformer transformer = TransformerFactory.newInstance()
164 transformer.setParameter(OutputKeys.INDENT, "yes");
165 transformer.setParameter(OutputKeys.METHOD, "xml");
166 transformer.transform(new DOMSource(transformed), new StreamResult(
168 LOG.debug("Transformed result is \n" + os.toString());
170 } catch (TransformerConfigurationException e) {
171 throw new TransformerException("Transformer configuration problem", e);
173 // Release the connection.
174 aMethod.releaseConnection();
179 * Gets the result of the HTTP method as an XHTML document.
183 * @return XHTML as a byte array.
184 * @throws IOException
185 * In case of problems obtaining the XHTML.
187 private byte[] getXhtml(HttpMethod aMethod) throws IOException {
188 // Transform the HTML into wellformed XML.
189 Tidy tidy = new Tidy();
192 tidy.setShowWarnings(false);
194 // We write the jtidy output to XML since the DOM tree it produces is
195 // not namespace aware and namespace awareness is required by XSLT.
196 // An alternative is to configure namespace awareness of the XML parser
197 // in a system wide way.
198 ByteArrayOutputStream os = new ByteArrayOutputStream();
199 Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), os);
200 DomUtils.removeDuplicateAttributes(w3cDoc);
201 LOG.debug("Content of response is \n" + os.toString());
203 ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
204 XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
205 serializer.serialize(w3cDoc);
208 return xhtml.toByteArray();
212 * Sleeps for a random time but no more than the maximum delay.
215 private void delay() {
217 Thread.sleep((long) ((float) _maxDelay * Math.random()));
218 } catch (InterruptedException e) {
219 return; // to satisfy checkstyle
224 * Executes the request and follows redirects if needed.
227 * HTTP client to use.
230 * @return Final HTTP method used (differs from the parameter passed in in
231 * case of redirection).
232 * @throws IOException
233 * In case of network problems.
235 private HttpMethod executeWithRedirects(HttpClient aClient,
236 HttpMethod aMethod) throws IOException {
238 int statusCode = aClient.executeMethod(aMethod);
240 switch (statusCode) {
241 case HttpStatus.SC_OK: {
244 case HttpStatus.SC_MOVED_PERMANENTLY:
245 case HttpStatus.SC_MOVED_TEMPORARILY:
246 case HttpStatus.SC_SEE_OTHER: {
247 aMethod.releaseConnection();
248 Header header = aMethod.getResponseHeader(REDIRECT_HEADER);
249 aMethod = new GetMethod(header.getValue());
250 return executeWithRedirects(aClient, aMethod); // TODO protect
255 throw new IOException("Method failed: "
256 + aMethod.getStatusLine());