2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler;
19 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.io.OutputStream;
23 import java.io.PrintStream;
25 import javax.xml.transform.OutputKeys;
26 import javax.xml.transform.Transformer;
27 import javax.xml.transform.TransformerConfigurationException;
28 import javax.xml.transform.TransformerException;
29 import javax.xml.transform.TransformerFactory;
30 import javax.xml.transform.dom.DOMSource;
31 import javax.xml.transform.stream.StreamResult;
33 import org.apache.commons.httpclient.Header;
34 import org.apache.commons.httpclient.HttpClient;
35 import org.apache.commons.httpclient.HttpException;
36 import org.apache.commons.httpclient.HttpMethod;
37 import org.apache.commons.httpclient.HttpStatus;
38 import org.apache.commons.httpclient.NameValuePair;
39 import org.apache.commons.httpclient.URIException;
40 import org.apache.commons.httpclient.methods.GetMethod;
41 import org.apache.commons.logging.Log;
42 import org.apache.commons.logging.LogFactory;
43 import org.apache.xml.serialize.OutputFormat;
44 import org.apache.xml.serialize.XMLSerializer;
45 import org.w3c.dom.Document;
46 import org.w3c.tidy.Tidy;
47 import org.wamblee.io.FileResource;
48 import org.wamblee.xml.DOMUtility;
49 import org.wamblee.xml.XSLT;
52 * General support claas for all kinds of requests.
54 public abstract class AbstractPageRequest implements PageRequest {
56 private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class);
58 private static final String REDIRECT_HEADER = "Location";
60 private int _maxTries;
62 private int _maxDelay;
64 private NameValuePair[] _params;
68 private PrintStream _os;
71 * Constructs the request.
72 * @param aMaxTries Maximum retries to perform.
73 * @param aMaxDelay Maximum delay before executing a request.
74 * @param aParams Request parameters to use.
75 * @param aXslt XSLT used to convert the response.
76 * @param aOs Output stream for logging (if null then no logging is done).
78 protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
79 NameValuePair[] aParams, String aXslt, PrintStream aOs) {
80 if (aParams == null) {
81 throw new IllegalArgumentException("aParams is null");
84 throw new IllegalArgumentException("aXslt is null");
86 _maxTries = aMaxTries;
87 _maxDelay = aMaxDelay;
96 * @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String)
98 public void overrideXslt(String aXslt) {
103 * Gets the parameters for the request.
104 * @return Request parameters.
106 protected NameValuePair[] getParameters() {
111 * Executes the request with a random delay and with a maximum number of
113 * @param aClient HTTP client to use.
114 * @param aMethod Method representing the request.
115 * @return XML document describing the response.
116 * @throws TransformerException In case transformation of the HTML to XML fails.
118 protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
119 throws TransformerException {
120 int triesLeft = _maxTries;
121 while (triesLeft > 0) {
124 return executeMethodWithoutRetries(aClient, aMethod);
125 } catch (TransformerException e) {
126 if (triesLeft == 0) {
131 throw new RuntimeException("Code should never reach this point");
135 * Executes the request without doing any retries in case XSLT transformation
137 * @param aClient HTTP client to use.
138 * @param aMethod Method to execute.
139 * @return XML document containing the result.
140 * @throws TransformerException In case transformation of the result to XML fails.
142 protected Document executeMethodWithoutRetries(HttpClient aClient,
143 HttpMethod aMethod) throws TransformerException {
145 aMethod = executeWithRedirects(aClient, aMethod);
146 byte[] xhtmlData = getXhtml(aMethod);
148 Document transformed = new XSLT().transform(xhtmlData,
149 new FileResource(new File(_xslt)));
150 _os.println("Transformed result is: ");
151 Transformer transformer = TransformerFactory.newInstance()
153 transformer.setParameter(OutputKeys.INDENT, "yes");
154 transformer.setParameter(OutputKeys.METHOD, "xml");
155 transformer.transform(new DOMSource(transformed), new StreamResult(
159 } catch (HttpException e) {
160 throw new RuntimeException(e.getMessage(), e);
161 } catch (IOException e) {
162 throw new RuntimeException(e.getMessage(), e);
163 } catch (TransformerConfigurationException e) {
164 throw new RuntimeException(e.getMessage(), e);
166 // Release the connection.
167 aMethod.releaseConnection();
172 * Gets the result of the HTTP method as an XHTML document.
173 * @param aMethod Method to invoke.
174 * @return XHTML as a byte array.
175 * @throws URIException In case of poblems with the URI
176 * @throws IOException In case of problems obtaining the XHTML.
178 private byte[] getXhtml(HttpMethod aMethod) throws URIException, IOException {
179 // Transform the HTML into wellformed XML.
180 Tidy tidy = new Tidy();
183 tidy.setShowWarnings(false);
185 _os.println("Content of '" + aMethod.getURI() + "'");
188 // We write the jtidy output to XML since the DOM tree it produces is
189 // not namespace aware and namespace awareness is required by XSLT.
190 // An alternative is to configure namespace awareness of the XML parser
191 // in a system wide way.
192 Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(),
194 DOMUtility.removeDuplicateAttributes(w3cDoc);
196 ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
197 XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
198 serializer.serialize(w3cDoc);
203 return xhtml.toByteArray();
207 * Sleeps for a random time but no more than the maximum delay.
210 private void delay() {
212 Thread.sleep((long) ((float) _maxDelay * Math.random()));
213 } catch (InterruptedException e) {
214 return; // to satisfy checkstyle
219 * Executes the request and follows redirects if needed.
220 * @param aClient HTTP client to use.
221 * @param aMethod Method to use.
222 * @return Final HTTP method used (differs from the parameter passed in in case
224 * @throws IOException In case of network problems.
226 private HttpMethod executeWithRedirects(HttpClient aClient,
227 HttpMethod aMethod) throws IOException {
229 int statusCode = aClient.executeMethod(aMethod);
231 switch (statusCode) {
232 case HttpStatus.SC_OK: {
235 case HttpStatus.SC_MOVED_PERMANENTLY:
236 case HttpStatus.SC_MOVED_TEMPORARILY:
237 case HttpStatus.SC_SEE_OTHER: {
238 aMethod.releaseConnection();
239 Header header = aMethod.getResponseHeader(REDIRECT_HEADER);
240 aMethod = new GetMethod(header.getValue());
241 return executeWithRedirects(aClient, aMethod); // TODO protect
246 throw new RuntimeException("Method failed: "
247 + aMethod.getStatusLine());