2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler;
19 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.io.PrintStream;
24 import javax.xml.transform.OutputKeys;
25 import javax.xml.transform.Transformer;
26 import javax.xml.transform.TransformerConfigurationException;
27 import javax.xml.transform.TransformerException;
28 import javax.xml.transform.TransformerFactory;
29 import javax.xml.transform.dom.DOMSource;
30 import javax.xml.transform.stream.StreamResult;
32 import org.apache.commons.httpclient.Header;
33 import org.apache.commons.httpclient.HttpClient;
34 import org.apache.commons.httpclient.HttpException;
35 import org.apache.commons.httpclient.HttpMethod;
36 import org.apache.commons.httpclient.HttpStatus;
37 import org.apache.commons.httpclient.NameValuePair;
38 import org.apache.commons.httpclient.methods.GetMethod;
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.w3c.dom.Document;
42 import org.w3c.tidy.Tidy;
43 import org.wamblee.io.FileResource;
44 import org.wamblee.xml.XSLT;
47 * General support claas for all kinds of requests.
49 public abstract class AbstractPageRequest implements PageRequest {
51 private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class);
53 private static final String REDIRECT_HEADER = "Location";
55 private int _maxTries;
57 private int _maxDelay;
59 private NameValuePair[] _params;
63 private PrintStream _os;
66 * Constructs the request.
67 * @param aMaxTries Maximum retries to perform.
68 * @param aMaxDelay Maximum delay before executing a request.
69 * @param aParams Request parameters to use.
70 * @param aXslt XSLT used to convert the response.
71 * @param aOs Output stream for logging (if null then no logging is done).
73 protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
74 NameValuePair[] aParams, String aXslt, PrintStream aOs) {
75 if (aParams == null) {
76 throw new IllegalArgumentException("aParams is null");
79 throw new IllegalArgumentException("aXslt is null");
81 _maxTries = aMaxTries;
82 _maxDelay = aMaxDelay;
91 * @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String)
93 public void overrideXslt(String aXslt) {
98 * Gets the parameters for the request.
99 * @return Request parameters.
101 protected NameValuePair[] getParameters() {
106 * Executes the request with a random delay and with a maximum number of
108 * @param aClient HTTP client to use.
109 * @param aMethod Method representing the request.
110 * @return XML document describing the response.
111 * @throws TransformerException In case transformation of the HTML to XML fails.
113 protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
114 throws TransformerException {
115 int triesLeft = _maxTries;
116 while (triesLeft > 0) {
119 return executeMethodWithoutRetries(aClient, aMethod);
120 } catch (TransformerException e) {
121 if (triesLeft == 0) {
126 throw new RuntimeException("Code should never reach this point");
130 * Executes the request without doing any retries in case XSLT transformation
132 * @param aClient HTTP client to use.
133 * @param aMethod Method to execute.
134 * @return XML document containing the result.
135 * @throws TransformerException In case transformation of the result to XML fails.
137 protected Document executeMethodWithoutRetries(HttpClient aClient,
138 HttpMethod aMethod) throws TransformerException {
140 // Execute the method.
141 aMethod = executeWithRedirects(aClient, aMethod);
143 // Transform the HTML into wellformed XML.
144 Tidy tidy = new Tidy();
147 tidy.setShowWarnings(false);
149 _os.println("Content of '" + aMethod.getURI() + "'");
152 // We let jtidy produce raw output because the DOM it produces is
153 // is not namespace aware. We let the XSLT processor parse the XML
155 // to ensure that the XSLT uses a namespace aware DOM tree. An
157 // is to configure namespace awareness of the XML parser in a system
159 ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
160 tidy.parse(aMethod.getResponseBodyAsStream(), xhtml);
161 _os.print(new String(xhtml.toByteArray()));
162 // Obtaining the XML as dom is not used.
163 // Document w3cDoc = tidy.parseDOM(method.getResponseBodyAsStream(),
169 byte[] xhtmlData = xhtml.toByteArray();
170 Document transformed = new XSLT().transform(xhtmlData,
171 new FileResource(new File(_xslt)));
172 _os.println("Transformed result is: ");
173 Transformer transformer = TransformerFactory.newInstance()
175 transformer.setParameter(OutputKeys.INDENT, "yes");
176 transformer.setParameter(OutputKeys.METHOD, "xml");
177 transformer.transform(new DOMSource(transformed), new StreamResult(
181 } catch (HttpException e) {
182 throw new RuntimeException(e.getMessage(), e);
183 } catch (IOException e) {
184 throw new RuntimeException(e.getMessage(), e);
185 } catch (TransformerConfigurationException e) {
186 throw new RuntimeException(e.getMessage(), e);
188 // Release the connection.
189 aMethod.releaseConnection();
194 * Sleeps for a random time but no more than the maximum delay.
197 private void delay() {
199 Thread.sleep((long) ((float) _maxDelay * Math.random()));
200 } catch (InterruptedException e) {
201 return; // to satisfy checkstyle
206 * Executes the request and follows redirects if needed.
207 * @param aClient HTTP client to use.
208 * @param aMethod Method to use.
209 * @return Final HTTP method used (differs from the parameter passed in in case
211 * @throws IOException In case of network problems.
213 private HttpMethod executeWithRedirects(HttpClient aClient,
214 HttpMethod aMethod) throws IOException {
216 int statusCode = aClient.executeMethod(aMethod);
218 switch (statusCode) {
219 case HttpStatus.SC_OK: {
222 case HttpStatus.SC_MOVED_PERMANENTLY:
223 case HttpStatus.SC_MOVED_TEMPORARILY:
224 case HttpStatus.SC_SEE_OTHER: {
225 aMethod.releaseConnection();
226 Header header = aMethod.getResponseHeader(REDIRECT_HEADER);
227 aMethod = new GetMethod(header.getValue());
228 return executeWithRedirects(aClient, aMethod); // TODO protect
233 throw new RuntimeException("Method failed: "
234 + aMethod.getStatusLine());