2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler;
19 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.io.PrintStream;
24 import javax.xml.transform.OutputKeys;
25 import javax.xml.transform.Transformer;
26 import javax.xml.transform.TransformerConfigurationException;
27 import javax.xml.transform.TransformerException;
28 import javax.xml.transform.TransformerFactory;
29 import javax.xml.transform.dom.DOMSource;
30 import javax.xml.transform.stream.StreamResult;
32 import org.apache.commons.httpclient.Header;
33 import org.apache.commons.httpclient.HttpClient;
34 import org.apache.commons.httpclient.HttpException;
35 import org.apache.commons.httpclient.HttpMethod;
36 import org.apache.commons.httpclient.HttpStatus;
37 import org.apache.commons.httpclient.NameValuePair;
38 import org.apache.commons.httpclient.methods.GetMethod;
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.w3c.dom.Document;
42 import org.w3c.tidy.Tidy;
43 import org.wamblee.io.FileResource;
44 import org.wamblee.xml.XSLT;
47 * General support claas for all kinds of requests.
49 public abstract class AbstractPageRequest implements PageRequest {
51 private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class);
52 private static final String REDIRECT_HEADER = "Location";
54 private int _maxTries;
55 private int _maxDelay;
57 private NameValuePair[] _params;
61 private PrintStream _os;
63 protected AbstractPageRequest(int aMaxTries, int aMaxDelay, NameValuePair[] aParams, String aXslt, PrintStream aOs) {
64 if ( aParams == null ) {
65 throw new IllegalArgumentException("aParams is null");
67 if ( aXslt == null ) {
68 throw new IllegalArgumentException("aXslt is null");
70 _maxTries = aMaxTries;
71 _maxDelay = aMaxDelay;
78 * @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String)
80 public void overrideXslt(String aXslt) {
84 protected NameValuePair[] getParameters() {
88 protected Document executeMethod(HttpClient client, HttpMethod method) throws TransformerException {
89 int triesLeft = _maxTries;
90 while ( triesLeft > 0 ) {
93 return executeMethodWithoutRetries(client, method);
94 } catch (TransformerException e) {
95 if ( triesLeft == 0 ) {
100 throw new RuntimeException("Code should never reach this point");
104 protected Document executeMethodWithoutRetries(HttpClient client, HttpMethod method) throws TransformerException {
106 // Execute the method.
107 method = executeWithRedirects(client, method);
109 // Transform the HTML into wellformed XML.
110 Tidy tidy = new Tidy();
113 tidy.setShowWarnings(false);
115 _os.println("Content of '" + method.getURI() + "'");
118 // We let jtidy produce raw output because the DOM it produces is
119 // is not namespace aware. We let the XSLT processor parse the XML again
120 // to ensure that the XSLT uses a namespace aware DOM tree. An alternative
121 // is to configure namespace awareness of the XML parser in a system wide way.
122 ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
123 tidy.parse(method.getResponseBodyAsStream(), xhtml);
124 _os.print(new String(xhtml.toByteArray()));
125 // Obtaining the XML as dom is not used.
126 //Document w3cDoc = tidy.parseDOM(method.getResponseBodyAsStream(),
132 byte[] xhtmlData = xhtml.toByteArray();
133 Document transformed = new XSLT().transform(xhtmlData, new FileResource(new File(_xslt)));
134 _os.println("Transformed result is: ");
135 Transformer transformer = TransformerFactory.newInstance().newTransformer();
136 transformer.setParameter(OutputKeys.INDENT, "yes");
137 transformer.setParameter(OutputKeys.METHOD, "xml");
138 transformer.transform(new DOMSource(transformed), new StreamResult(_os));
141 } catch (HttpException e) {
142 throw new RuntimeException(e.getMessage(), e);
143 } catch (IOException e) {
144 throw new RuntimeException(e.getMessage(), e);
145 } catch (TransformerConfigurationException e) {
146 throw new RuntimeException(e.getMessage(), e);
148 // Release the connection.
149 method.releaseConnection();
153 private void delay() {
155 Thread.sleep((long)((float)_maxDelay* Math.random()));
156 } catch (InterruptedException e) {
165 * @throws IOException
166 * @throws HttpException
168 private HttpMethod executeWithRedirects(HttpClient aClient, HttpMethod aMethod) throws IOException, HttpException {
170 int statusCode = aClient.executeMethod(aMethod);
172 switch (statusCode) {
173 case HttpStatus.SC_OK: {
176 case HttpStatus.SC_MOVED_PERMANENTLY:
177 case HttpStatus.SC_MOVED_TEMPORARILY:
178 case HttpStatus.SC_SEE_OTHER: {
179 aMethod.releaseConnection();
180 Header header = aMethod.getResponseHeader(REDIRECT_HEADER);
181 aMethod = new GetMethod(header.getValue());
182 return executeWithRedirects(aClient, aMethod); // TODO protect against infinite recursion.
185 throw new RuntimeException("Method failed: "
186 + aMethod.getStatusLine());