package org.wamblee.crawler;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.IOException;
-import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.xml.serialize.OutputFormat;
+import org.apache.xml.serialize.XMLSerializer;
import org.w3c.dom.Document;
import org.w3c.tidy.Tidy;
-import org.wamblee.io.FileResource;
-import org.wamblee.xml.XSLT;
+import org.wamblee.xml.DomUtils;
+import org.wamblee.xml.XslTransformer;
/**
* General support claas for all kinds of requests.
public abstract class AbstractPageRequest implements PageRequest {
private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class);
+
private static final String REDIRECT_HEADER = "Location";
-
- private int _maxTries;
- private int _maxDelay;
+
+ private int _maxTries;
+
+ private int _maxDelay;
private NameValuePair[] _params;
+
+ private NameValuePair[] _headers;
private String _xslt;
- private PrintStream _os;
+ private XslTransformer _transformer;
- protected AbstractPageRequest(int aMaxTries, int aMaxDelay, NameValuePair[] aParams, String aXslt, PrintStream aOs) {
- if ( aParams == null ) {
+ /**
+ * Constructs the request.
+ *
+ * @param aMaxTries
+ * Maximum retries to perform.
+ * @param aMaxDelay
+ * Maximum delay before executing a request.
+ * @param aParams
+ * Request parameters to use.
+ * @param aHeaders
+ * Request headers to use.
+ * @param aXslt
+ * XSLT used to convert the response.
+ */
+ protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
+ NameValuePair[] aParams, NameValuePair[] aHeaders, String aXslt, XslTransformer aTransformer) {
+ if (aParams == null) {
throw new IllegalArgumentException("aParams is null");
}
- if ( aXslt == null ) {
+ if (aHeaders == null) {
+ throw new IllegalArgumentException("aHeaders is null");
+ }
+ if (aXslt == null) {
throw new IllegalArgumentException("aXslt is null");
}
_maxTries = aMaxTries;
_maxDelay = aMaxDelay;
_params = aParams;
+ _headers = aHeaders;
_xslt = aXslt;
- _os = aOs;
+ _transformer = aTransformer;
}
-
- /* (non-Javadoc)
+
+ /*
+ * (non-Javadoc)
+ *
* @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String)
*/
public void overrideXslt(String aXslt) {
- _xslt = aXslt;
+ _xslt = aXslt;
}
-
- protected NameValuePair[] getParameters() {
- return _params;
+
+ /**
+ * Gets the parameters for the request.
+ *
+ * @param aParams Additional parameters to use, obtained from another page, most likely as
+ * hidden form fields.
+ * @return Request parameters.
+ */
+ protected NameValuePair[] getParameters(NameValuePair[] aParams) {
+ List<NameValuePair> params = new ArrayList<NameValuePair>();
+ params.addAll(Arrays.asList(_params));
+ params.addAll(Arrays.asList(aParams));
+ return params.toArray(new NameValuePair[0]);
}
- protected Document executeMethod(HttpClient client, HttpMethod method) throws TransformerException {
- int triesLeft = _maxTries;
- while ( triesLeft > 0 ) {
+ /**
+ * Gets the headers for the request.
+ * @return Request headers.
+ */
+ protected NameValuePair[] getHeaders() {
+ return _headers;
+ }
+
+ /**
+ * Executes the request with a random delay and with a maximum number of
+ * retries.
+ *
+ * @param aClient
+ * HTTP client to use.
+ * @param aMethod
+ * Method representing the request.
+ * @return XML document describing the response.
+ * @throws IOException
+ * In case of IO problems.
+ * @throws TransformerException
+ * In case transformation of the HTML to XML fails.
+ */
+ protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
+ throws IOException, TransformerException {
+
+ for (NameValuePair header: getHeaders()) {
+ aMethod.setRequestHeader(header.getName(), header.getValue());
+ }
+
+ int triesLeft = _maxTries;
+ while (triesLeft > 0) {
triesLeft--;
- try {
- return executeMethodWithoutRetries(client, method);
- } catch (TransformerException e) {
- if ( triesLeft == 0 ) {
+ try {
+ return executeMethodWithoutRetries(aClient, aMethod);
+ } catch (TransformerException e) {
+ if (triesLeft == 0) {
throw e;
}
}
}
throw new RuntimeException("Code should never reach this point");
}
-
- protected Document executeMethodWithoutRetries(HttpClient client, HttpMethod method) throws TransformerException {
+ /**
+ * Executes the request without doing any retries in case XSLT
+ * transformation fails.
+ *
+ * @param aClient
+ * HTTP client to use.
+ * @param aMethod
+ * Method to execute.
+ * @return XML document containing the result.
+ * @throws IOException
+ * In case of IO problems.
+ * @throws TransformerException
+ * In case transformation of the result to XML fails.
+ */
+ protected Document executeMethodWithoutRetries(HttpClient aClient,
+ HttpMethod aMethod) throws IOException, TransformerException {
try {
- // Execute the method.
- method = executeWithRedirects(client, method);
-
- // Transform the HTML into wellformed XML.
- Tidy tidy = new Tidy();
- tidy.setXHTML(true);
- tidy.setQuiet(true);
- tidy.setShowWarnings(false);
- if ( _os != null ) {
- _os.println("Content of '" + method.getURI() + "'");
- _os.println();
- }
- // We let jtidy produce raw output because the DOM it produces is
- // is not namespace aware. We let the XSLT processor parse the XML again
- // to ensure that the XSLT uses a namespace aware DOM tree. An alternative
- // is to configure namespace awareness of the XML parser in a system wide way.
- ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
- tidy.parse(method.getResponseBodyAsStream(), xhtml);
- _os.print(new String(xhtml.toByteArray()));
- // Obtaining the XML as dom is not used.
- //Document w3cDoc = tidy.parseDOM(method.getResponseBodyAsStream(),
- // _os);
- if ( _os != null ) {
- _os.println();
- }
- xhtml.flush();
- byte[] xhtmlData = xhtml.toByteArray();
- Document transformed = new XSLT().transform(xhtmlData, new FileResource(new File(_xslt)));
- _os.println("Transformed result is: ");
- Transformer transformer = TransformerFactory.newInstance().newTransformer();
+ aMethod = executeWithRedirects(aClient, aMethod);
+ byte[] xhtmlData = getXhtml(aMethod);
+
+
+ Document transformed = _transformer.transform(xhtmlData,
+ _transformer.resolve(_xslt));
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ Transformer transformer = TransformerFactory.newInstance()
+ .newTransformer();
transformer.setParameter(OutputKeys.INDENT, "yes");
transformer.setParameter(OutputKeys.METHOD, "xml");
- transformer.transform(new DOMSource(transformed), new StreamResult(_os));
-
+ transformer.transform(new DOMSource(transformed), new StreamResult(
+ os));
+ LOG.debug("Transformed result is \n" + os.toString());
return transformed;
- } catch (HttpException e) {
- throw new RuntimeException(e.getMessage(), e);
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage(), e);
- } catch (TransformerConfigurationException e) {
- throw new RuntimeException(e.getMessage(), e);
+ } catch (TransformerConfigurationException e) {
+ throw new TransformerException("Transformer configuration problem", e);
} finally {
// Release the connection.
- method.releaseConnection();
+ aMethod.releaseConnection();
}
}
-
- private void delay() {
+
+ /**
+ * Gets the result of the HTTP method as an XHTML document.
+ *
+ * @param aMethod
+ * Method to invoke.
+ * @return XHTML as a byte array.
+ * @throws IOException
+ * In case of problems obtaining the XHTML.
+ */
+ private byte[] getXhtml(HttpMethod aMethod) throws IOException {
+ // Transform the HTML into wellformed XML.
+ Tidy tidy = new Tidy();
+ tidy.setXHTML(true);
+ tidy.setQuiet(true);
+ tidy.setShowWarnings(false);
+
+ // We write the jtidy output to XML since the DOM tree it produces is
+ // not namespace aware and namespace awareness is required by XSLT.
+ // An alternative is to configure namespace awareness of the XML parser
+ // in a system wide way.
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), os);
+ DomUtils.removeDuplicateAttributes(w3cDoc);
+ LOG.debug("Content of response is \n" + os.toString());
+
+ ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
+ XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
+ serializer.serialize(w3cDoc);
+ xhtml.flush();
+
+ return xhtml.toByteArray();
+ }
+
+ /**
+ * Sleeps for a random time but no more than the maximum delay.
+ *
+ */
+ private void delay() {
try {
- Thread.sleep((long)((float)_maxDelay* Math.random()));
- } catch (InterruptedException e) {
- //
+ Thread.sleep((long) ((float) _maxDelay * Math.random()));
+ } catch (InterruptedException e) {
+ return; // to satisfy checkstyle
}
}
-
/**
+ * Executes the request and follows redirects if needed.
+ *
* @param aClient
+ * HTTP client to use.
* @param aMethod
+ * Method to use.
+ * @return Final HTTP method used (differs from the parameter passed in in
+ * case of redirection).
* @throws IOException
- * @throws HttpException
+ * In case of network problems.
*/
- private HttpMethod executeWithRedirects(HttpClient aClient, HttpMethod aMethod) throws IOException, HttpException {
+ private HttpMethod executeWithRedirects(HttpClient aClient,
+ HttpMethod aMethod) throws IOException {
delay();
int statusCode = aClient.executeMethod(aMethod);
- switch (statusCode) {
+ switch (statusCode) {
case HttpStatus.SC_OK: {
return aMethod;
}
aMethod.releaseConnection();
Header header = aMethod.getResponseHeader(REDIRECT_HEADER);
aMethod = new GetMethod(header.getValue());
- return executeWithRedirects(aClient, aMethod); // TODO protect against infinite recursion.
+ return executeWithRedirects(aClient, aMethod); // TODO protect
+ // against infinite
+ // recursion.
}
default: {
- throw new RuntimeException("Method failed: "
+ throw new IOException("Method failed: "
+ aMethod.getStatusLine());
}
}
git://wamblee.org / utils / blobdiff