X-Git-Url: http://wamblee.org/gitweb/?a=blobdiff_plain;f=crawler%2Fbasic%2Fsrc%2Forg%2Fwamblee%2Fcrawler%2FAbstractPageRequest.java;h=5cb4fae6e40bc2f3b0215118bad67ca52028e657;hb=e1aafb0930f726a00368ce3468a48193d0fb6fac;hp=b37834bb2fa61b5c93ca18460b1d804274d2bcf3;hpb=748219e39369b60c58fbefdb505145fb9d0a7d85;p=utils diff --git a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java index b37834bb..5cb4fae6 100644 --- a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java +++ b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java @@ -17,9 +17,7 @@ package org.wamblee.crawler; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; -import java.io.PrintStream; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; @@ -34,7 +32,6 @@ import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.NameValuePair; -import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -42,9 +39,8 @@ import org.apache.xml.serialize.OutputFormat; import org.apache.xml.serialize.XMLSerializer; import org.w3c.dom.Document; import org.w3c.tidy.Tidy; -import org.wamblee.io.FileResource; import org.wamblee.xml.DOMUtility; -import org.wamblee.xml.XSLT; +import org.wamblee.xml.XslTransformer; /** * General support claas for all kinds of requests. @@ -62,19 +58,23 @@ public abstract class AbstractPageRequest implements PageRequest { private NameValuePair[] _params; private String _xslt; - - private PrintStream _os; + + private XslTransformer _transformer; /** - * Constructs the request. - * @param aMaxTries Maximum retries to perform. - * @param aMaxDelay Maximum delay before executing a request. - * @param aParams Request parameters to use. - * @param aXslt XSLT used to convert the response. - * @param aOs Output stream for logging (if null then no logging is done). + * Constructs the request. + * + * @param aMaxTries + * Maximum retries to perform. + * @param aMaxDelay + * Maximum delay before executing a request. + * @param aParams + * Request parameters to use. + * @param aXslt + * XSLT used to convert the response. */ protected AbstractPageRequest(int aMaxTries, int aMaxDelay, - NameValuePair[] aParams, String aXslt, PrintStream aOs) { + NameValuePair[] aParams, String aXslt, XslTransformer aTransformer) { if (aParams == null) { throw new IllegalArgumentException("aParams is null"); } @@ -85,7 +85,7 @@ public abstract class AbstractPageRequest implements PageRequest { _maxDelay = aMaxDelay; _params = aParams; _xslt = aXslt; - _os = aOs; + _transformer = aTransformer; } /* @@ -98,21 +98,27 @@ public abstract class AbstractPageRequest implements PageRequest { } /** - * Gets the parameters for the request. - * @return Request parameters. + * Gets the parameters for the request. + * + * @return Request parameters. */ protected NameValuePair[] getParameters() { return _params; } /** - * Executes the request with a random delay and with a maximum number of - * retries. - * @param aClient HTTP client to use. - * @param aMethod Method representing the request. + * Executes the request with a random delay and with a maximum number of + * retries. + * + * @param aClient + * HTTP client to use. + * @param aMethod + * Method representing the request. * @return XML document describing the response. - * @throws IOException In case of IO problems. - * @throws TransformerException In case transformation of the HTML to XML fails. + * @throws IOException + * In case of IO problems. + * @throws TransformerException + * In case transformation of the HTML to XML fails. */ protected Document executeMethod(HttpClient aClient, HttpMethod aMethod) throws IOException, TransformerException { @@ -131,30 +137,35 @@ public abstract class AbstractPageRequest implements PageRequest { } /** - * Executes the request without doing any retries in case XSLT transformation - * fails. - * @param aClient HTTP client to use. - * @param aMethod Method to execute. - * @return XML document containing the result. - * @throws IOException In case of IO problems. - * @throws TransformerException In case transformation of the result to XML fails. + * Executes the request without doing any retries in case XSLT + * transformation fails. + * + * @param aClient + * HTTP client to use. + * @param aMethod + * Method to execute. + * @return XML document containing the result. + * @throws IOException + * In case of IO problems. + * @throws TransformerException + * In case transformation of the result to XML fails. */ protected Document executeMethodWithoutRetries(HttpClient aClient, HttpMethod aMethod) throws IOException, TransformerException { try { aMethod = executeWithRedirects(aClient, aMethod); byte[] xhtmlData = getXhtml(aMethod); - - Document transformed = new XSLT().transform(xhtmlData, - new FileResource(new File(_xslt))); - _os.println("Transformed result is: "); + + Document transformed = _transformer.transform(xhtmlData, + _transformer.resolve(_xslt)); + ByteArrayOutputStream os = new ByteArrayOutputStream(); Transformer transformer = TransformerFactory.newInstance() .newTransformer(); transformer.setParameter(OutputKeys.INDENT, "yes"); transformer.setParameter(OutputKeys.METHOD, "xml"); transformer.transform(new DOMSource(transformed), new StreamResult( - _os)); - + os)); + LOG.debug("Transformed result is \n" + os.toString()); return transformed; } catch (TransformerConfigurationException e) { throw new RuntimeException(e.getMessage(), e); @@ -165,43 +176,41 @@ public abstract class AbstractPageRequest implements PageRequest { } /** - * Gets the result of the HTTP method as an XHTML document. - * @param aMethod Method to invoke. - * @return XHTML as a byte array. - * @throws URIException In case of poblems with the URI - * @throws IOException In case of problems obtaining the XHTML. + * Gets the result of the HTTP method as an XHTML document. + * + * @param aMethod + * Method to invoke. + * @return XHTML as a byte array. + * @throws IOException + * In case of problems obtaining the XHTML. */ - private byte[] getXhtml(HttpMethod aMethod) throws URIException, IOException { - // Transform the HTML into wellformed XML. - Tidy tidy = new Tidy(); - tidy.setXHTML(true); - tidy.setQuiet(true); - tidy.setShowWarnings(false); - if (_os != null) { - _os.println("Content of '" + aMethod.getURI() + "'"); - _os.println(); - } - // We write the jtidy output to XML since the DOM tree it produces is - // not namespace aware and namespace awareness is required by XSLT. - // An alternative is to configure namespace awareness of the XML parser - // in a system wide way. - Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), - _os); - DOMUtility.removeDuplicateAttributes(w3cDoc); - - ByteArrayOutputStream xhtml = new ByteArrayOutputStream(); - XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat()); - serializer.serialize(w3cDoc); - xhtml.flush(); - if (_os != null) { - _os.println(); - } - return xhtml.toByteArray(); - } + private byte[] getXhtml(HttpMethod aMethod) throws IOException { + // Transform the HTML into wellformed XML. + Tidy tidy = new Tidy(); + tidy.setXHTML(true); + tidy.setQuiet(true); + tidy.setShowWarnings(false); + + // We write the jtidy output to XML since the DOM tree it produces is + // not namespace aware and namespace awareness is required by XSLT. + // An alternative is to configure namespace awareness of the XML parser + // in a system wide way. + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), os); + DOMUtility.removeDuplicateAttributes(w3cDoc); + LOG.debug("Content of response is \n" + os.toString()); + + ByteArrayOutputStream xhtml = new ByteArrayOutputStream(); + XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat()); + serializer.serialize(w3cDoc); + xhtml.flush(); + + return xhtml.toByteArray(); + } /** * Sleeps for a random time but no more than the maximum delay. - * + * */ private void delay() { try { @@ -212,12 +221,16 @@ public abstract class AbstractPageRequest implements PageRequest { } /** - * Executes the request and follows redirects if needed. - * @param aClient HTTP client to use. - * @param aMethod Method to use. - * @return Final HTTP method used (differs from the parameter passed in in case - * of redirection). - * @throws IOException In case of network problems. + * Executes the request and follows redirects if needed. + * + * @param aClient + * HTTP client to use. + * @param aMethod + * Method to use. + * @return Final HTTP method used (differs from the parameter passed in in + * case of redirection). + * @throws IOException + * In case of network problems. */ private HttpMethod executeWithRedirects(HttpClient aClient, HttpMethod aMethod) throws IOException { @@ -235,8 +248,8 @@ public abstract class AbstractPageRequest implements PageRequest { Header header = aMethod.getResponseHeader(REDIRECT_HEADER); aMethod = new GetMethod(header.getValue()); return executeWithRedirects(aClient, aMethod); // TODO protect - // against infinite - // recursion. + // against infinite + // recursion. } default: { throw new RuntimeException("Method failed: "