X-Git-Url: http://wamblee.org/gitweb/?a=blobdiff_plain;f=crawler%2Fbasic%2Fsrc%2Forg%2Fwamblee%2Fcrawler%2FAbstractPageRequest.java;h=fa41a680ec39d2ad0d9583de051bae337e475db8;hb=602681b15e687db2a7ea0e92796857a6df31bba2;hp=73132aa89736e8d06abd9a0e2cd1a310a4a96fa1;hpb=d4bb47fd284738756cd112b788a49caa1a9d5c38;p=utils diff --git a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java index 73132aa8..fa41a680 100644 --- a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java +++ b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java @@ -19,6 +19,7 @@ package org.wamblee.crawler; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintStream; import javax.xml.transform.OutputKeys; @@ -35,12 +36,16 @@ import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.NameValuePair; +import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.xml.serialize.OutputFormat; +import org.apache.xml.serialize.XMLSerializer; import org.w3c.dom.Document; import org.w3c.tidy.Tidy; import org.wamblee.io.FileResource; +import org.wamblee.xml.DOMUtility; import org.wamblee.xml.XSLT; /** @@ -49,127 +54,181 @@ import org.wamblee.xml.XSLT; public abstract class AbstractPageRequest implements PageRequest { private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class); + private static final String REDIRECT_HEADER = "Location"; - - private int _maxTries; - private int _maxDelay; + + private int _maxTries; + + private int _maxDelay; private NameValuePair[] _params; private String _xslt; - - private PrintStream _os; - protected AbstractPageRequest(int aMaxTries, int aMaxDelay, NameValuePair[] aParams, String aXslt, PrintStream aOs) { - if ( aParams == null ) { + private PrintStream _os; + + /** + * Constructs the request. + * @param aMaxTries Maximum retries to perform. + * @param aMaxDelay Maximum delay before executing a request. + * @param aParams Request parameters to use. + * @param aXslt XSLT used to convert the response. + * @param aOs Output stream for logging (if null then no logging is done). + */ + protected AbstractPageRequest(int aMaxTries, int aMaxDelay, + NameValuePair[] aParams, String aXslt, PrintStream aOs) { + if (aParams == null) { throw new IllegalArgumentException("aParams is null"); } - if ( aXslt == null ) { + if (aXslt == null) { throw new IllegalArgumentException("aXslt is null"); } _maxTries = aMaxTries; _maxDelay = aMaxDelay; _params = aParams; _xslt = aXslt; - _os = aOs; + _os = aOs; } - - /* (non-Javadoc) + + /* + * (non-Javadoc) + * * @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String) */ public void overrideXslt(String aXslt) { - _xslt = aXslt; + _xslt = aXslt; } - - protected NameValuePair[] getParameters() { + + /** + * Gets the parameters for the request. + * @return Request parameters. + */ + protected NameValuePair[] getParameters() { return _params; } - - protected Document executeMethod(HttpClient client, HttpMethod method) throws TransformerException { - int triesLeft = _maxTries; - while ( triesLeft > 0 ) { + + /** + * Executes the request with a random delay and with a maximum number of + * retries. + * @param aClient HTTP client to use. + * @param aMethod Method representing the request. + * @return XML document describing the response. + * @throws TransformerException In case transformation of the HTML to XML fails. + */ + protected Document executeMethod(HttpClient aClient, HttpMethod aMethod) + throws TransformerException { + int triesLeft = _maxTries; + while (triesLeft > 0) { triesLeft--; - try { - return executeMethodWithoutRetries(client, method); - } catch (TransformerException e) { - if ( triesLeft == 0 ) { + try { + return executeMethodWithoutRetries(aClient, aMethod); + } catch (TransformerException e) { + if (triesLeft == 0) { throw e; } } } throw new RuntimeException("Code should never reach this point"); } - - protected Document executeMethodWithoutRetries(HttpClient client, HttpMethod method) throws TransformerException { + /** + * Executes the request without doing any retries in case XSLT transformation + * fails. + * @param aClient HTTP client to use. + * @param aMethod Method to execute. + * @return XML document containing the result. + * @throws TransformerException In case transformation of the result to XML fails. + */ + protected Document executeMethodWithoutRetries(HttpClient aClient, + HttpMethod aMethod) throws TransformerException { try { - // Execute the method. - method = executeWithRedirects(client, method); - - // Transform the HTML into wellformed XML. - Tidy tidy = new Tidy(); - tidy.setXHTML(true); - tidy.setQuiet(true); - tidy.setShowWarnings(false); - if ( _os != null ) { - _os.println("Content of '" + method.getURI() + "'"); - _os.println(); - } - // We let jtidy produce raw output because the DOM it produces is - // is not namespace aware. We let the XSLT processor parse the XML again - // to ensure that the XSLT uses a namespace aware DOM tree. An alternative - // is to configure namespace awareness of the XML parser in a system wide way. - ByteArrayOutputStream xhtml = new ByteArrayOutputStream(); - tidy.parse(method.getResponseBodyAsStream(), xhtml); - _os.print(new String(xhtml.toByteArray())); - // Obtaining the XML as dom is not used. - //Document w3cDoc = tidy.parseDOM(method.getResponseBodyAsStream(), - // _os); - if ( _os != null ) { - _os.println(); - } - xhtml.flush(); - byte[] xhtmlData = xhtml.toByteArray(); - Document transformed = new XSLT().transform(xhtmlData, new FileResource(new File(_xslt))); + aMethod = executeWithRedirects(aClient, aMethod); + byte[] xhtmlData = getXhtml(aMethod); + + Document transformed = new XSLT().transform(xhtmlData, + new FileResource(new File(_xslt))); _os.println("Transformed result is: "); - Transformer transformer = TransformerFactory.newInstance().newTransformer(); + Transformer transformer = TransformerFactory.newInstance() + .newTransformer(); transformer.setParameter(OutputKeys.INDENT, "yes"); transformer.setParameter(OutputKeys.METHOD, "xml"); - transformer.transform(new DOMSource(transformed), new StreamResult(_os)); - + transformer.transform(new DOMSource(transformed), new StreamResult( + _os)); + return transformed; - } catch (HttpException e) { - throw new RuntimeException(e.getMessage(), e); - } catch (IOException e) { + } catch (HttpException e) { + throw new RuntimeException(e.getMessage(), e); + } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); - } catch (TransformerConfigurationException e) { + } catch (TransformerConfigurationException e) { throw new RuntimeException(e.getMessage(), e); } finally { // Release the connection. - method.releaseConnection(); + aMethod.releaseConnection(); } } - - private void delay() { + + /** + * Gets the result of the HTTP method as an XHTML document. + * @param aMethod Method to invoke. + * @return XHTML as a byte array. + * @throws URIException In case of poblems with the URI + * @throws IOException In case of problems obtaining the XHTML. + */ + private byte[] getXhtml(HttpMethod aMethod) throws URIException, IOException { + // Transform the HTML into wellformed XML. + Tidy tidy = new Tidy(); + tidy.setXHTML(true); + tidy.setQuiet(true); + tidy.setShowWarnings(false); + if (_os != null) { + _os.println("Content of '" + aMethod.getURI() + "'"); + _os.println(); + } + // We write the jtidy output to XML since the DOM tree it produces is + // not namespace aware and namespace awareness is required by XSLT. + // An alternative is to configure namespace awareness of the XML parser + // in a system wide way. + Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), + _os); + DOMUtility.removeDuplicateAttributes(w3cDoc); + + ByteArrayOutputStream xhtml = new ByteArrayOutputStream(); + XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat()); + serializer.serialize(w3cDoc); + xhtml.flush(); + if (_os != null) { + _os.println(); + } + return xhtml.toByteArray(); + } + + /** + * Sleeps for a random time but no more than the maximum delay. + * + */ + private void delay() { try { - Thread.sleep((long)((float)_maxDelay* Math.random())); - } catch (InterruptedException e) { - // + Thread.sleep((long) ((float) _maxDelay * Math.random())); + } catch (InterruptedException e) { + return; // to satisfy checkstyle } } - /** - * @param aClient - * @param aMethod - * @throws IOException - * @throws HttpException + * Executes the request and follows redirects if needed. + * @param aClient HTTP client to use. + * @param aMethod Method to use. + * @return Final HTTP method used (differs from the parameter passed in in case + * of redirection). + * @throws IOException In case of network problems. */ - private HttpMethod executeWithRedirects(HttpClient aClient, HttpMethod aMethod) throws IOException, HttpException { + private HttpMethod executeWithRedirects(HttpClient aClient, + HttpMethod aMethod) throws IOException { delay(); int statusCode = aClient.executeMethod(aMethod); - switch (statusCode) { + switch (statusCode) { case HttpStatus.SC_OK: { return aMethod; } @@ -179,7 +238,9 @@ public abstract class AbstractPageRequest implements PageRequest { aMethod.releaseConnection(); Header header = aMethod.getResponseHeader(REDIRECT_HEADER); aMethod = new GetMethod(header.getValue()); - return executeWithRedirects(aClient, aMethod); // TODO protect against infinite recursion. + return executeWithRedirects(aClient, aMethod); // TODO protect + // against infinite + // recursion. } default: { throw new RuntimeException("Method failed: "