(no commit message)

[utils] / crawler / basic / src / org / wamblee / crawler / AbstractPageRequest.java

diff --git a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
index 304772043f9aea11852c7b47d196390c8ed7c69d..b37834bb2fa61b5c93ca18460b1d804274d2bcf3 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
+++ b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
@@ -31,16 +31,19 @@ import javax.xml.transform.stream.StreamResult;
 
 import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
 import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.NameValuePair;
+import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.xml.serialize.OutputFormat;
+import org.apache.xml.serialize.XMLSerializer;
 import org.w3c.dom.Document;
 import org.w3c.tidy.Tidy;
 import org.wamblee.io.FileResource;
+import org.wamblee.xml.DOMUtility;
 import org.wamblee.xml.XSLT;
 
 /**
@@ -107,11 +110,12 @@ public abstract class AbstractPageRequest implements PageRequest {
      * retries. 
      * @param aClient HTTP client to use. 
      * @param aMethod Method representing the request. 
-     * @return XML document describing the response. 
+     * @return XML document describing the response.
+     * @throws IOException In case of IO problems.  
      * @throws TransformerException In case transformation of the HTML to XML fails.
      */
     protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
-            throws TransformerException {
+            throws IOException, TransformerException {
         int triesLeft = _maxTries;
         while (triesLeft > 0) {
             triesLeft--;
@@ -132,41 +136,15 @@ public abstract class AbstractPageRequest implements PageRequest {
      * @param aClient HTTP client to use. 
      * @param aMethod Method to execute. 
      * @return XML document containing the result. 
+     * @throws IOException In case of IO problems. 
      * @throws TransformerException In case transformation of the result to XML fails.
      */
     protected Document executeMethodWithoutRetries(HttpClient aClient,
-            HttpMethod aMethod) throws TransformerException {
+            HttpMethod aMethod) throws IOException, TransformerException {
         try {
-            // Execute the method.
             aMethod = executeWithRedirects(aClient, aMethod);
-
-            // Transform the HTML into wellformed XML.
-            Tidy tidy = new Tidy();
-            tidy.setXHTML(true);
-            tidy.setQuiet(true);
-            tidy.setShowWarnings(false);
-            if (_os != null) {
-                _os.println("Content of '" + aMethod.getURI() + "'");
-                _os.println();
-            }
-            // We let jtidy produce raw output because the DOM it produces is
-            // is not namespace aware. We let the XSLT processor parse the XML
-            // again
-            // to ensure that the XSLT uses a namespace aware DOM tree. An
-            // alternative
-            // is to configure namespace awareness of the XML parser in a system
-            // wide way.
-            ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
-            tidy.parse(aMethod.getResponseBodyAsStream(), xhtml);
-            _os.print(new String(xhtml.toByteArray()));
-            // Obtaining the XML as dom is not used.
-            // Document w3cDoc = tidy.parseDOM(method.getResponseBodyAsStream(),
-            // _os);
-            if (_os != null) {
-                _os.println();
-            }
-            xhtml.flush();
-            byte[] xhtmlData = xhtml.toByteArray();
+            byte[] xhtmlData = getXhtml(aMethod);
+        
             Document transformed = new XSLT().transform(xhtmlData,
                     new FileResource(new File(_xslt)));
             _os.println("Transformed result is: ");
@@ -178,10 +156,6 @@ public abstract class AbstractPageRequest implements PageRequest {
                     _os));
 
             return transformed;
-        } catch (HttpException e) {
-            throw new RuntimeException(e.getMessage(), e);
-        } catch (IOException e) {
-            throw new RuntimeException(e.getMessage(), e);
         } catch (TransformerConfigurationException e) {
             throw new RuntimeException(e.getMessage(), e);
         } finally {
@@ -190,6 +164,41 @@ public abstract class AbstractPageRequest implements PageRequest {
         }
     }
 
+    /**
+     * Gets the result of the HTTP method as an XHTML document. 
+     * @param aMethod Method to invoke.
+     * @return XHTML as a byte array.  
+     * @throws URIException In case of poblems with the URI
+     * @throws IOException In case of problems obtaining the XHTML.
+     */
+       private byte[] getXhtml(HttpMethod aMethod) throws URIException, IOException {
+               // Transform the HTML into wellformed XML.
+               Tidy tidy = new Tidy();
+               tidy.setXHTML(true);
+               tidy.setQuiet(true);
+               tidy.setShowWarnings(false);
+               if (_os != null) {
+                   _os.println("Content of '" + aMethod.getURI() + "'");
+                   _os.println();
+               }
+               // We write the jtidy output to XML since the DOM tree it produces is
+               // not namespace aware and namespace awareness is required by XSLT.
+               // An alternative is to configure namespace awareness of the XML parser 
+               // in a system wide way. 
+               Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(),
+                  _os);
+               DOMUtility.removeDuplicateAttributes(w3cDoc);
+               
+               ByteArrayOutputStream xhtml = new ByteArrayOutputStream(); 
+               XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
+               serializer.serialize(w3cDoc);
+               xhtml.flush();
+               if (_os != null) {
+           _os.println();
+        }
+               return xhtml.toByteArray();
+       }
+
     /**
      * Sleeps for a random time but no more than the maximum delay.
      *