support for parameters on actions.

[utils] / crawler / basic / src / org / wamblee / crawler / AbstractPageRequest.java

diff --git a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
index 7a3755febaa6fbe8d86259500317079358567b63..2e598005ff3a453150a50884cccfd649889ee3c6 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
+++ b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
@@ -18,6 +18,9 @@ package org.wamblee.crawler;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Transformer;
@@ -39,8 +42,7 @@ import org.apache.xml.serialize.OutputFormat;
 import org.apache.xml.serialize.XMLSerializer;
 import org.w3c.dom.Document;
 import org.w3c.tidy.Tidy;
-import org.wamblee.xml.ClasspathUriResolver;
-import org.wamblee.xml.DOMUtility;
+import org.wamblee.xml.DomUtils;
 import org.wamblee.xml.XslTransformer;
 
 /**
@@ -57,8 +59,12 @@ public abstract class AbstractPageRequest implements PageRequest {
     private int _maxDelay;
 
     private NameValuePair[] _params;
+    
+    private NameValuePair[] _headers;
 
     private String _xslt;
+    
+    private XslTransformer _transformer; 
 
     /**
      * Constructs the request.
@@ -69,21 +75,28 @@ public abstract class AbstractPageRequest implements PageRequest {
      *            Maximum delay before executing a request.
      * @param aParams
      *            Request parameters to use.
+     * @param aHeaders
+     *            Request headers to use. 
      * @param aXslt
      *            XSLT used to convert the response.
      */
     protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
-            NameValuePair[] aParams, String aXslt) {
+            NameValuePair[] aParams, NameValuePair[] aHeaders, String aXslt, XslTransformer aTransformer) {
         if (aParams == null) {
             throw new IllegalArgumentException("aParams is null");
         }
+        if (aHeaders == null) {
+            throw new IllegalArgumentException("aHeaders is null");
+        }
         if (aXslt == null) {
             throw new IllegalArgumentException("aXslt is null");
         }
         _maxTries = aMaxTries;
         _maxDelay = aMaxDelay;
         _params = aParams;
+        _headers = aHeaders;
         _xslt = aXslt;
+        _transformer = aTransformer;
     }
 
     /*
@@ -98,10 +111,23 @@ public abstract class AbstractPageRequest implements PageRequest {
     /**
      * Gets the parameters for the request.
      * 
+     * @param aParams Additional parameters to use, obtained from another page, most likely as
+     *    hidden form fields. 
      * @return Request parameters.
      */
-    protected NameValuePair[] getParameters() {
-        return _params;
+    protected NameValuePair[] getParameters(NameValuePair[] aParams) {
+        List<NameValuePair> params = new ArrayList<NameValuePair>(); 
+        params.addAll(Arrays.asList(_params));
+        params.addAll(Arrays.asList(aParams));
+        return params.toArray(new NameValuePair[0]);
+    }
+    
+    /**
+     * Gets the headers for the request. 
+     * @return Request headers. 
+     */
+    protected NameValuePair[] getHeaders() { 
+        return _headers;
     }
 
     /**
@@ -120,6 +146,11 @@ public abstract class AbstractPageRequest implements PageRequest {
      */
     protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
             throws IOException, TransformerException {
+        
+        for (NameValuePair header: getHeaders()) { 
+            aMethod.setRequestHeader(header.getName(), header.getValue());
+        }
+        
         int triesLeft = _maxTries;
         while (triesLeft > 0) {
             triesLeft--;
@@ -153,10 +184,10 @@ public abstract class AbstractPageRequest implements PageRequest {
         try {
             aMethod = executeWithRedirects(aClient, aMethod);
             byte[] xhtmlData = getXhtml(aMethod);
-
-            XslTransformer xsltProcessor = new XslTransformer(new ClasspathUriResolver());
-            Document transformed = xsltProcessor.transform(xhtmlData,
-                    xsltProcessor.resolve(_xslt));
+            
+     
+            Document transformed = _transformer.transform(xhtmlData,
+                    _transformer.resolve(_xslt));
             ByteArrayOutputStream os = new ByteArrayOutputStream(); 
             Transformer transformer = TransformerFactory.newInstance()
                     .newTransformer();
@@ -167,7 +198,7 @@ public abstract class AbstractPageRequest implements PageRequest {
             LOG.debug("Transformed result is \n" + os.toString());
             return transformed;
         } catch (TransformerConfigurationException e) {
-            throw new RuntimeException(e.getMessage(), e);
+            throw new TransformerException("Transformer configuration problem", e);
         } finally {
             // Release the connection.
             aMethod.releaseConnection();
@@ -196,7 +227,7 @@ public abstract class AbstractPageRequest implements PageRequest {
         // in a system wide way.
         ByteArrayOutputStream os = new ByteArrayOutputStream(); 
         Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), os);
-        DOMUtility.removeDuplicateAttributes(w3cDoc);
+        DomUtils.removeDuplicateAttributes(w3cDoc);
         LOG.debug("Content of response is \n" + os.toString()); 
 
         ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
@@ -251,7 +282,7 @@ public abstract class AbstractPageRequest implements PageRequest {
             // recursion.
         }
         default: {
-            throw new RuntimeException("Method failed: "
+            throw new IOException("Method failed: "
                     + aMethod.getStatusLine());
         }
         }