added support for configuring request headers.

Required since kiss checks for this header now.

diff --git a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
index 7607f764b3c20999842d14089e22cc52ee61ee40..28482d7fbd096fb8641db821cfcbaab339841726 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
+++ b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java
@@ -56,6 +56,8 @@ public abstract class AbstractPageRequest implements PageRequest {
     private int _maxDelay;
 
     private NameValuePair[] _params;
+    
+    private NameValuePair[] _headers;
 
     private String _xslt;
     
@@ -70,20 +72,26 @@ public abstract class AbstractPageRequest implements PageRequest {
      *            Maximum delay before executing a request.
      * @param aParams
      *            Request parameters to use.
+     * @param aHeaders
+     *            Request headers to use. 
      * @param aXslt
      *            XSLT used to convert the response.
      */
     protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
-            NameValuePair[] aParams, String aXslt, XslTransformer aTransformer) {
+            NameValuePair[] aParams, NameValuePair[] aHeaders, String aXslt, XslTransformer aTransformer) {
         if (aParams == null) {
             throw new IllegalArgumentException("aParams is null");
         }
+        if (aHeaders == null) {
+            throw new IllegalArgumentException("aHeaders is null");
+        }
         if (aXslt == null) {
             throw new IllegalArgumentException("aXslt is null");
         }
         _maxTries = aMaxTries;
         _maxDelay = aMaxDelay;
         _params = aParams;
+        _headers = aHeaders;
         _xslt = aXslt;
         _transformer = aTransformer;
     }
@@ -105,6 +113,14 @@ public abstract class AbstractPageRequest implements PageRequest {
     protected NameValuePair[] getParameters() {
         return _params;
     }
+    
+    /**
+     * Gets the headers for the request. 
+     * @return Request headers. 
+     */
+    protected NameValuePair[] getHeaders() { 
+        return _headers;
+    }
 
     /**
      * Executes the request with a random delay and with a maximum number of
@@ -122,6 +138,11 @@ public abstract class AbstractPageRequest implements PageRequest {
      */
     protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
             throws IOException, TransformerException {
+        
+        for (NameValuePair header: getHeaders()) { 
+            aMethod.setRequestHeader(header.getName(), header.getValue());
+        }
+        
         int triesLeft = _maxTries;
         while (triesLeft > 0) {
             triesLeft--;
@@ -155,7 +176,7 @@ public abstract class AbstractPageRequest implements PageRequest {
         try {
             aMethod = executeWithRedirects(aClient, aMethod);
             byte[] xhtmlData = getXhtml(aMethod);
-
+     
             Document transformed = _transformer.transform(xhtmlData,
                     _transformer.resolve(_xslt));
             ByteArrayOutputStream os = new ByteArrayOutputStream(); 

diff --git a/crawler/basic/src/org/wamblee/crawler/GetPageRequest.java b/crawler/basic/src/org/wamblee/crawler/GetPageRequest.java
index ac1f93018e70e64b770afb1bc273f48efc752808..40a3421167fafb0f0a3f08e19fbb994912177bc2 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/GetPageRequest.java
+++ b/crawler/basic/src/org/wamblee/crawler/GetPageRequest.java
@@ -37,10 +37,12 @@ public class GetPageRequest extends AbstractPageRequest {
      * @param aMaxTries Maximum number of retries. 
      * @param aMaxDelay Maximum delay before executing the request.
      * @param aParams Request parameters to use. 
+     * @param aHeaders Request headers to use. 
      * @param aXslt XSLT to use. 
      */
-    public GetPageRequest(int aMaxTries, int aMaxDelay, NameValuePair[] aParams, String aXslt, XslTransformer aTransformer) {
-        super(aMaxTries, aMaxDelay, aParams, aXslt, aTransformer);
+    public GetPageRequest(int aMaxTries, int aMaxDelay, NameValuePair[] aParams, 
+            NameValuePair[] aHeaders, String aXslt, XslTransformer aTransformer) {
+        super(aMaxTries, aMaxDelay, aParams, aHeaders, aXslt, aTransformer);
     }
     
     /*

diff --git a/crawler/basic/src/org/wamblee/crawler/PostPageRequest.java b/crawler/basic/src/org/wamblee/crawler/PostPageRequest.java
index a090627f0cebdb51c3cd7e3e80c97a5392b38c90..af160f60cfa581b180a5d9b2cc576da897705218 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/PostPageRequest.java
+++ b/crawler/basic/src/org/wamblee/crawler/PostPageRequest.java
@@ -36,10 +36,14 @@ public class PostPageRequest extends AbstractPageRequest {
      * @param aMaxTries Maximum number of retries. 
      * @param aMaxDelay Maximum delay before executing the request.
      * @param aParams Request parameters to use. 
+     * @param aHeaders Request headers to use. 
      * @param aXslt XSLT to use.  
      */
-    public PostPageRequest(int aMaxTries, int aMaxDelay, NameValuePair[] aParams, String aXslt, XslTransformer aTransformer) {
-        super(aMaxTries, aMaxDelay, aParams, aXslt, aTransformer);
+    public PostPageRequest(int aMaxTries, int aMaxDelay, 
+            NameValuePair[] aParams, 
+            NameValuePair[] aHeaders,
+            String aXslt, XslTransformer aTransformer) {
+        super(aMaxTries, aMaxDelay, aParams, aHeaders, aXslt, aTransformer);
     }
 
     /*

diff --git a/crawler/basic/src/org/wamblee/crawler/impl/ConfigurationParser.java b/crawler/basic/src/org/wamblee/crawler/impl/ConfigurationParser.java
index 7e15d4a3ae90c151a371f98214f48c3a78caf2ca..3069104d07ec9174d7eeef969a5efcc1905006e7 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/impl/ConfigurationParser.java
+++ b/crawler/basic/src/org/wamblee/crawler/impl/ConfigurationParser.java
@@ -48,6 +48,8 @@ public class ConfigurationParser {
     private static final String ELEM_XSLT = "xslt";
 
     private static final String ELEM_PARAM = "param";
+    
+    private static final String ELEM_HEADER = "header";
 
     private static final String AT_NAME = "name";
 
@@ -149,20 +151,17 @@ public class ConfigurationParser {
     private PageRequest parseRequestConfig(Element aElem) {
         String method = aElem.elementText(ELEM_METHOD);
         String xslt = aElem.elementText(ELEM_XSLT);
-        List<NameValuePair> params = new ArrayList<NameValuePair>();
-        for (Iterator i = aElem.elementIterator(ELEM_PARAM); i.hasNext();) {
-            Element paramElem = (Element) i.next();
-            NameValuePair param = parseParameter(paramElem);
-            params.add(param);
-        }
-
+        List<NameValuePair> params = parseNameValuePairs(aElem, ELEM_PARAM); 
+        List<NameValuePair> headers = parseNameValuePairs(aElem, ELEM_HEADER);
+        
         NameValuePair[] paramsArray = params.toArray(new NameValuePair[0]);
+        NameValuePair[] headersArray = headers.toArray(new NameValuePair[0]);
         PageRequest request;
         if (METHOD_POST.equals(method)) {
-            request = new PostPageRequest(MAX_TRIES, MAX_DELAY, paramsArray,
+            request = new PostPageRequest(MAX_TRIES, MAX_DELAY, paramsArray, headersArray,
                     xslt, _transformer);
         } else if (METHOD_GET.equals(method) || method == null) {
-            request = new GetPageRequest(MAX_TRIES, MAX_DELAY, paramsArray,
+            request = new GetPageRequest(MAX_TRIES, MAX_DELAY, paramsArray, headersArray,
                     xslt, _transformer);
         } else {
             throw new RuntimeException("Unknown request method '" + method
@@ -172,6 +171,20 @@ public class ConfigurationParser {
         return request;
     }
 
+    /**
+     * @param aElem
+     * @return
+     */
+    private List<NameValuePair> parseNameValuePairs(Element aElem, String aElemName) {
+        List<NameValuePair> headers = new ArrayList<NameValuePair>();
+        for (Iterator i = aElem.elementIterator(aElemName); i.hasNext();) {
+            Element paramElem = (Element) i.next();
+            NameValuePair header = parseParameter(paramElem);
+            headers.add(header);
+        }
+        return headers;
+    }
+
     /**
      * Parses a parameter definition. 
      * @param aParam Parameter.