support for parameters on actions.

diff --git a/crawler/basic/src/org/wamblee/crawler/impl/ActionImpl.java b/crawler/basic/src/org/wamblee/crawler/impl/ActionImpl.java
index 0ff4252d98804712981f46f842748e0bc024215b..ab7068cbda32ee8e8895a6677a2c6db431c8f60f 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/impl/ActionImpl.java
+++ b/crawler/basic/src/org/wamblee/crawler/impl/ActionImpl.java
@@ -16,6 +16,7 @@
 
 package org.wamblee.crawler.impl;
 
+import org.apache.commons.httpclient.NameValuePair;
 import org.dom4j.Element;
 import org.wamblee.crawler.Action;
 import org.wamblee.crawler.Crawler;
@@ -37,6 +38,8 @@ public class ActionImpl implements Action {
     private String _reference;
 
     private PageType _type;
+    
+    private NameValuePair[] _parameters; 
 
     /**
      * Constructs the action.
@@ -50,14 +53,16 @@ public class ActionImpl implements Action {
      *            Name of the action.
      * @param aReference
      *            URL of the reference.
+     * @param aParameters Parameters to use for the action. 
      */
     public ActionImpl(Crawler aCrawler, Element aContent, String aName,
-            String aReference) {
+            String aReference, NameValuePair[] aParameters) {
         _crawler = aCrawler;
         _content = aContent;
         _name = aName;
         _reference = aReference;
         _type = null;
+        _parameters = aParameters; 
     }
 
     /**
@@ -74,14 +79,16 @@ public class ActionImpl implements Action {
      *            URL of the reference.
      * @param aType
      *            Type of the referenced page.
+     * @param aParameters Parameters to use. 
      */
     public ActionImpl(Crawler aCrawler, Element aContent, String aName,
-            String aReference, PageType aType) {
+            String aReference, PageType aType, NameValuePair[] aParameters) {
         _crawler = aCrawler;
         _content = aContent;
         _name = aName;
         _reference = aReference;
         _type = aType;
+        _parameters = aParameters; 
     }
 
     /*
@@ -100,9 +107,9 @@ public class ActionImpl implements Action {
      */
     public Page execute() throws PageException {
         if (_type == null) {
-            return _crawler.getPage(_reference);
+            return _crawler.getPage(_reference, _parameters);
         }
-        return _crawler.getPage(_reference, _type);
+        return _crawler.getPage(_reference, _parameters, _type);
     }
 
     /*

diff --git a/crawler/basic/src/org/wamblee/crawler/impl/App.java b/crawler/basic/src/org/wamblee/crawler/impl/App.java
index f9b9bd45eef23de75507357193d2b4d7c01f6d9f..b0339f4b07ad7d37b765a7e0c79beda0ef5a0cae 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/impl/App.java
+++ b/crawler/basic/src/org/wamblee/crawler/impl/App.java
@@ -5,6 +5,7 @@ import java.io.FileInputStream;
 import java.io.InputStream;
 
 import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.NameValuePair;
 import org.dom4j.Element;
 import org.wamblee.crawler.Action;
 import org.wamblee.crawler.Configuration;
@@ -65,7 +66,7 @@ public final class App {
         Crawler crawler = new CrawlerImpl(client, config);
 
         System.out.println("Retrieving: " + starturl);
-        Page page = crawler.getPage(starturl);
+        Page page = crawler.getPage(starturl, new NameValuePair[0]);
         showPage(page);
         page = page.getAction("channels-favorites").execute();
         recordInterestingShows(page);

diff --git a/crawler/basic/src/org/wamblee/crawler/impl/CrawlerImpl.java b/crawler/basic/src/org/wamblee/crawler/impl/CrawlerImpl.java
index 0188ad31c94d2a8eb8a2d2e040fb7af3fd90f122..098ed91f42ee30d072b2ce5d5339e37d3e8a2726 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/impl/CrawlerImpl.java
+++ b/crawler/basic/src/org/wamblee/crawler/impl/CrawlerImpl.java
@@ -17,6 +17,7 @@
 package org.wamblee.crawler.impl;
 
 import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.NameValuePair;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.dom4j.Element;
@@ -58,11 +59,11 @@ public class CrawlerImpl implements Crawler {
      * 
      * @see org.wamblee.crawler.Crawler#getPage(java.lang.String)
      */
-    public Page getPage(String aUrl) throws PageException {
+    public Page getPage(String aUrl, NameValuePair[] aParams) throws PageException {
         LOG.debug("Getting page: url = '" + aUrl + "'");
         PageRequest request = _config.getRequest(aUrl);
-        Document content = request.execute(aUrl, _client);
-        return transformToDom4jDoc(content);
+        Document content = request.execute(aUrl, aParams, _client);
+        return transformToDom4jDoc(aUrl, content);
     }
 
     /*
@@ -71,11 +72,11 @@ public class CrawlerImpl implements Crawler {
      * @see org.wamblee.crawler.Crawler#getPage(java.lang.String,
      *      java.lang.String)
      */
-    public Page getPage(String aUrl, PageType aType) throws PageException {
+    public Page getPage(String aUrl, NameValuePair[] aParams, PageType aType) throws PageException {
         LOG.debug("Getting page: url = '" + aUrl + "', type = '" + aType + "'");
         PageRequest request = _config.getRequest(aType);
-        Document content = request.execute(aUrl, _client);
-        return transformToDom4jDoc(content);
+        Document content = request.execute(aUrl, aParams, _client);
+        return transformToDom4jDoc(aUrl, content);
     }
 
     /**
@@ -83,13 +84,13 @@ public class CrawlerImpl implements Crawler {
      * @param content DOM document. 
      * @return
      */
-    private Page transformToDom4jDoc(Document content) {
+    private Page transformToDom4jDoc(String aUrl, Document content) {
         DOMReader reader = new DOMReader();
         org.dom4j.Document dom4jDoc = reader.read(content);
         Element root = dom4jDoc.getRootElement();
         dom4jDoc.remove(root);
 
-        return new PageImpl(this, replaceReferencesWithContent(root));
+        return new PageImpl(aUrl, this, replaceReferencesWithContent(root));
     }
 
     /**

diff --git a/crawler/basic/src/org/wamblee/crawler/impl/PageImpl.java b/crawler/basic/src/org/wamblee/crawler/impl/PageImpl.java
index d478ed4b73f1a11181d46d16aa57cab7114710dd..1265bc2933ddb18f3c0e12cdce91cb40e0dc01a6 100644 (file)
--- a/crawler/basic/src/org/wamblee/crawler/impl/PageImpl.java
+++ b/crawler/basic/src/org/wamblee/crawler/impl/PageImpl.java
@@ -16,9 +16,12 @@
 
 package org.wamblee.crawler.impl;
 
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.httpclient.NameValuePair;
 import org.dom4j.DocumentHelper;
 import org.dom4j.Element;
 import org.dom4j.XPath;
@@ -39,7 +42,13 @@ public class PageImpl implements Page {
     private static final String ATT_HREF = "reference";
 
     private static final String ATT_TYPE = "type";
+    
+    private static final String ELEM_PARAM = "param"; 
+    
+    private static final String ATT_VALUE = "value";
 
+    private String _href; 
+    
     private Crawler _crawler;
 
     private Element _content;
@@ -51,7 +60,8 @@ public class PageImpl implements Page {
      * 
      * @param aContent
      */
-    public PageImpl(Crawler aCrawler, Element aContent) {
+    public PageImpl(String aHref, Crawler aCrawler, Element aContent) {
+        _href = aHref;
         _crawler = aCrawler;
         _content = aContent;
         _actions = computeActions();
@@ -70,15 +80,44 @@ public class PageImpl implements Page {
             String name = elem.attributeValue(ATT_NAME);
             String href = elem.attributeValue(ATT_HREF);
             String type = elem.attributeValue(ATT_TYPE);
+            NameValuePair[] params = getMandatoryParameters(elem);
+            href = absolutizeHref(_href, href); 
             if (type == null) {
-                names.add(new ActionImpl(_crawler, elem, name, href));
+                names.add(new ActionImpl(_crawler, elem, name, href, params));
             } else {
                 names.add(new ActionImpl(_crawler, elem, name, href,
-                        new PageType(type)));
+                        new PageType(type), params));
             }
         }
         return names.toArray(new Action[0]);
     }
+    
+    /**
+     * Absolutize the hyperlink 
+     * @param aPageHref Absolute page reference.
+     * @param aLinkHref Possibly relative link reference.
+     * @return Absolute hyperlink. 
+     */
+    private String absolutizeHref(String aPageHref, String aLinkHref) {
+        
+        try {
+            URL pageUrl = new URL(aPageHref);
+            URL newUrl = new URL(pageUrl, aLinkHref); 
+            return newUrl.toString(); // TODO need to use URL instead of String throughout the code. 
+        } catch (MalformedURLException e) {
+            throw new RuntimeException("Malformed URL", e); 
+        }
+    }
+
+    private NameValuePair[] getMandatoryParameters(Element aAction) {
+        List<NameValuePair> result = new ArrayList<NameValuePair>();
+        for (Element param: (List<Element>)aAction.elements(ELEM_PARAM)) { 
+            String name = param.attributeValue(ATT_NAME); 
+            String value = param.attributeValue(ATT_VALUE); 
+            result.add(new NameValuePair(name, value));
+        }
+        return result.toArray(new NameValuePair[0]);
+    }
 
     /*
      * (non-Javadoc)