X-Git-Url: http://wamblee.org/gitweb/?a=blobdiff_plain;f=crawler%2Fbasic%2Fsrc%2Forg%2Fwamblee%2Fcrawler%2Fimpl%2FConfigurationParser.java;h=b45053a2ab953c32cdec4a4d954220f86e0e1974;hb=8bdf7301b21a7824933fac2b75caf410b7dd5923;hp=2f2e5f5bf95cdf9cc866c182551da757de820769;hpb=9d2a49c65872cd9330670a3cace19faf493df04d;p=utils

diff --git a/crawler/basic/src/org/wamblee/crawler/impl/ConfigurationParser.java b/crawler/basic/src/org/wamblee/crawler/impl/ConfigurationParser.java
index 2f2e5f5b..b45053a2 100644
--- a/crawler/basic/src/org/wamblee/crawler/impl/ConfigurationParser.java
+++ b/crawler/basic/src/org/wamblee/crawler/impl/ConfigurationParser.java
@@ -59,14 +59,23 @@ public class ConfigurationParser {
 
     private static final int MAX_TRIES = 3;
 
-    private static final int MAX_DELAY = 5000;
+    private static final int MAX_DELAY = 100;
 
     private PrintStream _os;
-
+    
+    /**
+     * Constructs the configuration parser. 
+     * @param aOs The stream for logging requests
+     */
     public ConfigurationParser(PrintStream aOs) {
         _os = aOs;
     }
 
+    /**
+     * Parses the configuration from an input stream.
+     * @param aStream Input file. 
+     * @return Configuration. 
+     */
     public Configuration parse(InputStream aStream) {
         try {
             SAXReader reader = new SAXReader();
@@ -82,8 +91,9 @@ public class ConfigurationParser {
     }
 
     /**
-     * @param aRoot
-     * @return
+     * Parses the URL-based configuration. 
+     * @param aRoot Root of the configuration file document. 
+     * @return List of URL-based configurations. 
      */
     private List<UrlConfig> parseUrlConfigs(Element aRoot) {
         List<UrlConfig> configs = new ArrayList<UrlConfig>();
@@ -95,6 +105,11 @@ public class ConfigurationParser {
         return configs;
     }
 
+    /**
+     * Parses the page type based configurations. 
+     * @param aRoot Root of the configuration file document. 
+     * @return LIst of page type based configurations. 
+     */
     private List<PageTypeConfig> parsePageTypeConfigs(Element aRoot) {
         List<PageTypeConfig> configs = new ArrayList<PageTypeConfig>();
         for (Iterator i = aRoot.elementIterator(ELEM_TYPE); i.hasNext();) {
@@ -105,12 +120,22 @@ public class ConfigurationParser {
         return configs;
     }
 
+    /**
+     * Parses a URL-based configuration. 
+     * @param aUrlElem Configuration element. 
+     * @return Configuration. 
+     */
     private UrlConfig parseUrlConfig(Element aUrlElem) {
         String pattern = aUrlElem.elementText(ELEM_PATTERN);
         PageRequest request = parseRequestConfig(aUrlElem);
         return new UrlConfig(pattern, request);
     }
 
+    /**
+     * Parses a page type based configuration. 
+     * @param aTypeElem Configuration element. 
+     * @return Configuration. 
+     */
     private PageTypeConfig parsePageTypeConfig(Element aTypeElem) {
         String pattern = aTypeElem.elementText(ELEM_PATTERN);
         PageRequest request = parseRequestConfig(aTypeElem);
@@ -118,14 +143,15 @@ public class ConfigurationParser {
     }
 
     /**
-     * @param aUrlElem
-     * @return
+     * Parses a request configuration describing how to execute requests. 
+     * @param aElem Configuration element. 
+     * @return Page request. 
      */
-    private PageRequest parseRequestConfig(Element aUrlElem) {
-        String method = aUrlElem.elementText(ELEM_METHOD);
-        String xslt = aUrlElem.elementText(ELEM_XSLT);
+    private PageRequest parseRequestConfig(Element aElem) {
+        String method = aElem.elementText(ELEM_METHOD);
+        String xslt = aElem.elementText(ELEM_XSLT);
         List<NameValuePair> params = new ArrayList<NameValuePair>();
-        for (Iterator i = aUrlElem.elementIterator(ELEM_PARAM); i.hasNext();) {
+        for (Iterator i = aElem.elementIterator(ELEM_PARAM); i.hasNext();) {
             Element paramElem = (Element) i.next();
             NameValuePair param = parseParameter(paramElem);
             params.add(param);
@@ -135,10 +161,10 @@ public class ConfigurationParser {
         PageRequest request;
         if (METHOD_POST.equals(method)) {
             request = new PostPageRequest(MAX_TRIES, MAX_DELAY, paramsArray,
-                    xslt, _os);
+                    xslt);
         } else if (METHOD_GET.equals(method) || method == null) {
             request = new GetPageRequest(MAX_TRIES, MAX_DELAY, paramsArray,
-                    xslt, _os);
+                    xslt);
         } else {
             throw new RuntimeException("Unknown request method '" + method
                     + "'. Only " + METHOD_GET + " and " + METHOD_POST
@@ -147,6 +173,11 @@ public class ConfigurationParser {
         return request;
     }
 
+    /**
+     * Parses a parameter definition. 
+     * @param aParam Parameter. 
+     * @return Name value pair describing a parameter. 
+     */
     private NameValuePair parseParameter(Element aParam) {
         String name = aParam.attributeValue(AT_NAME);
         String value = aParam.attributeValue(AT_VALUE);