import org.wamblee.crawler.GetPageRequest;
import org.wamblee.crawler.PageRequest;
import org.wamblee.crawler.PostPageRequest;
+import org.wamblee.xml.XslTransformer;
/**
* Parsing of the configuration from an XML file.
private static final String ELEM_XSLT = "xslt";
private static final String ELEM_PARAM = "param";
+
+ private static final String ELEM_HEADER = "header";
private static final String AT_NAME = "name";
private static final int MAX_TRIES = 3;
- private static final int MAX_DELAY = 100;
+ private static final int MAX_DELAY = 1000;
+
+ private XslTransformer _transformer;
/**
* Constructs the configuration parser.
*/
- public ConfigurationParser() {
- // Empty
+ public ConfigurationParser(XslTransformer aTransformer) {
+ _transformer = aTransformer;
}
/**
private PageRequest parseRequestConfig(Element aElem) {
String method = aElem.elementText(ELEM_METHOD);
String xslt = aElem.elementText(ELEM_XSLT);
- List<NameValuePair> params = new ArrayList<NameValuePair>();
- for (Iterator i = aElem.elementIterator(ELEM_PARAM); i.hasNext();) {
- Element paramElem = (Element) i.next();
- NameValuePair param = parseParameter(paramElem);
- params.add(param);
- }
-
+ List<NameValuePair> params = parseNameValuePairs(aElem, ELEM_PARAM);
+ List<NameValuePair> headers = parseNameValuePairs(aElem, ELEM_HEADER);
+
NameValuePair[] paramsArray = params.toArray(new NameValuePair[0]);
+ NameValuePair[] headersArray = headers.toArray(new NameValuePair[0]);
PageRequest request;
if (METHOD_POST.equals(method)) {
- request = new PostPageRequest(MAX_TRIES, MAX_DELAY, paramsArray,
- xslt);
+ request = new PostPageRequest(MAX_TRIES, MAX_DELAY, paramsArray, headersArray,
+ xslt, _transformer);
} else if (METHOD_GET.equals(method) || method == null) {
- request = new GetPageRequest(MAX_TRIES, MAX_DELAY, paramsArray,
- xslt);
+ request = new GetPageRequest(MAX_TRIES, MAX_DELAY, paramsArray, headersArray,
+ xslt, _transformer);
} else {
throw new RuntimeException("Unknown request method '" + method
+ "'. Only " + METHOD_GET + " and " + METHOD_POST
return request;
}
+ /**
+ * @param aElem
+ * @return
+ */
+ private List<NameValuePair> parseNameValuePairs(Element aElem, String aElemName) {
+ List<NameValuePair> headers = new ArrayList<NameValuePair>();
+ for (Iterator i = aElem.elementIterator(aElemName); i.hasNext();) {
+ Element paramElem = (Element) i.next();
+ NameValuePair header = parseParameter(paramElem);
+ headers.add(header);
+ }
+ return headers;
+ }
+
/**
* Parses a parameter definition.
* @param aParam Parameter.