2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler.impl;
19 import java.io.InputStream;
20 import java.util.ArrayList;
21 import java.util.Iterator;
22 import java.util.List;
24 import org.apache.commons.httpclient.NameValuePair;
25 import org.dom4j.Document;
26 import org.dom4j.DocumentException;
27 import org.dom4j.Element;
28 import org.dom4j.io.SAXReader;
29 import org.wamblee.crawler.Configuration;
30 import org.wamblee.crawler.GetPageRequest;
31 import org.wamblee.crawler.PageRequest;
32 import org.wamblee.crawler.PostPageRequest;
33 import org.wamblee.xml.XslTransformer;
36 * Parsing of the configuration from an XML file.
38 * @author Erik Brakkee
40 public class ConfigurationParser {
42 private static final String ELEM_URL = "url";
44 private static final String ELEM_TYPE = "type";
46 private static final String ELEM_PATTERN = "pattern";
48 private static final String ELEM_METHOD = "method";
50 private static final String ELEM_XSLT = "xslt";
52 private static final String ELEM_PARAM = "param";
54 private static final String ELEM_HEADER = "header";
56 private static final String AT_NAME = "name";
58 private static final String AT_VALUE = "value";
60 private static final String METHOD_POST = "post";
62 private static final String METHOD_GET = "get";
64 private static final int MAX_TRIES = 3;
66 private static final int MAX_DELAY = 10000;
68 private XslTransformer _transformer;
71 * Constructs the configuration parser.
73 public ConfigurationParser(XslTransformer aTransformer) {
74 _transformer = aTransformer;
78 * Parses the configuration from an input stream.
79 * @param aStream Input file.
80 * @return Configuration.
82 public Configuration parse(InputStream aStream) {
84 SAXReader reader = new SAXReader();
85 Document document = reader.read(aStream);
87 Element root = document.getRootElement();
88 List<UrlConfig> urlConfigs = parseUrlConfigs(root);
89 List<PageTypeConfig> pageTypeConfigs = parsePageTypeConfigs(root);
90 return new ConfigurationImpl(urlConfigs, pageTypeConfigs);
91 } catch (DocumentException e) {
92 throw new RuntimeException("Problem parsing config file", e);
97 * Parses the URL-based configuration.
98 * @param aRoot Root of the configuration file document.
99 * @return List of URL-based configurations.
101 private List<UrlConfig> parseUrlConfigs(Element aRoot) {
102 List<UrlConfig> configs = new ArrayList<UrlConfig>();
103 for (Iterator i = aRoot.elementIterator(ELEM_URL); i.hasNext();) {
104 Element url = (Element) i.next();
105 UrlConfig config = parseUrlConfig(url);
112 * Parses the page type based configurations.
113 * @param aRoot Root of the configuration file document.
114 * @return LIst of page type based configurations.
116 private List<PageTypeConfig> parsePageTypeConfigs(Element aRoot) {
117 List<PageTypeConfig> configs = new ArrayList<PageTypeConfig>();
118 for (Iterator i = aRoot.elementIterator(ELEM_TYPE); i.hasNext();) {
119 Element url = (Element) i.next();
120 PageTypeConfig config = parsePageTypeConfig(url);
127 * Parses a URL-based configuration.
128 * @param aUrlElem Configuration element.
129 * @return Configuration.
131 private UrlConfig parseUrlConfig(Element aUrlElem) {
132 String pattern = aUrlElem.elementText(ELEM_PATTERN);
133 PageRequest request = parseRequestConfig(aUrlElem);
134 return new UrlConfig(pattern, request);
138 * Parses a page type based configuration.
139 * @param aTypeElem Configuration element.
140 * @return Configuration.
142 private PageTypeConfig parsePageTypeConfig(Element aTypeElem) {
143 String pattern = aTypeElem.elementText(ELEM_PATTERN);
144 PageRequest request = parseRequestConfig(aTypeElem);
145 return new PageTypeConfig(pattern, request);
149 * Parses a request configuration describing how to execute requests.
150 * @param aElem Configuration element.
151 * @return Page request.
153 private PageRequest parseRequestConfig(Element aElem) {
154 String method = aElem.elementText(ELEM_METHOD);
155 String xslt = aElem.elementText(ELEM_XSLT);
156 List<NameValuePair> params = parseNameValuePairs(aElem, ELEM_PARAM);
157 List<NameValuePair> headers = parseNameValuePairs(aElem, ELEM_HEADER);
159 NameValuePair[] paramsArray = params.toArray(new NameValuePair[0]);
160 NameValuePair[] headersArray = headers.toArray(new NameValuePair[0]);
162 if (METHOD_POST.equals(method)) {
163 request = new PostPageRequest(MAX_TRIES, MAX_DELAY, paramsArray, headersArray,
165 } else if (METHOD_GET.equals(method) || method == null) {
166 request = new GetPageRequest(MAX_TRIES, MAX_DELAY, paramsArray, headersArray,
169 throw new RuntimeException("Unknown request method '" + method
170 + "'. Only " + METHOD_GET + " and " + METHOD_POST
180 private List<NameValuePair> parseNameValuePairs(Element aElem, String aElemName) {
181 List<NameValuePair> headers = new ArrayList<NameValuePair>();
182 for (Iterator i = aElem.elementIterator(aElemName); i.hasNext();) {
183 Element paramElem = (Element) i.next();
184 NameValuePair header = parseParameter(paramElem);
191 * Parses a parameter definition.
192 * @param aParam Parameter.
193 * @return Name value pair describing a parameter.
195 private NameValuePair parseParameter(Element aParam) {
196 String name = aParam.attributeValue(AT_NAME);
197 String value = aParam.attributeValue(AT_VALUE);
198 return new NameValuePair(name, value);