2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler.impl;
19 import java.io.InputStream;
20 import java.io.PrintStream;
21 import java.util.ArrayList;
22 import java.util.Iterator;
23 import java.util.List;
25 import org.apache.commons.httpclient.NameValuePair;
26 import org.dom4j.Document;
27 import org.dom4j.DocumentException;
28 import org.dom4j.Element;
29 import org.dom4j.io.SAXReader;
30 import org.wamblee.crawler.Configuration;
31 import org.wamblee.crawler.GetPageRequest;
32 import org.wamblee.crawler.PageRequest;
33 import org.wamblee.crawler.PostPageRequest;
36 * Parsing of the configuration from an XML file.
38 public class ConfigurationParser {
40 private static final String ELEM_URL = "url";
41 private static final String ELEM_TYPE = "type";
42 private static final String ELEM_PATTERN = "pattern";
43 private static final String ELEM_METHOD= "method";
44 private static final String ELEM_XSLT = "xslt";
45 private static final String ELEM_PARAM = "param";
46 private static final String AT_NAME = "name";
47 private static final String AT_VALUE = "value";
49 private static final String METHOD_POST = "post";
50 private static final String METHOD_GET = "get";
52 private static final int MAX_TRIES = 3;
53 private static final int MAX_DELAY = 5000;
55 private PrintStream _os;
57 public ConfigurationParser(PrintStream aOs) {
61 public Configuration parse(InputStream aStream) {
63 SAXReader reader = new SAXReader();
64 Document document = reader.read(aStream);
66 Element root = document.getRootElement();
67 List<UrlConfig> urlConfigs = parseUrlConfigs(root);
68 List<PageTypeConfig> pageTypeConfigs = parsePageTypeConfigs(root);
69 return new ConfigurationImpl(urlConfigs, pageTypeConfigs);
70 } catch (DocumentException e) {
71 throw new RuntimeException("Problem parsing config file", e);
79 private List<UrlConfig> parseUrlConfigs(Element root) {
80 List<UrlConfig> configs = new ArrayList<UrlConfig>();
81 for (Iterator i = root.elementIterator(ELEM_URL); i.hasNext(); ) {
82 Element url = (Element)i.next();
83 UrlConfig config = parseUrlConfig(url);
89 private List<PageTypeConfig> parsePageTypeConfigs(Element root) {
90 List<PageTypeConfig> configs = new ArrayList<PageTypeConfig>();
91 for (Iterator i = root.elementIterator(ELEM_TYPE); i.hasNext(); ) {
92 Element url = (Element)i.next();
93 PageTypeConfig config = parsePageTypeConfig(url);
99 private UrlConfig parseUrlConfig(Element aUrlElem) {
100 String pattern = aUrlElem.elementText(ELEM_PATTERN);
101 PageRequest request = parseRequestConfig(aUrlElem);
102 return new UrlConfig(pattern, request);
105 private PageTypeConfig parsePageTypeConfig(Element aTypeElem) {
106 String pattern = aTypeElem.elementText(ELEM_PATTERN);
107 PageRequest request = parseRequestConfig(aTypeElem);
108 return new PageTypeConfig(pattern, request);
115 private PageRequest parseRequestConfig(Element aUrlElem) {
116 String method = aUrlElem.elementText(ELEM_METHOD);
117 String xslt = aUrlElem.elementText(ELEM_XSLT);
118 List<NameValuePair> params = new ArrayList<NameValuePair>();
119 for (Iterator i = aUrlElem.elementIterator(ELEM_PARAM); i.hasNext(); ) {
120 Element paramElem = (Element)i.next();
121 NameValuePair param = parseParameter(paramElem);
125 NameValuePair[] paramsArray = params.toArray(new NameValuePair[0]);
127 if ( METHOD_POST.equals(method)) {
128 request = new PostPageRequest(MAX_TRIES, MAX_DELAY, paramsArray, xslt, _os);
130 else if ( METHOD_GET.equals(method) || method == null ){
131 request = new GetPageRequest(MAX_TRIES, MAX_DELAY, paramsArray, xslt, _os);
133 throw new RuntimeException("Unknown request method '" + method + "'. Only " +
134 METHOD_GET + " and " + METHOD_POST + " are supported");
139 private NameValuePair parseParameter(Element aParam) {
140 String name = aParam.attributeValue(AT_NAME);
141 String value = aParam.attributeValue(AT_VALUE);
142 return new NameValuePair(name, value);