2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler.kiss.main;
20 import java.io.FileInputStream;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.regex.Matcher;
27 import java.util.regex.Pattern;
29 import javax.mail.MessagingException;
31 import org.apache.commons.httpclient.HttpClient;
32 import org.apache.commons.httpclient.NameValuePair;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.dom4j.Element;
36 import org.wamblee.crawler.Action;
37 import org.wamblee.crawler.Configuration;
38 import org.wamblee.crawler.Crawler;
39 import org.wamblee.crawler.Page;
40 import org.wamblee.crawler.PageException;
41 import org.wamblee.crawler.impl.ConfigurationParser;
42 import org.wamblee.crawler.impl.CrawlerImpl;
43 import org.wamblee.crawler.kiss.guide.Channel;
44 import org.wamblee.crawler.kiss.guide.PrintVisitor;
45 import org.wamblee.crawler.kiss.guide.Program;
46 import org.wamblee.crawler.kiss.guide.TVGuide;
47 import org.wamblee.crawler.kiss.guide.Time;
48 import org.wamblee.crawler.kiss.guide.TimeInterval;
49 import org.wamblee.crawler.kiss.notification.NotificationException;
50 import org.wamblee.crawler.kiss.notification.Notifier;
51 import org.wamblee.general.BeanFactory;
52 import org.wamblee.general.ClassPathHacker;
53 import org.wamblee.xml.ClasspathUriResolver;
54 import org.wamblee.xml.XslTransformer;
57 * The KiSS crawler for automatic recording of interesting TV shows.
60 public class KissCrawler {
62 private static final Log LOG = LogFactory.getLog(KissCrawler.class);
65 * Start URL of the electronic programme guide.
67 private static final String START_URL = "http://epg.kml.kiss-technology.com/login.php";
70 * Default socket timeout to use.
72 private static final int SOCKET_TIMEOUT = 10000;
75 * Regular expression for matching time interval strings in the retrieved
78 private static final String TIME_REGEX = "[^0-9]*([0-9]{2}):([0-9]{2})[^0-9]*([0-9]{2}):([0-9]{2}).*";
81 * Compiled pattern for the time regular expression.
83 private Pattern _pattern;
86 * Runs the KiSS crawler.
89 * Arguments: First argument is the crawler configuration file,
90 * and second is the program configuration file.
92 * In case of problems.
94 public static void main(String[] aArgs) throws Exception {
95 String crawlerConfig = new File(aArgs[0]).getCanonicalPath();
96 String programConfig = new File(aArgs[1]).getCanonicalPath();
98 BeanFactory factory = new StandaloneCrawlerBeanFactory();
99 Notifier notifier = factory.find(Notifier.class);
100 new KissCrawler(START_URL, SOCKET_TIMEOUT, crawlerConfig,
101 programConfig, notifier, new Report());
105 * Constructs the crawler. This retrieves the TV guide by crawling the KiSS
106 * EPG guide, filters the guide for interesting programs, tries to record
107 * them, and sends a summary mail to the user.
109 * @param aCrawlerConfig
110 * Configuration file for the crawler.
111 * @param aProgramConfig
112 * Configuration file describing interesting shows.
114 * Object used to send notifications of the results.
117 * @throws IOException
118 * In case of problems reading files.
119 * @throws NotificationException
120 * In case notification fails.
121 * @throws PageException
122 * In case of problems retrieving the TV guide.
124 public KissCrawler(String aCrawlerConfig, String aProgramConfig,
125 Notifier aNotifier, Report aReport) throws IOException,
126 NotificationException, PageException {
127 this(START_URL, SOCKET_TIMEOUT, aCrawlerConfig, aProgramConfig,
132 * Constructs the crawler. This retrieves the TV guide by crawling the KiSS
133 * EPG guide, filters the guide for interesting programs, tries to record
134 * them, and sends a summary mail to the user.
137 * Start URL of the electronic programme guide.
138 * @param aSocketTimeout
139 * Socket timeout to use.
140 * @param aCrawlerConfig
141 * Configuration file for the crawler.
142 * @param aProgramConfig
143 * Configuration file describing interesting shows.
145 * Object used to send notifications of the results.
148 * @throws IOException
149 * In case of problems reading files.
150 * @throws NotificationException
151 * In case notification fails.
152 * @throws PageException
153 * In case of problems retrieving the TV guide.
155 public KissCrawler(String aStartUrl, int aSocketTimeout,
156 String aCrawlerConfig, String aProgramConfig, Notifier aNotifier,
157 Report aReport) throws IOException, NotificationException,
160 _pattern = Pattern.compile(TIME_REGEX);
163 HttpClient client = new HttpClient();
164 // client.getHostConfiguration().setProxy("127.0.0.1", 3128);
165 client.getParams().setParameter("http.socket.timeout",
168 XslTransformer transformer = new XslTransformer(
169 new ClasspathUriResolver());
171 Crawler crawler = createCrawler(aCrawlerConfig, client, transformer);
172 InputStream programConfigFile = new FileInputStream(new File(
174 ProgramConfigurationParser parser = new ProgramConfigurationParser();
175 parser.parse(programConfigFile);
176 List<ProgramFilter> programFilters = parser.getFilters();
179 Page page = getStartPage(aStartUrl, crawler, aReport);
180 TVGuide guide = createGuide(page, aReport);
181 PrintVisitor printer = new PrintVisitor(System.out);
182 guide.accept(printer);
183 processResults(programFilters, guide, aNotifier, aReport);
184 } catch (PageException e) {
185 aReport.addMessage("Problem getting TV guide", e);
186 LOG.info("Problem getting TV guide", e);
189 aNotifier.send(aReport.asXml());
191 System.out.println("Crawler finished");
196 * Records interesting shows.
198 * @param aProgramCondition
199 * Condition determining which shows are interesting.
202 * @throws MessagingException
203 * In case of problems sending a summary mail.
205 private void processResults(List<ProgramFilter> aProgramCondition,
206 TVGuide aGuide, Notifier aNotifier, Report aReport) {
207 ProgramActionExecutor executor = new ProgramActionExecutor(aReport);
208 for (ProgramFilter filter : aProgramCondition) {
209 List<Program> programs = filter.apply(aGuide);
210 ProgramAction action = filter.getAction();
211 for (Program program : programs) {
212 action.execute(program, executor);
220 * Creates the crawler.
222 * @param aCrawlerConfig
223 * Crawler configuration file.
225 * Logging output stream for the crawler.
227 * HTTP Client to use.
229 * @throws FileNotFoundException
230 * In case configuration files cannot be found.
232 private Crawler createCrawler(String aCrawlerConfig, HttpClient aClient,
233 XslTransformer aTransformer) throws FileNotFoundException {
234 ConfigurationParser parser = new ConfigurationParser(aTransformer);
235 InputStream crawlerConfigFile = new FileInputStream(new File(
237 Configuration config = parser.parse(crawlerConfigFile);
238 Crawler crawler = new CrawlerImpl(aClient, config);
243 * Gets the start page of the electronic programme guide. This involves
244 * login and navigation to a suitable start page after logging in.
247 * URL of the electronic programme guide.
252 * @return Starting page.
254 private Page getStartPage(String aStartUrl, Crawler aCrawler, Report aReport)
255 throws PageException {
257 Page page = aCrawler.getPage(aStartUrl, new NameValuePair[0]);
258 page = page.getAction("login").execute();
259 Action favorites = page.getAction("channels-favorites");
260 if (favorites == null) {
261 String msg = "Channels favorites action not found on start page";
262 throw new PageException(msg);
264 return favorites.execute();
265 } catch (PageException e) {
266 String msg = "Could not complete login to electronic programme guide.";
267 throw new PageException(msg, e);
272 * Creates the TV guide by web crawling.
279 * @throws PageException
280 * In case of problem getting the tv guide.
282 private TVGuide createGuide(Page aPage, Report aReport)
283 throws PageException {
284 LOG.info("Obtaining full TV guide");
285 Action[] actions = aPage.getActions();
286 if (actions.length == 0) {
287 LOG.error("No channels found");
288 throw new PageException("No channels found");
290 List<Channel> channels = new ArrayList<Channel>();
291 for (Action action : actions) {
293 LOG.info("Getting channel info for '" + action.getName() + "'");
294 Action tomorrow = action.execute().getAction("tomorrow");
295 if (tomorrow == null) {
296 throw new PageException("Channel summary page for '"
298 + "' does not contain required information");
300 Channel channel = createChannel(action.getName(), tomorrow
301 .execute(), aReport);
302 channels.add(channel);
303 if (SystemProperties.isDebugMode()) {
304 break; // Only one channel is crawled.
306 } catch (PageException e) {
307 aReport.addMessage("Could not create channel information for '"
308 + action.getName() + "'");
309 LOG.error("Could not create channel information for '"
310 + action.getName() + "'", e);
313 return new TVGuide(channels);
317 * Create channel information for a specific channel.
322 * Starting page for the channel.
325 private Channel createChannel(String aChannel, Page aPage, Report aReport) {
326 LOG.info("Obtaining program for " + aChannel);
327 Action[] programActions = aPage.getActions();
328 List<Program> programs = new ArrayList<Program>();
329 for (Action action : programActions) {
330 String time = action.getContent().element("time").getText().trim();
331 Matcher matcher = _pattern.matcher(time);
332 if (matcher.matches()) {
333 Time begin = new Time(Integer.parseInt(matcher.group(1)),
334 Integer.parseInt(matcher.group(2)));
335 Time end = new Time(Integer.parseInt(matcher.group(3)), Integer
336 .parseInt(matcher.group(4)));
337 TimeInterval interval = new TimeInterval(begin, end);
338 String description = "";
339 String keywords = "";
341 if (!SystemProperties.isNoProgramDetailsRequired()) {
342 Element descriptionElem = action.getContent().element(
344 if (descriptionElem == null) {
346 Page programInfo = action.execute();
347 description = programInfo.getContent().element(
348 "description").getText().trim();
349 keywords = programInfo.getContent().element(
350 "keywords").getText().trim();
351 } catch (PageException e) {
352 String msg = "Program details could not be determined for '"
353 + action.getName() + "'";
354 aReport.addMessage(msg, e);
358 description = descriptionElem.getTextTrim();
361 Program program = new Program(aChannel, action.getName(),
362 description, keywords, interval, action);
364 LOG.info("Got program " + program);
365 programs.add(program);
368 return new Channel(aChannel, programs);