2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler.kiss.main;
20 import java.io.FileInputStream;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.regex.Matcher;
27 import java.util.regex.Pattern;
29 import javax.mail.MessagingException;
31 import org.apache.commons.httpclient.HttpClient;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.wamblee.crawler.Action;
35 import org.wamblee.crawler.Configuration;
36 import org.wamblee.crawler.Crawler;
37 import org.wamblee.crawler.Page;
38 import org.wamblee.crawler.PageException;
39 import org.wamblee.crawler.impl.ConfigurationParser;
40 import org.wamblee.crawler.impl.CrawlerImpl;
41 import org.wamblee.crawler.kiss.guide.Channel;
42 import org.wamblee.crawler.kiss.guide.PrintVisitor;
43 import org.wamblee.crawler.kiss.guide.Program;
44 import org.wamblee.crawler.kiss.guide.TVGuide;
45 import org.wamblee.crawler.kiss.guide.Time;
46 import org.wamblee.crawler.kiss.guide.TimeInterval;
47 import org.wamblee.crawler.kiss.notification.NotificationException;
48 import org.wamblee.crawler.kiss.notification.Notifier;
49 import org.wamblee.xml.ClasspathUriResolver;
50 import org.wamblee.xml.XslTransformer;
53 * The KiSS crawler for automatic recording of interesting TV shows.
56 public class KissCrawler {
58 private static final Log LOG = LogFactory.getLog(KissCrawler.class);
61 * Start URL of the electronic programme guide.
63 private static final String START_URL = "http://epg.kml.kiss-technology.com/login_core.php";
66 * Default socket timeout to use.
68 private static final int SOCKET_TIMEOUT = 10000;
71 * Regular expression for matching time interval strings in the retrieved
74 private static final String TIME_REGEX = "([0-9]{2}):([0-9]{2})[^0-9]*([0-9]{2}):([0-9]{2}).*";
77 * Compiled pattern for the time regular expression.
79 private Pattern _pattern;
82 * Runs the KiSS crawler.
85 * Arguments, currently all ignored because they are hardcoded.
87 * In case of problems.
89 public static void main(String[] aArgs) throws Exception {
90 String crawlerConfig = new File(aArgs[0]).getCanonicalPath();
91 String programConfig = new File(aArgs[1]).getCanonicalPath();
92 new KissCrawler(START_URL, SOCKET_TIMEOUT, crawlerConfig, programConfig);
96 * Constructs the crawler. This retrieves the TV guide by crawling the KiSS
97 * EPG guide, filters the guide for interesting programs, tries to record
98 * them, and sends a summary mail to the user.
100 * @param aCrawlerConfig
101 * Configuration file for the crawler.
102 * @param aProgramConfig
103 * Configuration file describing interesting shows.
104 * @throws IOException
105 * In case of problems reading files.
106 * @throws MessagingException
107 * In case of problems sending a mail notification.
109 public KissCrawler(String aCrawlerConfig,
110 String aProgramConfig) throws IOException, NotificationException, PageException {
111 this(START_URL, SOCKET_TIMEOUT, aCrawlerConfig, aProgramConfig);
116 * Constructs the crawler. This retrieves the TV guide by crawling the KiSS
117 * EPG guide, filters the guide for interesting programs, tries to record
118 * them, and sends a summary mail to the user.
121 * Start URL of the electronic programme guide.
122 * @param aSocketTimeout Socket timeout to use.
123 * @param aCrawlerConfig
124 * Configuration file for the crawler.
125 * @param aProgramConfig
126 * Configuration file describing interesting shows.
127 * @throws IOException
128 * In case of problems reading files.
129 * @throws MessagingException
130 * In case of problems sending a mail notification.
132 public KissCrawler(String aStartUrl, int aSocketTimeout, String aCrawlerConfig,
133 String aProgramConfig) throws IOException, NotificationException, PageException {
135 _pattern = Pattern.compile(TIME_REGEX);
138 HttpClient client = new HttpClient();
139 // client.getHostConfiguration().setProxy("127.0.0.1", 3128);
140 client.getParams().setParameter("http.socket.timeout", SOCKET_TIMEOUT);
142 XslTransformer transformer = new XslTransformer(
143 new ClasspathUriResolver());
145 Crawler crawler = createCrawler(aCrawlerConfig, client, transformer);
146 InputStream programConfigFile = new FileInputStream(new File(
148 ProgramConfigurationParser parser = new ProgramConfigurationParser(
150 parser.parse(programConfigFile);
151 List<ProgramFilter> programFilters = parser.getFilters();
153 Report report = new Report();
156 Page page = getStartPage(aStartUrl, crawler, report);
157 TVGuide guide = createGuide(page, report);
158 PrintVisitor printer = new PrintVisitor(System.out);
159 guide.accept(printer);
160 processResults(programFilters, guide, parser.getNotifier(),
162 } catch (PageException e) {
163 report.addMessage("Problem getting TV guide", e);
164 LOG.info("Problem getting TV guide", e);
167 parser.getNotifier().send(report.asXml());
169 System.out.println("Crawler finished");
174 * Records interesting shows.
176 * @param aProgramCondition
177 * Condition determining which shows are interesting.
180 * @throws MessagingException
181 * In case of problems sending a summary mail.
183 private void processResults(List<ProgramFilter> aProgramCondition,
184 TVGuide aGuide, Notifier aNotifier, Report aReport) {
185 ProgramActionExecutor executor = new ProgramActionExecutor(aReport);
186 for (ProgramFilter filter : aProgramCondition) {
187 List<Program> programs = filter.apply(aGuide);
188 ProgramAction action = filter.getAction();
189 for (Program program : programs) {
190 action.execute(program, executor);
198 * Creates the crawler.
200 * @param aCrawlerConfig
201 * Crawler configuration file.
203 * Logging output stream for the crawler.
205 * HTTP Client to use.
207 * @throws FileNotFoundException
208 * In case configuration files cannot be found.
210 private Crawler createCrawler(String aCrawlerConfig, HttpClient aClient,
211 XslTransformer aTransformer) throws FileNotFoundException {
212 ConfigurationParser parser = new ConfigurationParser(aTransformer);
213 InputStream crawlerConfigFile = new FileInputStream(new File(
215 Configuration config = parser.parse(crawlerConfigFile);
216 Crawler crawler = new CrawlerImpl(aClient, config);
221 * Gets the start page of the electronic programme guide. This involves
222 * login and navigation to a suitable start page after logging in.
225 * URL of the electronic programme guide.
230 * @return Starting page.
232 private Page getStartPage(String aStartUrl, Crawler aCrawler, Report aReport)
233 throws PageException {
235 Page page = aCrawler.getPage(aStartUrl);
236 Action favorites = page.getAction("channels-favorites");
237 if (favorites == null) {
238 String msg = "Channels favorites action not found on start page";
239 throw new PageException(msg);
241 return favorites.execute();
242 } catch (PageException e) {
243 String msg = "Could not complete login to electronic programme guide.";
244 throw new PageException(msg, e);
249 * Creates the TV guide by web crawling.
257 private TVGuide createGuide(Page aPage, Report aReport) {
258 LOG.info("Obtaining full TV guide");
259 Action[] actions = aPage.getActions();
260 List<Channel> channels = new ArrayList<Channel>();
261 for (Action action : actions) {
263 LOG.info("Getting channel info for '" + action.getName() + "'");
264 Action rightNow = action.execute().getAction("right-now");
265 if (rightNow == null) {
266 throw new PageException("Channel summary page for '"
268 + "' does not contain required information");
270 Channel channel = createChannel(action.getName(), rightNow
271 .execute(), aReport);
272 channels.add(channel);
273 if (SystemProperties.isDebugMode()) {
274 break; // Only one channel is crawled.
276 } catch (PageException e) {
277 aReport.addMessage("Could not create channel information for '"
278 + action.getName() + "'");
279 LOG.error("Could not create channel information for '"
280 + action.getName() + "'", e);
283 return new TVGuide(channels);
287 * Create channel information for a specific channel.
292 * Starting page for the channel.
295 private Channel createChannel(String aChannel, Page aPage, Report aReport) {
296 LOG.info("Obtaining program for " + aChannel);
297 Action[] programActions = aPage.getActions();
298 List<Program> programs = new ArrayList<Program>();
299 for (Action action : programActions) {
300 String time = action.getContent().element("time").getText().trim();
301 Matcher matcher = _pattern.matcher(time);
302 if (matcher.matches()) {
303 Time begin = new Time(Integer.parseInt(matcher.group(1)),
304 Integer.parseInt(matcher.group(2)));
305 Time end = new Time(Integer.parseInt(matcher.group(3)), Integer
306 .parseInt(matcher.group(4)));
307 TimeInterval interval = new TimeInterval(begin, end);
308 String description = "";
309 String keywords = "";
310 if (!SystemProperties.isNoProgramDetailsRequired()) {
312 Page programInfo = action.execute();
313 description = programInfo.getContent().element(
314 "description").getText().trim();
315 keywords = programInfo.getContent().element("keywords")
317 } catch (PageException e) {
318 String msg = "Program details could not be determined for '"
319 + action.getName() + "'";
320 aReport.addMessage(msg, e);
324 Program program = new Program(aChannel, action.getName(),
325 description, keywords, interval, action);
327 LOG.info("Got program " + program);
328 programs.add(program);
331 return new Channel(aChannel, programs);