2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler.kiss;
19 import java.io.ByteArrayOutputStream;
21 import java.io.FileInputStream;
22 import java.io.FileNotFoundException;
23 import java.io.FileOutputStream;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.io.PrintStream;
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.Date;
30 import java.util.List;
31 import java.util.Properties;
32 import java.util.regex.Matcher;
33 import java.util.regex.Pattern;
35 import javax.mail.MessagingException;
36 import javax.mail.Session;
37 import javax.mail.internet.InternetAddress;
38 import javax.xml.transform.TransformerException;
40 import org.apache.commons.httpclient.HttpClient;
41 import org.apache.commons.logging.Log;
42 import org.apache.commons.logging.LogFactory;
43 import org.apache.commons.mail.EmailException;
44 import org.apache.commons.mail.HtmlEmail;
45 import org.apache.xml.serialize.OutputFormat;
46 import org.apache.xml.serialize.XMLSerializer;
47 import org.w3c.dom.Document;
48 import org.wamblee.crawler.Action;
49 import org.wamblee.crawler.Configuration;
50 import org.wamblee.crawler.Crawler;
51 import org.wamblee.crawler.Page;
52 import org.wamblee.crawler.PageException;
53 import org.wamblee.crawler.impl.ConfigurationParser;
54 import org.wamblee.crawler.impl.CrawlerImpl;
55 import org.wamblee.io.FileResource;
56 import org.wamblee.xml.XSLT;
59 * The KiSS crawler for automatic recording of interesting TV shows.
62 public class KissCrawler {
64 private static final Log LOG = LogFactory.getLog(KissCrawler.class);
67 * Log file name for the crawler.
69 private static final String LOG_FILE = "kiss.log";
72 * Start URL of the electronic programme guide.
74 private static final String START_URL = "http://epg.kml.kiss-technology.com/login_core.php";
77 * Crawler configuration file.
79 private static final String CRAWLER_CONFIG = "config.xml";
82 * Configuration file describing interesting programs.
84 private static final String PROGRAM_CONFIG = "programs.xml";
87 * Regular expression for matching time interval strings in the retrieved
90 private static final String TIME_REGEX = "([0-9]{2}):([0-9]{2})[^0-9]*([0-9]{2}):([0-9]{2}).*";
93 * Compiled pattern for the time regular expression.
95 private Pattern _pattern;
98 * Runs the KiSS crawler.
101 * Arguments, currently all ignored because they are hardcoded.
103 * In case of problems.
105 public static void main(String[] aArgs) throws Exception {
106 new KissCrawler(START_URL, CRAWLER_CONFIG, PROGRAM_CONFIG);
110 * Constructs the crawler. This retrieves the TV guide by crawling the KiSS
111 * EPG guide, filters the guide for interesting programs, tries to record
112 * them, and sends a summary mail to the user.
115 * Start URL of the electronic programme guide.
116 * @param aCrawlerConfig
117 * Configuration file for the crawler.
118 * @param aProgramConfig
119 * Configuration file describing interesting shows.
120 * @throws IOException
121 * In case of problems reading files.
122 * @throws MessagingException
123 * In case of problems sending a mail notification.
125 public KissCrawler(String aStartUrl, String aCrawlerConfig,
126 String aProgramConfig) throws IOException, MessagingException {
128 _pattern = Pattern.compile(TIME_REGEX);
130 FileOutputStream fos = new FileOutputStream(new File(LOG_FILE));
131 PrintStream os = new PrintStream(fos);
134 HttpClient client = new HttpClient();
135 //client.getHostConfiguration().setProxy("127.0.0.1", 3128);
137 Crawler crawler = createCrawler(aCrawlerConfig, os, client);
138 InputStream programConfigFile = new FileInputStream(new File(
140 ProgramConfigurationParser parser = new ProgramConfigurationParser();
141 parser.parse(programConfigFile);
142 List<ProgramFilter> programFilters = parser.getFilters();
144 Page page = getStartPage(aStartUrl, crawler);
145 TVGuide guide = createGuide(page);
146 PrintVisitor printer = new PrintVisitor(System.out);
147 guide.accept(printer);
148 processResults(programFilters, guide, parser.getNotifier());
152 System.out.println("Output written on '" + LOG_FILE + "'");
157 * Records interesting shows.
159 * @param aProgramCondition
160 * Condition determining which shows are interesting.
163 * @throws MessagingException
164 * In case of problems sending a summary mail.
166 private void processResults(List<ProgramFilter> aProgramCondition,
167 TVGuide aGuide, Notifier aNotifier) throws MessagingException {
168 ProgramActionExecutor executor = new ProgramActionExecutor();
169 for (ProgramFilter filter : aProgramCondition) {
170 List<Program> programs = filter.apply(aGuide);
171 ProgramAction action = filter.getAction();
172 for (Program program : programs) {
173 action.execute(program, executor);
178 aNotifier.send(executor.getXmlReport());
179 } catch (NotificationException e) {
180 throw new RuntimeException(e);
185 * Creates the crawler.
187 * @param aCrawlerConfig
188 * Crawler configuration file.
190 * Logging output stream for the crawler.
192 * HTTP Client to use.
194 * @throws FileNotFoundException
195 * In case configuration files cannot be found.
197 private Crawler createCrawler(String aCrawlerConfig, PrintStream aOs,
198 HttpClient aClient) throws FileNotFoundException {
199 ConfigurationParser parser = new ConfigurationParser(aOs);
200 InputStream crawlerConfigFile = new FileInputStream(new File(
202 Configuration config = parser.parse(crawlerConfigFile);
203 Crawler crawler = new CrawlerImpl(aClient, config);
208 * Gets the start page of the electronic programme guide. This involves
209 * login and navigation to a suitable start page after logging in.
212 * URL of the electronic programme guide.
215 * @return Starting page.
217 private Page getStartPage(String aStartUrl, Crawler aCrawler) {
219 Page page = aCrawler.getPage(aStartUrl);
220 return page.getAction("channels-favorites").execute();
221 } catch (PageException e) {
222 throw new RuntimeException(
223 "Could not login to electronic program guide", e);
228 * Creates the TV guide by web crawling.
234 private TVGuide createGuide(Page aPage) {
235 LOG.info("Obtaining full TV guide");
236 Action[] actions = aPage.getActions();
237 List<Channel> channels = new ArrayList<Channel>();
238 for (Action action : actions) {
240 LOG.info("Getting channel info for '" + action.getName() + "'");
241 Channel channel = createChannel(action.getName(), action
242 .execute().getAction("right-now").execute());
243 channels.add(channel);
244 if (SystemProperties.isDebugMode()) {
245 break; // Only one channel is crawled.
247 } catch (PageException e) {
248 LOG.error("Could not create channel information for '"
249 + action.getName() + "'", e);
252 return new TVGuide(channels);
256 * Create channel information for a specific channel.
261 * Starting page for the channel.
264 private Channel createChannel(String aChannel, Page aPage) {
265 LOG.info("Obtaining program for " + aChannel);
266 Action[] programActions = aPage.getActions();
267 List<Program> programs = new ArrayList<Program>();
268 for (Action action : programActions) {
269 String time = action.getContent().element("time").getText().trim();
270 Matcher matcher = _pattern.matcher(time);
271 if (matcher.matches()) {
272 Time begin = new Time(Integer.parseInt(matcher.group(1)),
273 Integer.parseInt(matcher.group(2)));
274 Time end = new Time(Integer.parseInt(matcher.group(3)), Integer
275 .parseInt(matcher.group(4)));
276 TimeInterval interval = new TimeInterval(begin, end);
277 String description = "";
278 String keywords = "";
279 if (!SystemProperties.isNoProgramDetailsRequired()) {
281 Page programInfo = action.execute();
282 description = programInfo.getContent().element(
283 "description").getText().trim();
284 keywords = programInfo.getContent().element("keywords")
286 } catch (PageException e) {
288 "Program details could not be determined for '"
289 + action.getName() + "'", e);
292 Program program = new Program(aChannel, action.getName(),
293 description, keywords, interval, action);
295 LOG.info("Got program " + program);
296 programs.add(program);
299 return new Channel(aChannel, programs);
303 * Sends a summary mail to the user.
307 * @throws MessagingException
308 * In case of problems sending mail.
310 private void sendMail(ProgramActionExecutor aExecutor) throws MessagingException {
311 String textReport = aExecutor.getReport();
312 System.out.println("Text report: \n" + textReport);
313 System.out.println("XML report:\n" + aExecutor.getXmlReport().asXML());
316 Properties props = new Properties();
317 props.put("mail.transport.protocol", "smtp");
318 props.put("mail.smtp.host", "falcon");
319 props.put("mail.smtp.port", "25");
321 Session mailSession = Session.getInstance(props);
322 InternetAddress from = new InternetAddress("erik@brakkee.org");
324 HtmlEmail mail = new HtmlEmail();
325 mail.setMailSession(mailSession);
327 mail.setFrom("erik@brakkee.org");
328 mail.setTo(Arrays.asList(new InternetAddress[] { from }));
329 mail.setSentDate(new Date());
330 mail.setSubject("KiSS Crawler Update");
331 String html = aExecutor.getXmlReport().asXML();
332 Document document = new XSLT().transform(html.getBytes(), new FileResource(new File("reportToHtml.xsl")));
333 ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
334 XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
335 serializer.serialize(document);
336 mail.setHtmlMsg(xhtml.toString());
337 mail.setTextMsg(textReport);
339 } catch (EmailException e) {
340 throw new RuntimeException(e);
341 } catch (TransformerException e) {
342 throw new RuntimeException(e);
343 } catch (IOException e) {
344 throw new RuntimeException(e);