2 * Copyright 2005 the original author or authors.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.wamblee.crawler.kiss;
19 import java.io.ByteArrayOutputStream;
21 import java.io.FileInputStream;
22 import java.io.FileNotFoundException;
23 import java.io.FileOutputStream;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.io.PrintStream;
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.Date;
30 import java.util.List;
31 import java.util.Properties;
32 import java.util.regex.Matcher;
33 import java.util.regex.Pattern;
35 import javax.mail.MessagingException;
36 import javax.mail.Session;
37 import javax.mail.internet.InternetAddress;
38 import javax.xml.transform.TransformerException;
40 import org.apache.commons.httpclient.HttpClient;
41 import org.apache.commons.logging.Log;
42 import org.apache.commons.logging.LogFactory;
43 import org.apache.commons.mail.EmailException;
44 import org.apache.commons.mail.HtmlEmail;
45 import org.apache.xml.serialize.OutputFormat;
46 import org.apache.xml.serialize.XMLSerializer;
47 import org.w3c.dom.Document;
48 import org.wamblee.crawler.Action;
49 import org.wamblee.crawler.Configuration;
50 import org.wamblee.crawler.Crawler;
51 import org.wamblee.crawler.Page;
52 import org.wamblee.crawler.PageException;
53 import org.wamblee.crawler.impl.ConfigurationParser;
54 import org.wamblee.crawler.impl.CrawlerImpl;
55 import org.wamblee.io.FileResource;
56 import org.wamblee.xml.XSLT;
59 * The KiSS crawler for automatic recording of interesting TV shows.
62 public class KissCrawler {
64 private static final Log LOG = LogFactory.getLog(KissCrawler.class);
67 * Log file name for the crawler.
69 private static final String LOG_FILE = "kiss.log";
72 * Start URL of the electronic programme guide.
74 private static final String START_URL = "http://epg.kml.kiss-technology.com/login_core.php";
77 * Crawler configuration file.
79 private static final String CRAWLER_CONFIG = "config.xml";
82 * Configuration file describing interesting programs.
84 private static final String PROGRAM_CONFIG = "programs.xml";
87 * Regular expression for matching time interval strings in the retrieved
90 private static final String TIME_REGEX = "([0-9]{2}):([0-9]{2})[^0-9]*([0-9]{2}):([0-9]{2}).*";
93 * Compiled pattern for the time regular expression.
95 private Pattern _pattern;
98 * Runs the KiSS crawler.
101 * Arguments, currently all ignored because they are hardcoded.
103 * In case of problems.
105 public static void main(String[] aArgs) throws Exception {
106 new KissCrawler(START_URL, CRAWLER_CONFIG, PROGRAM_CONFIG);
110 * Constructs the crawler. This retrieves the TV guide by crawling the KiSS
111 * EPG guide, filters the guide for interesting programs, tries to record
112 * them, and sends a summary mail to the user.
115 * Start URL of the electronic programme guide.
116 * @param aCrawlerConfig
117 * Configuration file for the crawler.
118 * @param aProgramConfig
119 * Configuration file describing interesting shows.
120 * @throws IOException
121 * In case of problems reading files.
122 * @throws MessagingException
123 * In case of problems sending a mail notification.
125 public KissCrawler(String aStartUrl, String aCrawlerConfig,
126 String aProgramConfig) throws IOException, MessagingException {
128 _pattern = Pattern.compile(TIME_REGEX);
130 FileOutputStream fos = new FileOutputStream(new File(LOG_FILE));
131 PrintStream os = new PrintStream(fos);
134 HttpClient client = new HttpClient();
135 //client.getHostConfiguration().setProxy("127.0.0.1", 3128);
137 Crawler crawler = createCrawler(aCrawlerConfig, os, client);
138 InputStream programConfigFile = new FileInputStream(new File(
140 ProgramConfigurationParser parser = new ProgramConfigurationParser();
141 parser.parse(programConfigFile);
142 List<ProgramFilter> programFilters = parser.getFilters();
144 Page page = getStartPage(aStartUrl, crawler);
145 TVGuide guide = createGuide(page);
146 PrintVisitor printer = new PrintVisitor(System.out);
147 guide.accept(printer);
148 processResults(programFilters, guide, parser.getNotifier());
152 System.out.println("Output written on '" + LOG_FILE + "'");
157 * Records interesting shows.
159 * @param aProgramCondition
160 * Condition determining which shows are interesting.
163 * @throws MessagingException
164 * In case of problems sending a summary mail.
166 private void processResults(List<ProgramFilter> aProgramCondition,
167 TVGuide aGuide, Notifier aNotifier) throws MessagingException {
168 ProgramActionExecutor executor = new ProgramActionExecutor();
169 for (ProgramFilter filter : aProgramCondition) {
170 List<Program> programs = filter.apply(aGuide);
171 ProgramAction action = filter.getAction();
172 for (Program program : programs) {
173 action.execute(program, executor);
178 aNotifier.send(executor.getXmlReport());
179 } catch (NotificationException e) {
180 throw new RuntimeException(e);
186 * Creates the crawler.
188 * @param aCrawlerConfig
189 * Crawler configuration file.
191 * Logging output stream for the crawler.
193 * HTTP Client to use.
195 * @throws FileNotFoundException
196 * In case configuration files cannot be found.
198 private Crawler createCrawler(String aCrawlerConfig, PrintStream aOs,
199 HttpClient aClient) throws FileNotFoundException {
200 ConfigurationParser parser = new ConfigurationParser(aOs);
201 InputStream crawlerConfigFile = new FileInputStream(new File(
203 Configuration config = parser.parse(crawlerConfigFile);
204 Crawler crawler = new CrawlerImpl(aClient, config);
209 * Gets the start page of the electronic programme guide. This involves
210 * login and navigation to a suitable start page after logging in.
213 * URL of the electronic programme guide.
216 * @return Starting page.
218 private Page getStartPage(String aStartUrl, Crawler aCrawler) {
220 Page page = aCrawler.getPage(aStartUrl);
221 return page.getAction("channels-favorites").execute();
222 } catch (PageException e) {
223 throw new RuntimeException(
224 "Could not login to electronic program guide", e);
229 * Creates the TV guide by web crawling.
235 private TVGuide createGuide(Page aPage) {
236 LOG.info("Obtaining full TV guide");
237 Action[] actions = aPage.getActions();
238 List<Channel> channels = new ArrayList<Channel>();
239 for (Action action : actions) {
241 LOG.info("Getting channel info for '" + action.getName() + "'");
242 Channel channel = createChannel(action.getName(), action
243 .execute().getAction("right-now").execute());
244 channels.add(channel);
245 if (SystemProperties.isDebugMode()) {
246 break; // Only one channel is crawled.
248 } catch (PageException e) {
249 LOG.error("Could not create channel information for '"
250 + action.getName() + "'", e);
253 return new TVGuide(channels);
257 * Create channel information for a specific channel.
262 * Starting page for the channel.
265 private Channel createChannel(String aChannel, Page aPage) {
266 LOG.info("Obtaining program for " + aChannel);
267 Action[] programActions = aPage.getActions();
268 List<Program> programs = new ArrayList<Program>();
269 for (Action action : programActions) {
270 String time = action.getContent().element("time").getText().trim();
271 Matcher matcher = _pattern.matcher(time);
272 if (matcher.matches()) {
273 Time begin = new Time(Integer.parseInt(matcher.group(1)),
274 Integer.parseInt(matcher.group(2)));
275 Time end = new Time(Integer.parseInt(matcher.group(3)), Integer
276 .parseInt(matcher.group(4)));
277 TimeInterval interval = new TimeInterval(begin, end);
278 String description = "";
279 String keywords = "";
280 if (!SystemProperties.isNoProgramDetailsRequired()) {
282 Page programInfo = action.execute();
283 description = programInfo.getContent().element(
284 "description").getText().trim();
285 keywords = programInfo.getContent().element("keywords")
287 } catch (PageException e) {
289 "Program details could not be determined for '"
290 + action.getName() + "'", e);
293 Program program = new Program(aChannel, action.getName(),
294 description, keywords, interval, action);
296 LOG.info("Got program " + program);
297 programs.add(program);
300 return new Channel(aChannel, programs);
304 * Sends a summary mail to the user.
308 * @throws MessagingException
309 * In case of problems sending mail.
311 private void sendMail(ProgramActionExecutor aExecutor) throws MessagingException {
312 String textReport = aExecutor.getReport();
313 System.out.println("Text report: \n" + textReport);
314 System.out.println("XML report:\n" + aExecutor.getXmlReport().asXML());
317 Properties props = new Properties();
318 props.put("mail.transport.protocol", "smtp");
319 props.put("mail.smtp.host", "falcon");
320 props.put("mail.smtp.port", "25");
322 Session mailSession = Session.getInstance(props);
323 InternetAddress from = new InternetAddress("erik@brakkee.org");
325 HtmlEmail mail = new HtmlEmail();
326 mail.setMailSession(mailSession);
328 mail.setFrom("erik@brakkee.org");
329 mail.setTo(Arrays.asList(new InternetAddress[] { from }));
330 mail.setSentDate(new Date());
331 mail.setSubject("KiSS Crawler Update");
332 String html = aExecutor.getXmlReport().asXML();
333 Document document = new XSLT().transform(html.getBytes(), new FileResource(new File("reportToHtml.xsl")));
334 ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
335 XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
336 serializer.serialize(document);
337 mail.setHtmlMsg(xhtml.toString());
338 mail.setTextMsg(textReport);
340 } catch (EmailException e) {
341 throw new RuntimeException(e);
342 } catch (TransformerException e) {
343 throw new RuntimeException(e);
344 } catch (IOException e) {
345 throw new RuntimeException(e);