--- /dev/null
+/*
+ * Copyright 2005 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.wamblee.crawler.kiss.main;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.mail.MessagingException;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.NameValuePair;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Element;
+import org.wamblee.crawler.Action;
+import org.wamblee.crawler.Configuration;
+import org.wamblee.crawler.Crawler;
+import org.wamblee.crawler.Page;
+import org.wamblee.crawler.PageException;
+import org.wamblee.crawler.impl.ConfigurationParser;
+import org.wamblee.crawler.impl.CrawlerImpl;
+import org.wamblee.crawler.kiss.guide.Channel;
+import org.wamblee.crawler.kiss.guide.PrintVisitor;
+import org.wamblee.crawler.kiss.guide.Program;
+import org.wamblee.crawler.kiss.guide.TVGuide;
+import org.wamblee.crawler.kiss.guide.Time;
+import org.wamblee.crawler.kiss.guide.TimeInterval;
+import org.wamblee.crawler.kiss.notification.NotificationException;
+import org.wamblee.crawler.kiss.notification.Notifier;
+import org.wamblee.general.BeanFactory;
+import org.wamblee.xml.ClasspathUriResolver;
+import org.wamblee.xml.XslTransformer;
+
+/**
+ * The KiSS crawler for automatic recording of interesting TV shows.
+ *
+ */
+public class KissCrawler {
+
+ private static final Log LOG = LogFactory.getLog(KissCrawler.class);
+
+ /**
+ * Start URL of the electronic programme guide.
+ */
+ private static final String START_URL = "http://epg.kml.kiss-technology.com/login.php";
+
+ /**
+ * Default socket timeout to use.
+ */
+ private static final int SOCKET_TIMEOUT = 10000;
+
+ /**
+ * Regular expression for matching time interval strings in the retrieved
+ * pages.
+ */
+ private static final String TIME_REGEX = "[^0-9]*([0-9]{2}):([0-9]{2})[^0-9]*([0-9]{2}):([0-9]{2}).*";
+
+ /**
+ * Compiled pattern for the time regular expression.
+ */
+ private Pattern _pattern;
+
+ /**
+ * Runs the KiSS crawler.
+ *
+ * @param aArgs
+ * Arguments, currently all ignored because they are hardcoded.
+ * @throws Exception
+ * In case of problems.
+ */
+ public static void main(String[] aArgs) throws Exception {
+ String crawlerConfig = new File(aArgs[0]).getCanonicalPath();
+ String programConfig = new File(aArgs[1]).getCanonicalPath();
+
+ BeanFactory factory = new StandaloneCrawlerBeanFactory();
+ Notifier notifier = factory.find(Notifier.class);
+ new KissCrawler(START_URL, SOCKET_TIMEOUT, crawlerConfig,
+ programConfig, notifier, new Report());
+ }
+
+ /**
+ * Constructs the crawler. This retrieves the TV guide by crawling the KiSS
+ * EPG guide, filters the guide for interesting programs, tries to record
+ * them, and sends a summary mail to the user.
+ *
+ * @param aCrawlerConfig
+ * Configuration file for the crawler.
+ * @param aProgramConfig
+ * Configuration file describing interesting shows.
+ * @param aNotifier
+ * Object used to send notifications of the results.
+ * @param aReport
+ * Report to use.
+ * @throws IOException
+ * In case of problems reading files.
+ * @throws NotificationException
+ * In case notification fails.
+ * @throws PageException
+ * In case of problems retrieving the TV guide.
+ */
+ public KissCrawler(String aCrawlerConfig, String aProgramConfig,
+ Notifier aNotifier, Report aReport) throws IOException,
+ NotificationException, PageException {
+ this(START_URL, SOCKET_TIMEOUT, aCrawlerConfig, aProgramConfig,
+ aNotifier, aReport);
+ }
+
+ /**
+ * Constructs the crawler. This retrieves the TV guide by crawling the KiSS
+ * EPG guide, filters the guide for interesting programs, tries to record
+ * them, and sends a summary mail to the user.
+ *
+ * @param aStartUrl
+ * Start URL of the electronic programme guide.
+ * @param aSocketTimeout
+ * Socket timeout to use.
+ * @param aCrawlerConfig
+ * Configuration file for the crawler.
+ * @param aProgramConfig
+ * Configuration file describing interesting shows.
+ * @param aNotifier
+ * Object used to send notifications of the results.
+ * @param aReport
+ * Report to use.
+ * @throws IOException
+ * In case of problems reading files.
+ * @throws NotificationException
+ * In case notification fails.
+ * @throws PageException
+ * In case of problems retrieving the TV guide.
+ */
+ public KissCrawler(String aStartUrl, int aSocketTimeout,
+ String aCrawlerConfig, String aProgramConfig, Notifier aNotifier,
+ Report aReport) throws IOException, NotificationException,
+ PageException {
+
+ _pattern = Pattern.compile(TIME_REGEX);
+
+ try {
+ HttpClient client = new HttpClient();
+ // client.getHostConfiguration().setProxy("127.0.0.1", 3128);
+ client.getParams().setParameter("http.socket.timeout",
+ SOCKET_TIMEOUT);
+
+ XslTransformer transformer = new XslTransformer(
+ new ClasspathUriResolver());
+
+ Crawler crawler = createCrawler(aCrawlerConfig, client, transformer);
+ InputStream programConfigFile = new FileInputStream(new File(
+ aProgramConfig));
+ ProgramConfigurationParser parser = new ProgramConfigurationParser();
+ parser.parse(programConfigFile);
+ List<ProgramFilter> programFilters = parser.getFilters();
+
+ try {
+ Page page = getStartPage(aStartUrl, crawler, aReport);
+ TVGuide guide = createGuide(page, aReport);
+ PrintVisitor printer = new PrintVisitor(System.out);
+ guide.accept(printer);
+ processResults(programFilters, guide, aNotifier, aReport);
+ } catch (PageException e) {
+ aReport.addMessage("Problem getting TV guide", e);
+ LOG.info("Problem getting TV guide", e);
+ throw e;
+ }
+ aNotifier.send(aReport.asXml());
+ } finally {
+ System.out.println("Crawler finished");
+ }
+ }
+
+ /**
+ * Records interesting shows.
+ *
+ * @param aProgramCondition
+ * Condition determining which shows are interesting.
+ * @param aGuide
+ * Television guide.
+ * @throws MessagingException
+ * In case of problems sending a summary mail.
+ */
+ private void processResults(List<ProgramFilter> aProgramCondition,
+ TVGuide aGuide, Notifier aNotifier, Report aReport) {
+ ProgramActionExecutor executor = new ProgramActionExecutor(aReport);
+ for (ProgramFilter filter : aProgramCondition) {
+ List<Program> programs = filter.apply(aGuide);
+ ProgramAction action = filter.getAction();
+ for (Program program : programs) {
+ action.execute(program, executor);
+ }
+ }
+ executor.commit();
+
+ }
+
+ /**
+ * Creates the crawler.
+ *
+ * @param aCrawlerConfig
+ * Crawler configuration file.
+ * @param aOs
+ * Logging output stream for the crawler.
+ * @param aClient
+ * HTTP Client to use.
+ * @return Crawler.
+ * @throws FileNotFoundException
+ * In case configuration files cannot be found.
+ */
+ private Crawler createCrawler(String aCrawlerConfig, HttpClient aClient,
+ XslTransformer aTransformer) throws FileNotFoundException {
+ ConfigurationParser parser = new ConfigurationParser(aTransformer);
+ InputStream crawlerConfigFile = new FileInputStream(new File(
+ aCrawlerConfig));
+ Configuration config = parser.parse(crawlerConfigFile);
+ Crawler crawler = new CrawlerImpl(aClient, config);
+ return crawler;
+ }
+
+ /**
+ * Gets the start page of the electronic programme guide. This involves
+ * login and navigation to a suitable start page after logging in.
+ *
+ * @param aStartUrl
+ * URL of the electronic programme guide.
+ * @param aCrawler
+ * Crawler to use.
+ * @param aReport
+ * Report to use.
+ * @return Starting page.
+ */
+ private Page getStartPage(String aStartUrl, Crawler aCrawler, Report aReport)
+ throws PageException {
+ try {
+ Page page = aCrawler.getPage(aStartUrl, new NameValuePair[0]);
+ page = page.getAction("login").execute();
+ Action favorites = page.getAction("channels-favorites");
+ if (favorites == null) {
+ String msg = "Channels favorites action not found on start page";
+ throw new PageException(msg);
+ }
+ return favorites.execute();
+ } catch (PageException e) {
+ String msg = "Could not complete login to electronic programme guide.";
+ throw new PageException(msg, e);
+ }
+ }
+
+ /**
+ * Creates the TV guide by web crawling.
+ *
+ * @param aPage
+ * Starting page.
+ * @param aReport
+ * Report to use.
+ * @return TV guide.
+ * @throws PageException
+ * In case of problem getting the tv guide.
+ */
+ private TVGuide createGuide(Page aPage, Report aReport)
+ throws PageException {
+ LOG.info("Obtaining full TV guide");
+ Action[] actions = aPage.getActions();
+ if (actions.length == 0) {
+ LOG.error("No channels found");
+ throw new PageException("No channels found");
+ }
+ List<Channel> channels = new ArrayList<Channel>();
+ for (Action action : actions) {
+ try {
+ LOG.info("Getting channel info for '" + action.getName() + "'");
+ Action tomorrow = action.execute().getAction("tomorrow");
+ if (tomorrow == null) {
+ throw new PageException("Channel summary page for '"
+ + action.getName()
+ + "' does not contain required information");
+ }
+ Channel channel = createChannel(action.getName(), tomorrow
+ .execute(), aReport);
+ channels.add(channel);
+ if (SystemProperties.isDebugMode()) {
+ break; // Only one channel is crawled.
+ }
+ } catch (PageException e) {
+ aReport.addMessage("Could not create channel information for '"
+ + action.getName() + "'");
+ LOG.error("Could not create channel information for '"
+ + action.getName() + "'", e);
+ }
+ }
+ return new TVGuide(channels);
+ }
+
+ /**
+ * Create channel information for a specific channel.
+ *
+ * @param aChannel
+ * Channel name.
+ * @param aPage
+ * Starting page for the channel.
+ * @return Channel.
+ */
+ private Channel createChannel(String aChannel, Page aPage, Report aReport) {
+ LOG.info("Obtaining program for " + aChannel);
+ Action[] programActions = aPage.getActions();
+ List<Program> programs = new ArrayList<Program>();
+ for (Action action : programActions) {
+ String time = action.getContent().element("time").getText().trim();
+ Matcher matcher = _pattern.matcher(time);
+ if (matcher.matches()) {
+ Time begin = new Time(Integer.parseInt(matcher.group(1)),
+ Integer.parseInt(matcher.group(2)));
+ Time end = new Time(Integer.parseInt(matcher.group(3)), Integer
+ .parseInt(matcher.group(4)));
+ TimeInterval interval = new TimeInterval(begin, end);
+ String description = "";
+ String keywords = "";
+
+ if (!SystemProperties.isNoProgramDetailsRequired()) {
+ Element descriptionElem = action.getContent().element(
+ "description");
+ if (descriptionElem == null) {
+ try {
+ Page programInfo = action.execute();
+ description = programInfo.getContent().element(
+ "description").getText().trim();
+ keywords = programInfo.getContent().element(
+ "keywords").getText().trim();
+ } catch (PageException e) {
+ String msg = "Program details could not be determined for '"
+ + action.getName() + "'";
+ aReport.addMessage(msg, e);
+ LOG.warn(msg, e);
+ }
+ } else {
+ description = descriptionElem.getTextTrim();
+ }
+ }
+ Program program = new Program(aChannel, action.getName(),
+ description, keywords, interval, action);
+
+ LOG.info("Got program " + program);
+ programs.add(program);
+ }
+ }
+ return new Channel(aChannel, programs);
+ }
+}
git://wamblee.org / utils / blobdiff