From 6345295b7993feca996f225e12f390702c4ff274 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 22 Mar 2006 21:36:11 +0000 Subject: [PATCH 01/16] --- .../wamblee/conditions/AndConditionTest.java | 62 +++++++++++++++++++ .../wamblee/conditions/OrConditionTest.java | 3 - 2 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 support/test/org/wamblee/conditions/AndConditionTest.java diff --git a/support/test/org/wamblee/conditions/AndConditionTest.java b/support/test/org/wamblee/conditions/AndConditionTest.java new file mode 100644 index 00000000..3ada4b4a --- /dev/null +++ b/support/test/org/wamblee/conditions/AndConditionTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2005 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wamblee.conditions; + +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +/** + * Tests the Or Condition. + */ +public class AndConditionTest extends TestCase { + + public void checkResult(boolean aFirst, boolean aSecond, boolean aResult) { + AndCondition and = new AndCondition(new FixedCondition(aFirst), + new FixedCondition(aSecond)); + assertEquals(aResult, and.matches(0)); + } + + public void checkResult(boolean[] aValues, boolean aResult) { + List> conditions = new ArrayList>(); + for (boolean value: aValues) { + conditions.add(new FixedCondition(value)); + } + AndCondition and = new AndCondition(conditions); + assertEquals(aResult, and.matches(new Integer(0))); + } + + /** + * Checks all combinations of two conditions. + * + */ + public void testTwoConditions() { + checkResult(false, false, false); + checkResult(true, false, false); + checkResult(false, true, false); + checkResult(true, true, true); + } + + public void testMultipleConditions() { + checkResult(new boolean[]{ false, false, false} , false); + checkResult(new boolean[]{ true, false, false }, false); + checkResult(new boolean[]{ false, true, false }, false); + checkResult(new boolean[]{ false, false, true }, false); + checkResult(new boolean[]{ true, true, true }, true); + } +} diff --git a/support/test/org/wamblee/conditions/OrConditionTest.java b/support/test/org/wamblee/conditions/OrConditionTest.java index 26541ed7..2fad3110 100644 --- a/support/test/org/wamblee/conditions/OrConditionTest.java +++ b/support/test/org/wamblee/conditions/OrConditionTest.java @@ -26,9 +26,6 @@ import junit.framework.TestCase; */ public class OrConditionTest extends TestCase { - private Condition _false = new FixedCondition(false); - private Condition _true = new FixedCondition(true); - public void checkResult(boolean aFirst, boolean aSecond, boolean aResult) { OrCondition or = new OrCondition(new FixedCondition(aFirst), new FixedCondition(aSecond)); -- 2.31.1 From 158c9636c3d84019ae124d3cbe2da75be7cc1d40 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 22 Mar 2006 21:53:58 +0000 Subject: [PATCH 02/16] --- .../PropertyRegexConditionTest.java | 77 +++++++++++++++++++ .../test/org/wamblee/conditions/TestBean.java | 37 +++++++++ 2 files changed, 114 insertions(+) create mode 100644 support/test/org/wamblee/conditions/PropertyRegexConditionTest.java create mode 100644 support/test/org/wamblee/conditions/TestBean.java diff --git a/support/test/org/wamblee/conditions/PropertyRegexConditionTest.java b/support/test/org/wamblee/conditions/PropertyRegexConditionTest.java new file mode 100644 index 00000000..5830666c --- /dev/null +++ b/support/test/org/wamblee/conditions/PropertyRegexConditionTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2005 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wamblee.conditions; + +import junit.framework.TestCase; + +/** + * Tests {@link org.wamblee.conditions.PropertyRegexCondition}. + */ +public class PropertyRegexConditionTest extends TestCase { + + private boolean match(String aProperty, String aRegex, boolean aToLower, TestBean aBean) { + PropertyRegexCondition condition = new PropertyRegexCondition(aProperty, aRegex, aToLower ); + return condition.matches(aBean); + } + + private void checkMatch(String aProperty, String aRegex, boolean aToLower, TestBean aBean, boolean aResult) { + assertEquals( aResult, match(aProperty, aRegex, aToLower, aBean)); + } + + /** + * Verifies correct matching behavior for several cases. + * + */ + public void testMatchProperty() { + TestBean bean = new TestBean("Hallo"); + checkMatch("value", "Hallo", false, bean, true); + checkMatch("value", "all", false, bean, false); + checkMatch("value", ".a.*o", false, bean, true); + checkMatch("value", "hallo", false, bean, false); // no match when not converting to lower case. + checkMatch("value", "hallo", true, bean, true); // match! + } + + /** + * Uses property regex condition for non-existing property. + * Verifies that a runtime exception is thrown. + * + */ + public void testWrongProperty() { + TestBean bean = new TestBean("Hallo"); + try { + match("bla", ".*", false, bean); + } catch (RuntimeException e) { + return; // ok + } + fail(); + } + + /** + * Applies condition to a private property. Verifies that a runtime + * exception is thrown. + * + */ + public void testPrivateProperty() { + TestBean bean = new TestBean("Hallo"); + try { + match("privateValue", ".*", false, bean); + } catch (RuntimeException e) { + return; // ok + } + fail(); + } +} diff --git a/support/test/org/wamblee/conditions/TestBean.java b/support/test/org/wamblee/conditions/TestBean.java new file mode 100644 index 00000000..14162139 --- /dev/null +++ b/support/test/org/wamblee/conditions/TestBean.java @@ -0,0 +1,37 @@ +/* + * Copyright 2005 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wamblee.conditions; + +/** + * + */ +public class TestBean { + + private String _value; + + public TestBean(String aValue) { + _value = aValue; + } + + public String getValue() { + return _value; + } + + private String getPrivateValue() { + return _value; + } +} -- 2.31.1 From b32609e869c53140bbc02d3c34afe4bf34d61da1 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 22 Mar 2006 22:00:54 +0000 Subject: [PATCH 03/16] some extra logging. --- .../wamblee/crawler/kiss/main/ProgramActionExecutor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java index b9c103b7..44132b5f 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java @@ -22,6 +22,8 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.dom4j.DocumentFactory; import org.dom4j.Element; import org.wamblee.crawler.kiss.guide.Program; @@ -33,6 +35,8 @@ import org.wamblee.crawler.kiss.guide.Program.RecordingResult; * in what order and makes decisions in case of conflicts. */ public class ProgramActionExecutor { + + private static final Log LOG = LogFactory.getLog(ProgramActionExecutor.class); /** * A map of category name to a set of program. Useful for displaying the @@ -74,6 +78,7 @@ public class ProgramActionExecutor { * Program to record. */ public void recordProgram(int aPriority, Program aProgram) { + LOG.info("priority = " + aPriority + ", program: " + aProgram); _showsToRecord.add(aProgram); } @@ -86,6 +91,7 @@ public class ProgramActionExecutor { * Program. */ public void interestingProgram(String aCategory, Program aProgram) { + LOG.info("category = '" + aCategory + "', program: " + aProgram); Set programs = _interestingShows.get(aCategory); if (programs == null) { programs = new TreeSet(new Program.TimeSorter()); -- 2.31.1 From 59528dfd9877eca88ab0426bac6c26b2d6fe886d Mon Sep 17 00:00:00 2001 From: erik Date: Thu, 23 Mar 2006 20:18:45 +0000 Subject: [PATCH 04/16] Added priority mechanism. Corrected stylesheets to deal with recording conflicts as well, this status was missing. --- .../src/org/wamblee/crawler/impl/App.java | 2 - crawler/kiss/conf/kiss/programs.xml | 2 + crawler/kiss/conf/kiss/reportToHtml.xsl | 3 ++ crawler/kiss/conf/kiss/reportToText.xsl | 5 ++ .../kiss/main/ProgramActionExecutor.java | 52 +++++++++++++++---- .../kiss/main/ProgramConfigurationParser.java | 15 ++++-- .../kiss/main/RecordProgramAction.java | 4 +- 7 files changed, 65 insertions(+), 18 deletions(-) diff --git a/crawler/basic/src/org/wamblee/crawler/impl/App.java b/crawler/basic/src/org/wamblee/crawler/impl/App.java index c1af31b2..d4ca4709 100644 --- a/crawler/basic/src/org/wamblee/crawler/impl/App.java +++ b/crawler/basic/src/org/wamblee/crawler/impl/App.java @@ -5,8 +5,6 @@ import java.io.FileInputStream; import java.io.InputStream; import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.dom4j.Element; import org.wamblee.crawler.Action; import org.wamblee.crawler.Configuration; diff --git a/crawler/kiss/conf/kiss/programs.xml b/crawler/kiss/conf/kiss/programs.xml index 82de82fa..57f14cf6 100644 --- a/crawler/kiss/conf/kiss/programs.xml +++ b/crawler/kiss/conf/kiss/programs.xml @@ -48,10 +48,12 @@ + 10 battlestar + 10 star trek diff --git a/crawler/kiss/conf/kiss/reportToHtml.xsl b/crawler/kiss/conf/kiss/reportToHtml.xsl index f3901ea3..9017431d 100644 --- a/crawler/kiss/conf/kiss/reportToHtml.xsl +++ b/crawler/kiss/conf/kiss/reportToHtml.xsl @@ -44,6 +44,9 @@ Already recorded programs + + Conflicts with other recorded program + Programs that could not be recorded for technical reasons. diff --git a/crawler/kiss/conf/kiss/reportToText.xsl b/crawler/kiss/conf/kiss/reportToText.xsl index e46e4328..9d85cfaa 100644 --- a/crawler/kiss/conf/kiss/reportToText.xsl +++ b/crawler/kiss/conf/kiss/reportToText.xsl @@ -33,6 +33,11 @@ + + Conflicts with other recorded program + + + Programs that could not be recorded for technical reasons. diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java index 44132b5f..4b2476d0 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java @@ -17,6 +17,7 @@ package org.wamblee.crawler.kiss.main; import java.util.EnumMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -27,6 +28,7 @@ import org.apache.commons.logging.LogFactory; import org.dom4j.DocumentFactory; import org.dom4j.Element; import org.wamblee.crawler.kiss.guide.Program; +import org.wamblee.crawler.kiss.guide.TimeInterval; import org.wamblee.crawler.kiss.guide.Program.RecordingResult; /** @@ -35,8 +37,9 @@ import org.wamblee.crawler.kiss.guide.Program.RecordingResult; * in what order and makes decisions in case of conflicts. */ public class ProgramActionExecutor { - - private static final Log LOG = LogFactory.getLog(ProgramActionExecutor.class); + + private static final Log LOG = LogFactory + .getLog(ProgramActionExecutor.class); /** * A map of category name to a set of program. Useful for displaying the @@ -45,9 +48,9 @@ public class ProgramActionExecutor { private Map> _interestingShows; /** - * Set of programs to record. + * Map of priority to set of programs. */ - private Set _showsToRecord; + private Map> _showsToRecord; /** * Map or recording result to a set of programs. @@ -60,7 +63,7 @@ public class ProgramActionExecutor { */ public ProgramActionExecutor() { _interestingShows = new TreeMap>(); - _showsToRecord = new TreeSet(new Program.TimeSorter()); + _showsToRecord = new TreeMap>(); _recordings = new EnumMap>( RecordingResult.class); for (RecordingResult result : RecordingResult.values()) { @@ -79,7 +82,14 @@ public class ProgramActionExecutor { */ public void recordProgram(int aPriority, Program aProgram) { LOG.info("priority = " + aPriority + ", program: " + aProgram); - _showsToRecord.add(aProgram); + // Putting -priority into the set makes sure that iteration order + // over the priorities will go from higher priority to lower priority. + Set programs = _showsToRecord.get(-aPriority); + if (programs == null) { + programs = new TreeSet(new Program.TimeSorter()); + _showsToRecord.put(-aPriority, programs); + } + programs.add(aProgram); } /** @@ -105,10 +115,34 @@ public class ProgramActionExecutor { * */ public void commit() { - for (Program program : _showsToRecord) { - RecordingResult result = program.record(); - _recordings.get(result).add(program); + Set previouslyRecorded = new HashSet(); + for (Integer priority : _showsToRecord.keySet()) { + for (Program program : _showsToRecord.get(priority)) { + TimeInterval interval = program.getInterval(); + if ( recordingConflictExists(previouslyRecorded, interval)) { + _recordings.get(RecordingResult.CONFLICT).add(program); + } else { + RecordingResult result = program.record(); + _recordings.get(result).add(program); + previouslyRecorded.add(interval); + } + } + } + } + + /** + * Checks an interval for overlap with a previously recorded program. + * @param aPreviouslyRecorded Previously recorded programs. + * @param interval Interval. + * @return True iff there is a recording conflict. + */ + private boolean recordingConflictExists(Set aPreviouslyRecorded, TimeInterval interval) { + for (TimeInterval recordedInterval: aPreviouslyRecorded ) { + if ( interval.overlap(recordedInterval)) { + return true; + } } + return false; } /** diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramConfigurationParser.java b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramConfigurationParser.java index 103ec070..c16f4f93 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramConfigurationParser.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramConfigurationParser.java @@ -38,11 +38,9 @@ import org.wamblee.crawler.kiss.notification.Notifier; * Parse the configuration of desired programs. */ class ProgramConfigurationParser { - - /** - * - */ private static final int DEFAULT_SMTP_PORT = 25; + + private static final int DEFAULT_PRIORITY = 1; private static final String ELEM_PASSWORD = "password"; @@ -74,6 +72,8 @@ class ProgramConfigurationParser { // Configuration of interesting programs. private static final String ELEM_PROGRAM = "program"; + + private static final String ELEM_PRIORITY = "priority"; private static final String ELEM_PATTERN = "match"; @@ -117,7 +117,12 @@ class ProgramConfigurationParser { } Element actionElem = program.element(ELEM_ACTION); - ProgramAction action = new RecordProgramAction(1); + int priority = DEFAULT_PRIORITY; + String priorityString = program.elementTextTrim(ELEM_PRIORITY); + if ( priorityString != null ) { + priority = Integer.valueOf(priorityString); + } + ProgramAction action = new RecordProgramAction(priority); if (actionElem != null) { if (actionElem.getText().equals(ACTION_NOTIFY)) { action = new InterestingProgramAction(category); diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/main/RecordProgramAction.java b/crawler/kiss/src/org/wamblee/crawler/kiss/main/RecordProgramAction.java index 8458a957..4cbe12bc 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/main/RecordProgramAction.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/main/RecordProgramAction.java @@ -27,8 +27,8 @@ public class RecordProgramAction implements ProgramAction { /** * Constructs the action. - * @param aPriority Priority of the recording action. Lower values mean - * higher priority. + * @param aPriority Priority of the recording action. Higher values have higher + * priority. */ public RecordProgramAction(int aPriority) { _priority = aPriority; -- 2.31.1 From ab0c55d0e7d52029e63cf39f2c8f56da50df7874 Mon Sep 17 00:00:00 2001 From: erik Date: Thu, 23 Mar 2006 21:52:51 +0000 Subject: [PATCH 05/16] --- .../src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java index 4b2476d0..90fd6e1f 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java @@ -112,7 +112,6 @@ public class ProgramActionExecutor { /** * Makes sure that the actions are performed. - * */ public void commit() { Set previouslyRecorded = new HashSet(); -- 2.31.1 From 98f3ac509d56543244e2288ae701b87bb913a5e0 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 24 Mar 2006 11:07:30 +0000 Subject: [PATCH 06/16] --- crawler/kiss/conf/kiss/utilities.xsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crawler/kiss/conf/kiss/utilities.xsl b/crawler/kiss/conf/kiss/utilities.xsl index 4e465a0a..e6863db2 100644 --- a/crawler/kiss/conf/kiss/utilities.xsl +++ b/crawler/kiss/conf/kiss/utilities.xsl @@ -87,10 +87,10 @@ + + - - -- 2.31.1 From 212e782211f732a2c96f987e53ec24b263b099f0 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 24 Mar 2006 13:26:47 +0000 Subject: [PATCH 07/16] Programs for which there was already an attempt to record them will no longer be mentioned as interesting. --- .../crawler/kiss/main/ProgramActionExecutor.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java index 90fd6e1f..51aef60f 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramActionExecutor.java @@ -152,6 +152,8 @@ public class ProgramActionExecutor { public Element getReport() { DocumentFactory factory = DocumentFactory.getInstance(); Element report = factory.createElement("report"); + + Set reportedPrograms = new HashSet(); for (RecordingResult result : RecordingResult.values()) { if (_recordings.get(result).size() > 0) { @@ -160,6 +162,7 @@ public class ProgramActionExecutor { for (Program program : _recordings.get(result)) { recordingResult.add(program.asXml()); + reportedPrograms.add(program); } } } @@ -173,7 +176,16 @@ public class ProgramActionExecutor { categoryElem.addAttribute("name", category); } for (Program program : _interestingShows.get(category)) { - categoryElem.add(program.asXml()); + if ( !reportedPrograms.contains(program)) { + categoryElem.add(program.asXml()); + } else { + LOG.info("Category '" + category + "', program " + program + " already reported"); + } + } + if ( categoryElem.elements().size() == 0 ) { + // Remove empty category element. + LOG.info("Removing element for category '" + category + "'"); + interesting.remove(categoryElem); } } -- 2.31.1 From 0f3efe8e07f9831f1081ff43a7350673e1a295e1 Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 17:43:42 +0000 Subject: [PATCH 08/16] Corrected sorting of programs. --- crawler/kiss/conf/kiss/programs.xml | 30 ++++++++++++++----- .../org/wamblee/crawler/kiss/guide/Time.java | 18 ++++++++++- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/crawler/kiss/conf/kiss/programs.xml b/crawler/kiss/conf/kiss/programs.xml index 57f14cf6..0417895f 100644 --- a/crawler/kiss/conf/kiss/programs.xml +++ b/crawler/kiss/conf/kiss/programs.xml @@ -1,9 +1,9 @@ - erik@brakkee.org + kiss@brakkee.org erik@brakkee.org - KiSS Crawler Update + Recording summary for today falcon 25 @@ -27,19 +27,25 @@ horror|actie|thriller + + wetenschap + notify + wetenschap + + science fiction notify - (sci-fi)|(science fiction) + sf-|(sci-fi)|(science fiction) documentaires - notify (zembla)|(uur.*wolf)|(andere tijden) + 20 star.*gate @@ -58,6 +64,17 @@ + 9 + lois.*clark + + + + 8 + jag + + + + 5 shouf shouf @@ -70,12 +87,9 @@ + wetenschap notify brainiac - - lois.*clark - - diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java index 52695e10..2a15073b 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java @@ -24,6 +24,16 @@ import java.text.NumberFormat; */ public class Time implements Comparable { + /** + * + */ + private static final int HOURS_PER_DAY = 24; + + /** + * + */ + private static final int EARLY_HOUR = 3; + /** * Number of seconds per minute. */ @@ -87,7 +97,13 @@ public class Time implements Comparable { * @return Converted value. */ float asFloat() { - return (float) _hour + (float) _minute / (float) SECONDS_PER_MINUTE; + int hour = _hour; + // Hack to make sure that programs appearing shortly after midnight are sorted + // after those running during the day. + if ( hour <= EARLY_HOUR ) { + hour += HOURS_PER_DAY; + } + return (float) hour + (float) _minute / (float) SECONDS_PER_MINUTE; } /* -- 2.31.1 From b2b021fe638cc5b140b1a72f2203f787445a1c60 Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 18:02:05 +0000 Subject: [PATCH 09/16] --- .../wamblee/crawler/kiss/guide/Program.java | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Program.java b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Program.java index 28cbec5b..241089b1 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Program.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Program.java @@ -32,6 +32,22 @@ import org.wamblee.crawler.kiss.main.SystemProperties; */ public class Program { + private static final String ELEM_PROGRAM = "program"; + + private static final String ELEM_NAME = "name"; + + private static final String ELEM_KEYWORDS = "keywords"; + + private static final String ELEM_DESCRIPTION = "description"; + + private static final String ELEM_CHANNEL = "channel"; + + private static final String ELEM_INTERVAL = "interval"; + + private static final String ELEM_END_TIME = "end"; + + private static final String ELEM_BEGIN_TIME = "begin"; + /** * Lexicographical comparison of programs based on (time, title, channel). * @@ -106,8 +122,9 @@ public class Program { } /** - * Gets the description. - * @return Description. + * Gets the description. + * + * @return Description. */ public String getDescription() { return _description; @@ -321,15 +338,16 @@ public class Program { */ public Element asXml() { DocumentFactory factory = DocumentFactory.getInstance(); - Element program = factory.createElement("program"); - program.addElement("name").setText(getName()); - program.addElement("description").setText(getDescription()); - program.addElement("keywords").setText(getKeywords()); - program.addElement("channel").setText(getChannel()); - Element interval = program.addElement("interval"); - interval.addElement("begin").setText( + Element program = factory.createElement(ELEM_PROGRAM); + program.addElement(ELEM_NAME).setText(getName()); + program.addElement(ELEM_DESCRIPTION).setText(getDescription()); + program.addElement(ELEM_KEYWORDS).setText(getKeywords()); + program.addElement(ELEM_CHANNEL).setText(getChannel()); + Element interval = program.addElement(ELEM_INTERVAL); + interval.addElement(ELEM_BEGIN_TIME).setText( getInterval().getBegin().toString()); - interval.addElement("end").setText(getInterval().getEnd().toString()); + interval.addElement(ELEM_END_TIME).setText( + getInterval().getEnd().toString()); return program; } } -- 2.31.1 From 5f696182f6ccc69bedbfd168b4e659bc377f4d55 Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 18:16:13 +0000 Subject: [PATCH 10/16] --- crawler/kiss/conf/kiss/programs.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crawler/kiss/conf/kiss/programs.xml b/crawler/kiss/conf/kiss/programs.xml index 0417895f..428368ac 100644 --- a/crawler/kiss/conf/kiss/programs.xml +++ b/crawler/kiss/conf/kiss/programs.xml @@ -92,4 +92,9 @@ brainiac + + auto + wegmisbruikers|(blik.*op.*weg) + + -- 2.31.1 From 27ddeddb3d0ce203a7b6ceaed0bb673b288b7ec3 Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 18:52:16 +0000 Subject: [PATCH 11/16] --- .../wamblee/crawler/kiss/guide/Program.java | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Program.java b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Program.java index 241089b1..9f63ac4e 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Program.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Program.java @@ -93,42 +93,27 @@ public class Program { /** * Successfully recorded. */ - OK("Successfully recorded programs"), + OK, /** * Already recorded program. */ - DUPLICATE("Already recorded programs"), + DUPLICATE, /** * Recording conflict with another program. */ - CONFLICT("Programs in conflict with another recorded program"), + CONFLICT, /** * Program occurred in the past. */ - OLDSHOW("Programs that occurred in the past"), + OLDSHOW, /** * Program could not be recorded for technical reasons. */ - ERROR("Programs that could not be recorded for technical reasons"); - - private String _description; - - private RecordingResult(String aDescription) { - _description = aDescription; - } - - /** - * Gets the description. - * - * @return Description. - */ - public String getDescription() { - return _description; - } + ERROR; }; /** -- 2.31.1 From 09730f5e6f4e37c558c2040fbaac69194c07c8f4 Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 19:11:00 +0000 Subject: [PATCH 12/16] --- crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java index 2a15073b..2d05bac1 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/Time.java @@ -126,7 +126,7 @@ public class Time implements Comparable { */ public int compareTo(Object aObject) { if (!(aObject instanceof Time)) { - throw new RuntimeException("object not an instance of Time"); + throw new IllegalArgumentException("object not an instance of Time"); } Time time = (Time) aObject; return new Float(asFloat()).compareTo(new Float(time.asFloat())); -- 2.31.1 From dde8e6476bfbf590bc14953debe29312888399fe Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 19:12:48 +0000 Subject: [PATCH 13/16] --- .../org/wamblee/crawler/kiss/guide/AbstractVisitor.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java index 4b27e4f1..01445907 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java @@ -37,8 +37,7 @@ public abstract class AbstractVisitor implements Visitor { * @param aChannel Channel to visit. */ public void visitChannel(Channel aChannel) { - List programs = aChannel.getPrograms(); - for (Program program: programs) { + for (Program program: aChannel.getPrograms() ) { program.accept(this); } } @@ -47,9 +46,8 @@ public abstract class AbstractVisitor implements Visitor { * Visits the TV guide by visiting all channels of the guide. * @param aGuide TV guide to visit. */ - public void visitTvGuide(TVGuide aGuide) { - List channels = aGuide.getChannels(); - for (Channel channel: channels) { + public void visitTvGuide(TVGuide aGuide) { + for (Channel channel: aGuide.getChannels()) { channel.accept(this); } } -- 2.31.1 From 63953f83791ba80830cb68f8efd24248f9385f4a Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 19:13:05 +0000 Subject: [PATCH 14/16] --- .../kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java index 01445907..f4b07bff 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/guide/AbstractVisitor.java @@ -16,7 +16,6 @@ package org.wamblee.crawler.kiss.guide; -import java.util.List; /** -- 2.31.1 From 1bf172222a944885867d7a939ed147c8acbdae2f Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 19:14:46 +0000 Subject: [PATCH 15/16] --- .../kiss/src/org/wamblee/crawler/kiss/main/ProgramAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramAction.java b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramAction.java index a2da90fd..5211a5e9 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramAction.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/main/ProgramAction.java @@ -29,7 +29,7 @@ public interface ProgramAction { * @param aProgram * Program to execute the action for. * @param aExecutor - * Report to use. + * Executor to use. */ void execute(Program aProgram, ProgramActionExecutor aExecutor); } -- 2.31.1 From 951167e3811e07ba8c8c02226fe08a8bca6acc3f Mon Sep 17 00:00:00 2001 From: erik Date: Sat, 25 Mar 2006 21:03:17 +0000 Subject: [PATCH 16/16] stylesheets are now searched in the classpath. --- .../wamblee/crawler/AbstractPageRequest.java | 8 +- .../{conf/kiss => src}/channel-overview.xsl | 0 .../{conf/kiss => src}/channel-right-now.xsl | 0 .../{conf/kiss => src}/channels-favorites.xsl | 0 crawler/kiss/{conf/kiss => src}/identity.xsl | 0 crawler/kiss/{conf/kiss => src}/login.xsl | 0 .../kiss/notification/MailNotifier.java | 15 ++-- .../kiss/{conf/kiss => src}/program-info.xsl | 0 crawler/kiss/{conf/kiss => src}/recorded.xsl | 0 .../kiss/{conf/kiss => src}/reportToHtml.xsl | 0 .../kiss/{conf/kiss => src}/reportToText.xsl | 0 crawler/kiss/{conf/kiss => src}/utilities.xsl | 0 support/src/org/wamblee/xml/XSLT.java | 77 +++++++++++++------ 13 files changed, 64 insertions(+), 36 deletions(-) rename crawler/kiss/{conf/kiss => src}/channel-overview.xsl (100%) rename crawler/kiss/{conf/kiss => src}/channel-right-now.xsl (100%) rename crawler/kiss/{conf/kiss => src}/channels-favorites.xsl (100%) rename crawler/kiss/{conf/kiss => src}/identity.xsl (100%) rename crawler/kiss/{conf/kiss => src}/login.xsl (100%) rename crawler/kiss/{conf/kiss => src}/program-info.xsl (100%) rename crawler/kiss/{conf/kiss => src}/recorded.xsl (100%) rename crawler/kiss/{conf/kiss => src}/reportToHtml.xsl (100%) rename crawler/kiss/{conf/kiss => src}/reportToText.xsl (100%) rename crawler/kiss/{conf/kiss => src}/utilities.xsl (100%) diff --git a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java index baf9510b..66627ca5 100644 --- a/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java +++ b/crawler/basic/src/org/wamblee/crawler/AbstractPageRequest.java @@ -17,7 +17,6 @@ package org.wamblee.crawler; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import javax.xml.transform.OutputKeys; @@ -40,7 +39,7 @@ import org.apache.xml.serialize.OutputFormat; import org.apache.xml.serialize.XMLSerializer; import org.w3c.dom.Document; import org.w3c.tidy.Tidy; -import org.wamblee.io.FileResource; +import org.wamblee.xml.ClasspathUriResolver; import org.wamblee.xml.DOMUtility; import org.wamblee.xml.XSLT; @@ -155,8 +154,9 @@ public abstract class AbstractPageRequest implements PageRequest { aMethod = executeWithRedirects(aClient, aMethod); byte[] xhtmlData = getXhtml(aMethod); - Document transformed = new XSLT().transform(xhtmlData, - new FileResource(new File(_xslt))); + XSLT xsltProcessor = new XSLT(new ClasspathUriResolver()); + Document transformed = xsltProcessor.transform(xhtmlData, + xsltProcessor.resolve(_xslt)); ByteArrayOutputStream os = new ByteArrayOutputStream(); Transformer transformer = TransformerFactory.newInstance() .newTransformer(); diff --git a/crawler/kiss/conf/kiss/channel-overview.xsl b/crawler/kiss/src/channel-overview.xsl similarity index 100% rename from crawler/kiss/conf/kiss/channel-overview.xsl rename to crawler/kiss/src/channel-overview.xsl diff --git a/crawler/kiss/conf/kiss/channel-right-now.xsl b/crawler/kiss/src/channel-right-now.xsl similarity index 100% rename from crawler/kiss/conf/kiss/channel-right-now.xsl rename to crawler/kiss/src/channel-right-now.xsl diff --git a/crawler/kiss/conf/kiss/channels-favorites.xsl b/crawler/kiss/src/channels-favorites.xsl similarity index 100% rename from crawler/kiss/conf/kiss/channels-favorites.xsl rename to crawler/kiss/src/channels-favorites.xsl diff --git a/crawler/kiss/conf/kiss/identity.xsl b/crawler/kiss/src/identity.xsl similarity index 100% rename from crawler/kiss/conf/kiss/identity.xsl rename to crawler/kiss/src/identity.xsl diff --git a/crawler/kiss/conf/kiss/login.xsl b/crawler/kiss/src/login.xsl similarity index 100% rename from crawler/kiss/conf/kiss/login.xsl rename to crawler/kiss/src/login.xsl diff --git a/crawler/kiss/src/org/wamblee/crawler/kiss/notification/MailNotifier.java b/crawler/kiss/src/org/wamblee/crawler/kiss/notification/MailNotifier.java index 47c3edd5..536135b2 100644 --- a/crawler/kiss/src/org/wamblee/crawler/kiss/notification/MailNotifier.java +++ b/crawler/kiss/src/org/wamblee/crawler/kiss/notification/MailNotifier.java @@ -4,7 +4,6 @@ */ package org.wamblee.crawler.kiss.notification; -import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Date; @@ -16,8 +15,7 @@ import javax.xml.transform.TransformerException; import org.apache.commons.mail.EmailException; import org.apache.commons.mail.HtmlEmail; import org.dom4j.Element; -import org.wamblee.io.FileResource; -import org.wamblee.io.InputResource; +import org.wamblee.xml.ClasspathUriResolver; import org.wamblee.xml.XSLT; /** @@ -80,10 +78,8 @@ public class MailNotifier implements Notifier { mail.setSentDate(new Date()); mail.setSubject(_subject); - String htmlText = transformReport(aReport, new FileResource( - new File(_htmlXslt))); - String plainText = transformReport(aReport, new FileResource( - new File(_textXslt))); + String htmlText = transformReport(aReport, _htmlXslt); + String plainText = transformReport(aReport, _textXslt); mail.setHtmlMsg(htmlText); mail.setTextMsg(plainText); @@ -113,9 +109,10 @@ public class MailNotifier implements Notifier { * @throws TransformerException * In case of problems transforming. */ - private String transformReport(Element aReport, InputResource aXslt) + private String transformReport(Element aReport, String aXslt) throws IOException, TransformerException { String reportXmlText = aReport.asXML(); - return new XSLT().textTransform(reportXmlText.getBytes(), aXslt); + XSLT xsltProcessor = new XSLT(new ClasspathUriResolver()); + return xsltProcessor.textTransform(reportXmlText.getBytes(), xsltProcessor.resolve(aXslt)); } } diff --git a/crawler/kiss/conf/kiss/program-info.xsl b/crawler/kiss/src/program-info.xsl similarity index 100% rename from crawler/kiss/conf/kiss/program-info.xsl rename to crawler/kiss/src/program-info.xsl diff --git a/crawler/kiss/conf/kiss/recorded.xsl b/crawler/kiss/src/recorded.xsl similarity index 100% rename from crawler/kiss/conf/kiss/recorded.xsl rename to crawler/kiss/src/recorded.xsl diff --git a/crawler/kiss/conf/kiss/reportToHtml.xsl b/crawler/kiss/src/reportToHtml.xsl similarity index 100% rename from crawler/kiss/conf/kiss/reportToHtml.xsl rename to crawler/kiss/src/reportToHtml.xsl diff --git a/crawler/kiss/conf/kiss/reportToText.xsl b/crawler/kiss/src/reportToText.xsl similarity index 100% rename from crawler/kiss/conf/kiss/reportToText.xsl rename to crawler/kiss/src/reportToText.xsl diff --git a/crawler/kiss/conf/kiss/utilities.xsl b/crawler/kiss/src/utilities.xsl similarity index 100% rename from crawler/kiss/conf/kiss/utilities.xsl rename to crawler/kiss/src/utilities.xsl diff --git a/support/src/org/wamblee/xml/XSLT.java b/support/src/org/wamblee/xml/XSLT.java index ae603976..7ffe63b3 100644 --- a/support/src/org/wamblee/xml/XSLT.java +++ b/support/src/org/wamblee/xml/XSLT.java @@ -18,8 +18,8 @@ package org.wamblee.xml; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; -import java.io.InputStream; import javax.xml.transform.Result; import javax.xml.transform.Source; @@ -27,25 +27,60 @@ import javax.xml.transform.Transformer; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; +import javax.xml.transform.URIResolver; import javax.xml.transform.dom.DOMResult; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import org.w3c.dom.Document; -import org.wamblee.io.InputResource; +import org.wamblee.io.FileResource; /** * XSLT utilities. */ public class XSLT { + private TransformerFactory _factory; + + /** + * Constructs the URL resolver. + * + * @param aResolver + * URI resolver to use. + */ + public XSLT(URIResolver aResolver) { + _factory = TransformerFactory.newInstance(); + _factory.setURIResolver(aResolver); + } + /** * Constructs the XSLT processor. * */ public XSLT() { - // Empty. + _factory = TransformerFactory.newInstance(); + } + + /** + * Resolves an XSLT based on URI. + * @param aXslt XSLT to resolve, + * @return Source for the XSLT + * @throws TransformerException In case the XSLT cannot be found. + */ + public Source resolve(String aXslt) throws TransformerException { + URIResolver resolver = _factory.getURIResolver(); + if (resolver == null) { + if (new File(aXslt).canRead()) { + try { + return new StreamSource(new FileResource(new File(aXslt)) + .getInputStream()); + } catch (IOException e) { + throw new TransformerException(e.getMessage(), e); + } + } + } + return resolver.resolve(aXslt, ""); } /** @@ -62,7 +97,7 @@ public class XSLT { * @throws TransformerException * In case transformation fails. */ - public Document transform(Document aDocument, InputResource aXslt) + public Document transform(Document aDocument, Source aXslt) throws IOException, TransformerException { Source source = new DOMSource(aDocument); DOMResult result = new DOMResult(); @@ -83,24 +118,28 @@ public class XSLT { * @throws TransformerException * In case transformation fails. */ - public Document transform(byte[] aDocument, InputResource aXslt) + public Document transform(byte[] aDocument, Source aXslt) throws IOException, TransformerException { Source source = new StreamSource(new ByteArrayInputStream(aDocument)); DOMResult result = new DOMResult(); transform(source, result, aXslt); return (Document) result.getNode(); } - + /** - * Transforms a document to a text output. This supports XSLT transformations - * that result in text documents. - * @param aDocument Document to transform. - * @param aXslt XSL transformation. - * @return Transformed document. + * Transforms a document to a text output. This supports XSLT + * transformations that result in text documents. + * + * @param aDocument + * Document to transform. + * @param aXslt + * XSL transformation. + * @return Transformed document. */ - public String textTransform(byte[] aDocument, InputResource aXslt) throws IOException, TransformerException { + public String textTransform(byte[] aDocument, Source aXslt) + throws IOException, TransformerException { Source source = new StreamSource(new ByteArrayInputStream(aDocument)); - ByteArrayOutputStream os = new ByteArrayOutputStream(); + ByteArrayOutputStream os = new ByteArrayOutputStream(); StreamResult result = new StreamResult(os); transform(source, result, aXslt); return new String(os.toByteArray()); @@ -120,22 +159,14 @@ public class XSLT { * @throws TransformerException * In case transformation fails. */ - public void transform(Source aSource, Result aResult, InputResource aXslt) + public void transform(Source aSource, Result aResult, Source aXslt) throws IOException, TransformerException { - InputStream xslt = null; try { - xslt = aXslt.getInputStream(); - Source xsltSource = new StreamSource(xslt); - Transformer transformer = TransformerFactory.newInstance() - .newTransformer(xsltSource); + Transformer transformer = _factory.newTransformer(aXslt); transformer.transform(aSource, aResult); } catch (TransformerConfigurationException e) { throw new RuntimeException( "Configuration problem of XSLT transformation", e); - } finally { - if (xslt != null) { - xslt.close(); - } } } } -- 2.31.1