git://wamblee.org
/
utils
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
(no commit message)
[utils]
/
trunk
/
crawler
/
basic
/
test
/
org
/
wamblee
/
crawler
/
Main.java
diff --git
a/trunk/crawler/basic/src/Main.java
b/trunk/crawler/basic/test/org/wamblee/crawler/Main.java
similarity index 82%
rename from
trunk/crawler/basic/src/Main.java
rename to
trunk/crawler/basic/test/org/wamblee/crawler/Main.java
index 8287a6ecaed381a6ce0558b79618dbc377ae0f49..6d157d51157661bcd22445d35ff8b3bc059dc6bb 100644
(file)
--- a/
trunk/crawler/basic/src/Main.java
+++ b/
trunk/crawler/basic/test/org/wamblee/crawler/Main.java
@@
-1,3
+1,4
@@
+package org.wamblee.crawler;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@
-33,18
+34,36
@@
import org.w3c.tidy.Tidy;
/**
*
*/
/**
*
*/
-public class Main {
+public final class Main {
+
+ /**
+ *
+ */
+ private static final int PROXY_PORT = 10000;
+
+ /**
+ *
+ */
+ private static final int MAX_REDIRECTS = 10;
+
+ /**
+ * Disabled constructor.
+ *
+ */
+ private Main() {
+ // Empty
+ }
private static final String BASE = "http://epg.kml.kiss-technology.com/";
private static final String BASE = "http://epg.kml.kiss-technology.com/";
- private static int
count
= 0;
+ private static int
COUNT
= 0;
public static void main(String[] aArgs) {
HttpClientParams clientParams = new HttpClientParams();
public static void main(String[] aArgs) {
HttpClientParams clientParams = new HttpClientParams();
- clientParams.setIntParameter(HttpClientParams.MAX_REDIRECTS,
10
);
+ clientParams.setIntParameter(HttpClientParams.MAX_REDIRECTS,
MAX_REDIRECTS
);
clientParams.setBooleanParameter(HttpClientParams.REJECT_RELATIVE_REDIRECT, false);
HttpClient client = new HttpClient(clientParams);
clientParams.setBooleanParameter(HttpClientParams.REJECT_RELATIVE_REDIRECT, false);
HttpClient client = new HttpClient(clientParams);
- client.getHostConfiguration().setProxy("localhost",
10000
);
+ client.getHostConfiguration().setProxy("localhost",
PROXY_PORT
);
clientParams = client.getParams();
Object obj = clientParams.getParameter(HttpClientParams.MAX_REDIRECTS);
clientParams = client.getParams();
Object obj = clientParams.getParameter(HttpClientParams.MAX_REDIRECTS);
@@
-81,27
+100,27
@@
public class Main {
}
/**
}
/**
- * @param
c
lient
- * @param
m
ethod
+ * @param
aC
lient
+ * @param
aM
ethod
*/
*/
- private static int executeMethod(HttpClient
client, HttpMethod m
ethod) {
+ private static int executeMethod(HttpClient
aClient, HttpMethod aM
ethod) {
//method.setFollowRedirects(true);
try {
// Execute the method.
//method.setFollowRedirects(true);
try {
// Execute the method.
- int statusCode =
client.executeMethod(m
ethod);
+ int statusCode =
aClient.executeMethod(aM
ethod);
if (statusCode != HttpStatus.SC_OK) {
if (statusCode != HttpStatus.SC_OK) {
- System.err.println("Method failed: " +
m
ethod.getStatusLine());
+ System.err.println("Method failed: " +
aM
ethod.getStatusLine());
}
// Read the response body.
}
// Read the response body.
- String filename = "output" +
count
++;
+ String filename = "output" +
COUNT
++;
FileOutputStream os = new FileOutputStream(new File(filename));
//os.write(method.getResponseBody());
Tidy tidy = new Tidy();
tidy.setXHTML(true);
FileOutputStream os = new FileOutputStream(new File(filename));
//os.write(method.getResponseBody());
Tidy tidy = new Tidy();
tidy.setXHTML(true);
- tidy.parse(
m
ethod.getResponseBodyAsStream(), os);
+ tidy.parse(
aM
ethod.getResponseBodyAsStream(), os);
os.close();
System.out.println("Written response to file: " + filename);
return statusCode;
os.close();
System.out.println("Written response to file: " + filename);
return statusCode;
@@
-111,7
+130,7
@@
public class Main {
throw new RuntimeException("Fatal transport error: " + e.getMessage());
} finally {
// Release the connection.
throw new RuntimeException("Fatal transport error: " + e.getMessage());
} finally {
// Release the connection.
-
m
ethod.releaseConnection();
+
aM
ethod.releaseConnection();
}
}
}
}