1 package org.wamblee.crawler;
3 import java.io.FileOutputStream;
4 import java.io.IOException;
6 import org.apache.commons.httpclient.Header;
7 import org.apache.commons.httpclient.HttpClient;
8 import org.apache.commons.httpclient.HttpException;
9 import org.apache.commons.httpclient.HttpMethod;
10 import org.apache.commons.httpclient.HttpStatus;
11 import org.apache.commons.httpclient.NameValuePair;
12 import org.apache.commons.httpclient.methods.GetMethod;
13 import org.apache.commons.httpclient.methods.PostMethod;
14 import org.apache.commons.httpclient.params.HttpClientParams;
15 import org.apache.commons.httpclient.params.HttpMethodParams;
16 import org.w3c.tidy.Tidy;
19 * Copyright 2005 the original author or authors.
21 * Licensed under the Apache License, Version 2.0 (the "License");
22 * you may not use this file except in compliance with the License.
23 * You may obtain a copy of the License at
25 * http://www.apache.org/licenses/LICENSE-2.0
27 * Unless required by applicable law or agreed to in writing, software
28 * distributed under the License is distributed on an "AS IS" BASIS,
29 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
30 * See the License for the specific language governing permissions and
31 * limitations under the License.
37 public final class Main {
42 private static final int PROXY_PORT = 10000;
47 private static final int MAX_REDIRECTS = 10;
50 * Disabled constructor.
57 private static final String BASE = "http://epg.kml.kiss-technology.com/";
59 private static int COUNT = 0;
61 public static void main(String[] aArgs) {
62 HttpClientParams clientParams = new HttpClientParams();
63 clientParams.setIntParameter(HttpClientParams.MAX_REDIRECTS, MAX_REDIRECTS);
64 clientParams.setBooleanParameter(HttpClientParams.REJECT_RELATIVE_REDIRECT, false);
65 HttpClient client = new HttpClient(clientParams);
66 client.getHostConfiguration().setProxy("localhost", PROXY_PORT);
68 clientParams = client.getParams();
69 Object obj = clientParams.getParameter(HttpClientParams.MAX_REDIRECTS);
70 System.out.println("Max redirects = " + obj);
71 HttpMethod method = new GetMethod(BASE + "l.php");
73 executeMethod(client, method);
75 PostMethod postMethod = new PostMethod(BASE + "login_core.php");
76 HttpMethodParams params = new HttpMethodParams();
77 params.setParameter("user", "erik@brakkee.org");
78 params.setParameter("passwd", "ebra1969");
79 params.setParameter("SavePlayerID", "");
80 params.setParameter("GMode", "TextMode");
81 params.setParameter("submit", "Login");
83 NameValuePair[] data = new NameValuePair[] {
84 new NameValuePair("user", "erik@brakkee.org"),
85 new NameValuePair("passwd", "ebra1969"),
86 new NameValuePair("GMode", "TextMode"),
87 new NameValuePair("submit", "Login")
89 postMethod.addParameters(data);
92 executeMethod(client, postMethod);
94 Header header = postMethod.getResponseHeader("Location");
95 System.out.println("Redirecting to: " + header.getValue());
96 method = new GetMethod(header.getValue());
97 executeMethod(client, method);
106 private static int executeMethod(HttpClient aClient, HttpMethod aMethod) {
107 //method.setFollowRedirects(true);
109 // Execute the method.
110 int statusCode = aClient.executeMethod(aMethod);
112 if (statusCode != HttpStatus.SC_OK) {
113 System.err.println("Method failed: " + aMethod.getStatusLine());
116 // Read the response body.
117 String filename = "output" + COUNT++;
118 FileOutputStream os = new FileOutputStream(new File(filename));
119 //os.write(method.getResponseBody());
121 Tidy tidy = new Tidy();
123 tidy.parse(aMethod.getResponseBodyAsStream(), os);
125 System.out.println("Written response to file: " + filename);
127 } catch (HttpException e) {
128 throw new RuntimeException("Fatal protocol violation: " + e.getMessage());
129 } catch (IOException e) {
130 throw new RuntimeException("Fatal transport error: " + e.getMessage());
132 // Release the connection.
133 aMethod.releaseConnection();