wamblee.org Git - utils/blob - crawler/basic/src/Main.java

   1 import java.io.File;
   2 import java.io.FileOutputStream;
   3 import java.io.IOException;
   4
   5 import org.apache.commons.httpclient.Header;
   6 import org.apache.commons.httpclient.HttpClient;
   7 import org.apache.commons.httpclient.HttpException;
   8 import org.apache.commons.httpclient.HttpMethod;
   9 import org.apache.commons.httpclient.HttpStatus;
  10 import org.apache.commons.httpclient.NameValuePair;
  11 import org.apache.commons.httpclient.methods.GetMethod;
  12 import org.apache.commons.httpclient.methods.PostMethod;
  13 import org.apache.commons.httpclient.params.HttpClientParams;
  14 import org.apache.commons.httpclient.params.HttpMethodParams;
  15 import org.w3c.tidy.Tidy;
  16
  17 /*
  18  * Copyright 2005 the original author or authors.
  19  *
  20  * Licensed under the Apache License, Version 2.0 (the "License");
  21  * you may not use this file except in compliance with the License.
  22  * You may obtain a copy of the License at
  23  *
  24  *      http://www.apache.org/licenses/LICENSE-2.0
  25  *
  26  * Unless required by applicable law or agreed to in writing, software
  27  * distributed under the License is distributed on an "AS IS" BASIS,
  28  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  29  * See the License for the specific language governing permissions and
  30  * limitations under the License.
  31  */
  32
  33 /**
  34  *
  35  */
  36 public class Main {
  37
  38     private static final String BASE = "http://epg.kml.kiss-technology.com/";
  39
  40     private static int count = 0;
  41
  42     public static void main(String[] aArgs) {
  43         HttpClientParams clientParams = new HttpClientParams();
  44         clientParams.setIntParameter(HttpClientParams.MAX_REDIRECTS, 10);
  45         clientParams.setBooleanParameter(HttpClientParams.REJECT_RELATIVE_REDIRECT, false);
  46         HttpClient client = new HttpClient(clientParams);
  47         client.getHostConfiguration().setProxy("localhost", 10000);
  48
  49         clientParams = client.getParams();
  50         Object obj = clientParams.getParameter(HttpClientParams.MAX_REDIRECTS);
  51         System.out.println("Max redirects = " + obj);
  52         HttpMethod method = new GetMethod(BASE + "l.php");
  53
  54         executeMethod(client, method);
  55
  56         PostMethod postMethod = new PostMethod(BASE + "login_core.php");
  57         HttpMethodParams params = new HttpMethodParams();
  58         params.setParameter("user", "erik@brakkee.org");
  59         params.setParameter("passwd", "ebra1969");
  60         params.setParameter("SavePlayerID", "");
  61         params.setParameter("GMode", "TextMode");
  62         params.setParameter("submit", "Login");
  63
  64         NameValuePair[] data = new NameValuePair[] {
  65                 new NameValuePair("user", "erik@brakkee.org"),
  66                 new NameValuePair("passwd", "ebra1969"),
  67                 new NameValuePair("GMode", "TextMode"),
  68                 new NameValuePair("submit", "Login")
  69         };
  70         postMethod.addParameters(data);
  71
  72
  73         executeMethod(client, postMethod);
  74
  75         Header header = postMethod.getResponseHeader("Location");
  76         System.out.println("Redirecting to: " + header.getValue());
  77         method = new GetMethod(header.getValue());
  78         executeMethod(client, method);
  79
  80
  81     }
  82
  83     /**
  84      * @param client
  85      * @param method
  86      */
  87     private static int executeMethod(HttpClient client, HttpMethod method) {
  88         //method.setFollowRedirects(true);
  89         try {
  90             // Execute the method.
  91             int statusCode = client.executeMethod(method);
  92
  93             if (statusCode != HttpStatus.SC_OK) {
  94               System.err.println("Method failed: " + method.getStatusLine());
  95             }
  96
  97             // Read the response body.
  98             String filename = "output" + count++;
  99             FileOutputStream os = new FileOutputStream(new File(filename));
 100             //os.write(method.getResponseBody());
 101
 102             Tidy tidy = new Tidy();
 103             tidy.setXHTML(true);
 104             tidy.parse(method.getResponseBodyAsStream(), os);
 105             os.close();
 106             System.out.println("Written response to file: " + filename);
 107             return statusCode;
 108           } catch (HttpException e) {
 109             throw new RuntimeException("Fatal protocol violation: " + e.getMessage());
 110           } catch (IOException e) {
 111             throw new RuntimeException("Fatal transport error: " + e.getMessage());
 112           } finally {
 113             // Release the connection.
 114             method.releaseConnection();
 115           }
 116     }
 117
 118 }