wamblee.org Git - utils/blob - crawler/basic/test/org/wamblee/crawler/Main.java

   1 package org.wamblee.crawler;
   2 import java.io.File;
   3 import java.io.FileOutputStream;
   4 import java.io.IOException;
   5
   6 import org.apache.commons.httpclient.Header;
   7 import org.apache.commons.httpclient.HttpClient;
   8 import org.apache.commons.httpclient.HttpException;
   9 import org.apache.commons.httpclient.HttpMethod;
  10 import org.apache.commons.httpclient.HttpStatus;
  11 import org.apache.commons.httpclient.NameValuePair;
  12 import org.apache.commons.httpclient.methods.GetMethod;
  13 import org.apache.commons.httpclient.methods.PostMethod;
  14 import org.apache.commons.httpclient.params.HttpClientParams;
  15 import org.apache.commons.httpclient.params.HttpMethodParams;
  16 import org.w3c.tidy.Tidy;
  17
  18 /*
  19  * Copyright 2005 the original author or authors.
  20  *
  21  * Licensed under the Apache License, Version 2.0 (the "License");
  22  * you may not use this file except in compliance with the License.
  23  * You may obtain a copy of the License at
  24  *
  25  *      http://www.apache.org/licenses/LICENSE-2.0
  26  *
  27  * Unless required by applicable law or agreed to in writing, software
  28  * distributed under the License is distributed on an "AS IS" BASIS,
  29  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  30  * See the License for the specific language governing permissions and
  31  * limitations under the License.
  32  */
  33
  34 /**
  35  *
  36  */
  37 public final class Main {
  38
  39     /**
  40      *
  41      */
  42     private static final int PROXY_PORT = 10000;
  43
  44     /**
  45      *
  46      */
  47     private static final int MAX_REDIRECTS = 10;
  48
  49     /**
  50      * Disabled constructor.
  51      *
  52      */
  53     private Main() {
  54         // Empty
  55     }
  56
  57     private static final String BASE = "http://epg.kml.kiss-technology.com/";
  58
  59     private static int COUNT = 0;
  60
  61     public static void main(String[] aArgs) {
  62         HttpClientParams clientParams = new HttpClientParams();
  63         clientParams.setIntParameter(HttpClientParams.MAX_REDIRECTS, MAX_REDIRECTS);
  64         clientParams.setBooleanParameter(HttpClientParams.REJECT_RELATIVE_REDIRECT, false);
  65         HttpClient client = new HttpClient(clientParams);
  66         client.getHostConfiguration().setProxy("localhost", PROXY_PORT);
  67
  68         clientParams = client.getParams();
  69         Object obj = clientParams.getParameter(HttpClientParams.MAX_REDIRECTS);
  70         System.out.println("Max redirects = " + obj);
  71         HttpMethod method = new GetMethod(BASE + "l.php");
  72
  73         executeMethod(client, method);
  74
  75         PostMethod postMethod = new PostMethod(BASE + "login_core.php");
  76         HttpMethodParams params = new HttpMethodParams();
  77         params.setParameter("user", "erik@brakkee.org");
  78         params.setParameter("passwd", "ebra1969");
  79         params.setParameter("SavePlayerID", "");
  80         params.setParameter("GMode", "TextMode");
  81         params.setParameter("submit", "Login");
  82
  83         NameValuePair[] data = new NameValuePair[] {
  84                 new NameValuePair("user", "erik@brakkee.org"),
  85                 new NameValuePair("passwd", "ebra1969"),
  86                 new NameValuePair("GMode", "TextMode"),
  87                 new NameValuePair("submit", "Login")
  88         };
  89         postMethod.addParameters(data);
  90
  91
  92         executeMethod(client, postMethod);
  93
  94         Header header = postMethod.getResponseHeader("Location");
  95         System.out.println("Redirecting to: " + header.getValue());
  96         method = new GetMethod(header.getValue());
  97         executeMethod(client, method);
  98
  99
 100     }
 101
 102     /**
 103      * @param aClient
 104      * @param aMethod
 105      */
 106     private static int executeMethod(HttpClient aClient, HttpMethod aMethod) {
 107         //method.setFollowRedirects(true);
 108         try {
 109             // Execute the method.
 110             int statusCode = aClient.executeMethod(aMethod);
 111
 112             if (statusCode != HttpStatus.SC_OK) {
 113               System.err.println("Method failed: " + aMethod.getStatusLine());
 114             }
 115
 116             // Read the response body.
 117             String filename = "output" + COUNT++;
 118             FileOutputStream os = new FileOutputStream(new File(filename));
 119             //os.write(method.getResponseBody());
 120
 121             Tidy tidy = new Tidy();
 122             tidy.setXHTML(true);
 123             tidy.parse(aMethod.getResponseBodyAsStream(), os);
 124             os.close();
 125             System.out.println("Written response to file: " + filename);
 126             return statusCode;
 127           } catch (HttpException e) {
 128             throw new RuntimeException("Fatal protocol violation: " + e.getMessage());
 129           } catch (IOException e) {
 130             throw new RuntimeException("Fatal transport error: " + e.getMessage());
 131           } finally {
 132             // Release the connection.
 133             aMethod.releaseConnection();
 134           }
 135     }
 136
 137 }