/*
 * Copyright 2005 the original author or authors.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.wamblee.crawler;

import java.io.ByteArrayOutputStream;
import java.io.IOException;

import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.xml.serialize.OutputFormat;
import org.apache.xml.serialize.XMLSerializer;
import org.w3c.dom.Document;
import org.w3c.tidy.Tidy;
import org.wamblee.xml.DomUtils;
import org.wamblee.xml.XslTransformer;

/**
 * General support claas for all kinds of requests.
 */
public abstract class AbstractPageRequest implements PageRequest {

    private static final Log LOG = LogFactory.getLog(AbstractPageRequest.class);

    private static final String REDIRECT_HEADER = "Location";

    private int _maxTries;

    private int _maxDelay;

    private NameValuePair[] _params;
    
    private NameValuePair[] _headers;

    private String _xslt;
    
    private XslTransformer _transformer; 

    /**
     * Constructs the request.
     * 
     * @param aMaxTries
     *            Maximum retries to perform.
     * @param aMaxDelay
     *            Maximum delay before executing a request.
     * @param aParams
     *            Request parameters to use.
     * @param aHeaders
     *            Request headers to use. 
     * @param aXslt
     *            XSLT used to convert the response.
     */
    protected AbstractPageRequest(int aMaxTries, int aMaxDelay,
            NameValuePair[] aParams, NameValuePair[] aHeaders, String aXslt, XslTransformer aTransformer) {
        if (aParams == null) {
            throw new IllegalArgumentException("aParams is null");
        }
        if (aHeaders == null) {
            throw new IllegalArgumentException("aHeaders is null");
        }
        if (aXslt == null) {
            throw new IllegalArgumentException("aXslt is null");
        }
        _maxTries = aMaxTries;
        _maxDelay = aMaxDelay;
        _params = aParams;
        _headers = aHeaders;
        _xslt = aXslt;
        _transformer = aTransformer;
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.wamblee.crawler.PageRequest#overrideXslt(java.lang.String)
     */
    public void overrideXslt(String aXslt) {
        _xslt = aXslt;
    }

    /**
     * Gets the parameters for the request.
     * 
     * @return Request parameters.
     */
    protected NameValuePair[] getParameters() {
        return _params;
    }
    
    /**
     * Gets the headers for the request. 
     * @return Request headers. 
     */
    protected NameValuePair[] getHeaders() { 
        return _headers;
    }

    /**
     * Executes the request with a random delay and with a maximum number of
     * retries.
     * 
     * @param aClient
     *            HTTP client to use.
     * @param aMethod
     *            Method representing the request.
     * @return XML document describing the response.
     * @throws IOException
     *             In case of IO problems.
     * @throws TransformerException
     *             In case transformation of the HTML to XML fails.
     */
    protected Document executeMethod(HttpClient aClient, HttpMethod aMethod)
            throws IOException, TransformerException {
        
        for (NameValuePair header: getHeaders()) { 
            aMethod.setRequestHeader(header.getName(), header.getValue());
        }
        
        int triesLeft = _maxTries;
        while (triesLeft > 0) {
            triesLeft--;
            try {
                return executeMethodWithoutRetries(aClient, aMethod);
            } catch (TransformerException e) {
                if (triesLeft == 0) {
                    throw e;
                }
            }
        }
        throw new RuntimeException("Code should never reach this point");
    }

    /**
     * Executes the request without doing any retries in case XSLT
     * transformation fails.
     * 
     * @param aClient
     *            HTTP client to use.
     * @param aMethod
     *            Method to execute.
     * @return XML document containing the result.
     * @throws IOException
     *             In case of IO problems.
     * @throws TransformerException
     *             In case transformation of the result to XML fails.
     */
    protected Document executeMethodWithoutRetries(HttpClient aClient,
            HttpMethod aMethod) throws IOException, TransformerException {
        try {
            aMethod = executeWithRedirects(aClient, aMethod);
            byte[] xhtmlData = getXhtml(aMethod);
     
            Document transformed = _transformer.transform(xhtmlData,
                    _transformer.resolve(_xslt));
            ByteArrayOutputStream os = new ByteArrayOutputStream(); 
            Transformer transformer = TransformerFactory.newInstance()
                    .newTransformer();
            transformer.setParameter(OutputKeys.INDENT, "yes");
            transformer.setParameter(OutputKeys.METHOD, "xml");
            transformer.transform(new DOMSource(transformed), new StreamResult(
                    os));
            LOG.debug("Transformed result is \n" + os.toString());
            return transformed;
        } catch (TransformerConfigurationException e) {
            throw new TransformerException("Transformer configuration problem", e);
        } finally {
            // Release the connection.
            aMethod.releaseConnection();
        }
    }

    /**
     * Gets the result of the HTTP method as an XHTML document.
     * 
     * @param aMethod
     *            Method to invoke.
     * @return XHTML as a byte array.
     * @throws IOException
     *             In case of problems obtaining the XHTML.
     */
    private byte[] getXhtml(HttpMethod aMethod) throws IOException {
        // Transform the HTML into wellformed XML.
        Tidy tidy = new Tidy();
        tidy.setXHTML(true);
        tidy.setQuiet(true);
        tidy.setShowWarnings(false);
      
        // We write the jtidy output to XML since the DOM tree it produces is
        // not namespace aware and namespace awareness is required by XSLT.
        // An alternative is to configure namespace awareness of the XML parser
        // in a system wide way.
        ByteArrayOutputStream os = new ByteArrayOutputStream(); 
        Document w3cDoc = tidy.parseDOM(aMethod.getResponseBodyAsStream(), os);
        DomUtils.removeDuplicateAttributes(w3cDoc);
        LOG.debug("Content of response is \n" + os.toString()); 

        ByteArrayOutputStream xhtml = new ByteArrayOutputStream();
        XMLSerializer serializer = new XMLSerializer(xhtml, new OutputFormat());
        serializer.serialize(w3cDoc);
        xhtml.flush();

        return xhtml.toByteArray();
    }

    /**
     * Sleeps for a random time but no more than the maximum delay.
     * 
     */
    private void delay() {
        try {
            Thread.sleep((long) ((float) _maxDelay * Math.random()));
        } catch (InterruptedException e) {
            return; // to satisfy checkstyle
        }
    }

    /**
     * Executes the request and follows redirects if needed.
     * 
     * @param aClient
     *            HTTP client to use.
     * @param aMethod
     *            Method to use.
     * @return Final HTTP method used (differs from the parameter passed in in
     *         case of redirection).
     * @throws IOException
     *             In case of network problems.
     */
    private HttpMethod executeWithRedirects(HttpClient aClient,
            HttpMethod aMethod) throws IOException {
        delay();
        int statusCode = aClient.executeMethod(aMethod);

        switch (statusCode) {
        case HttpStatus.SC_OK: {
            return aMethod;
        }
        case HttpStatus.SC_MOVED_PERMANENTLY:
        case HttpStatus.SC_MOVED_TEMPORARILY:
        case HttpStatus.SC_SEE_OTHER: {
            aMethod.releaseConnection();
            Header header = aMethod.getResponseHeader(REDIRECT_HEADER);
            aMethod = new GetMethod(header.getValue());
            return executeWithRedirects(aClient, aMethod); // TODO protect
            // against infinite
            // recursion.
        }
        default: {
            throw new IOException("Method failed: "
                    + aMethod.getStatusLine());
        }
        }
    }
}