httpclient 获取到网页内容自动判断内容编码

在“导航189”网站中编写爬虫程序中使用的httpclient 来获取网页内容，但是在获取网页内容时有编码的问题，这里介绍的一个方法是使用EntityUtils中的toString来返回网页的内容，原理是这样的，在请求的返回header中获取编码，如果没有找到返回的编码就使用默认编码来返回，代码实现如下：

/**
     * 处理GET请求，返回整个页面
     * 
     * @param url 访问地址
     * @param params 编码参数
     * @return
     * @throws Exception
     * @throws Exception
     */
    public synchronized String doGet(String url, String... params) throws Exception {
        DefaultHttpClient httpclient = new DefaultHttpClient();
        HttpProtocolParams.setUserAgent(httpclient.getParams(),
                "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.9) Gecko/20100315 Firefox/3.5.9");
        String charset = "UTF-8";
        if (null != params && params.length >= 1) {
            charset = params[0];
        }
        HttpGet httpget = new HttpGet();
        String content = "";
        httpget.setURI(new java.net.URI(url));
        HttpResponse response = httpclient.execute(httpget);
        HttpEntity entity = response.getEntity();
        if (entity != null) {
            //使用EntityUtils的toString方法，传递默认编码，在EntityUtils中的默认编码是ISO-8859-1
            content = EntityUtils.toString(entity, charset);
            httpget.abort();
            httpclient.getConnectionManager().shutdown();
        }
        return content;
    }

调用如下：

          //导航189的网站返回编码是GBK,所以传递和不传递编码都能返回正确的数据
            doGet("http://www.dh189.com/", "GBK");
           doGet("http://www.dh189.com/");
           //民生银行的网站在请求后并没有返回编码，所以要设置编码，不设置则是乱码
            doGet("http://www.cmbc.com.cn/");
            doGet("http://www.cmbc.com.cn/","GBK");

EntityUtils 内部实现是这样的：

/**
     * Get the entity content as a String, using the provided default character set
     * if none is found in the entity. 
     * If defaultCharset is null, the default "ISO-8859-1" is used.
     * 
     * @param entity must not be null
     * @param defaultCharset character set to be applied if none found in the entity
     * @return the entity content as a String
     * @throws ParseException if header elements cannot be parsed
     * @throws IllegalArgumentException if entity is null or if content length > Integer.MAX_VALUE
     * @throws IOException if an error occurs reading the input stream
     */
    public static String toString(
            final HttpEntity entity, final String defaultCharset) throws IOException, ParseException {
        if (entity == null) {
            throw new IllegalArgumentException("HTTP entity may not be null");
        }
        InputStream instream = entity.getContent();
        if (instream == null) {
            return "";
        }
        if (entity.getContentLength() > Integer.MAX_VALUE) {
            throw new IllegalArgumentException("HTTP entity too large to be buffered in memory");
        }
        int i = (int)entity.getContentLength();
        if (i < 0) {
            i = 4096;
        }
        //获取请求中的编码
        String charset = getContentCharSet(entity);
        //若没返回编码则使用传递的编码
        if (charset == null) {
            charset = defaultCharset;
        }
        if (charset == null) {
            charset = HTTP.DEFAULT_CONTENT_CHARSET;
        }
        Reader reader = new InputStreamReader(instream, charset);
        CharArrayBuffer buffer = new CharArrayBuffer(i); 
        try {
            char[] tmp = new char[1024];
            int l;
            while((l = reader.read(tmp)) != -1) {
                buffer.append(tmp, 0, l);
            }
        } finally {
            reader.close();
        }
        return buffer.toString();
    }

EntityUtils 中获取编码的方法如下：

/**
     * Obtains character set of the entity, if known.
     * 
     * @param entity must not be null
     * @return the character set, or null if not found
     * @throws ParseException if header elements cannot be parsed
     * @throws IllegalArgumentException if entity is null
     */
    public static String getContentCharSet(final HttpEntity entity)
        throws ParseException {

        if (entity == null) {
            throw new IllegalArgumentException("HTTP entity may not be null");
        }
        String charset = null;
        if (entity.getContentType() != null) { 
            HeaderElement values[] = entity.getContentType().getElements();
            if (values.length > 0) {
                NameValuePair param = values[0].getParameterByName("charset");
                if (param != null) {
                    charset = param.getValue();
                }
            }
        }
        return charset;
    }