Java中常见的编码集问题总结

package com.guor.demo.charset;

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class CommonUtils {
    public static List<Map<String, Object>> readCSVToList(String filePath) throws Exception {
        List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new FileReader(filePath));
            String[] headtilte = reader.readLine().split(",");
            String line = null;
            while ((line = reader.readLine()) != null) {
                HashMap<String, Object> hashMap = new HashMap<String, Object>();
                String[] itemArray = line.split(",");
                for (int i = 0; i < itemArray.length; i++) {
                    hashMap.put(headtilte[i], itemArray[i]);
                }
                list.add(hashMap);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (null != reader) {
                reader.close();
            }
        }
        return list;
    }

    public static void main(String[] args) throws Exception {
        String filePath = "H:\\CSDN\\netty\\编码集问题\\test.csv";
        System.out.println(readCSVToList(filePath));
    }
}

public static List<String[]> readCSVToListByUnicode(String filePath) {
        FileInputStream fis = null;
        UnicodeInputStream uin = null;
        InputStreamReader in= null;
        try {
            //logger.info("读取文件"+filePath+"编码集");
            List<String[]> list = new ArrayList<String[]>();
            //URL url = new URL(filePath);
            File f  = new File(filePath);
            fis = new FileInputStream(f);
            uin = new UnicodeInputStream(fis,enc);  //如果是本地将url.openStream -> new FileInputStream(f)
            enc = uin.getEncoding(); // check and skip possible BOM bytes
            if (enc == null){
                in = new InputStreamReader(uin);
            }else {
                in = new InputStreamReader(uin, enc);
            }
            long start = System.currentTimeMillis();
            //logger.info("读取文件"+filePath+"查看耗时开始");
            BufferedReader reader = new BufferedReader(in);
            //BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream("D:/tags.txt"),"utf-8"));
            //String tmp =reader.readLine();

            String[] headtilte = reader.readLine().split(",");
            list.add(headtilte);
            String line = null;
            String[] itemArray = null;
            while ((line = reader.readLine()) != null) {
                itemArray = line.split(",");
                list.add(itemArray);
            }
            long end = System.currentTimeMillis();
            //logger.info("读取文件"+filePath+"查看耗时="+(end-start)+"毫秒");
            return list;
        }catch (Exception e){
            logger.info("读取文件"+filePath+"，异常：", e);
            return null;
        }finally {
            try {
                if(in!=null){
                    in.close();
                }
                if(uin!=null){
                    uin.close();
                }
                if(fis!=null){
                    fis.close();
                }
            }catch (Exception e){
                logger.info("读取文件"+filePath+"，关闭流异常：", e);
            }
        }
    }

package com.guor.demo.utils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.io.PushbackInputStream;

public class UnicodeInputStream extends InputStream {

    private static final Logger logger = LoggerFactory.getLogger(UnicodeInputStream.class);

    PushbackInputStream internalIn;

    boolean isInited = false;

    String defaultEnc;

    String encoding;

    private static final int BOM_SIZE = 4;

    public UnicodeInputStream(InputStream in, String defaultEnc) {
        internalIn = new PushbackInputStream(in, BOM_SIZE);
        this.defaultEnc = defaultEnc;
    }

    public String getDefaultEncoding() {
        return defaultEnc;
    }

    public String getEncoding() {
        if (!isInited) {
            try {
                init();
            } catch (IOException ex) {
                IllegalStateException ise = new IllegalStateException("Init method failed.");
                ise.initCause(ise);
                throw ise;
            }
        }
        return encoding;
    }

    /**
     * Read-ahead four bytes and check for BOM marks. Extra bytes are
     * unread back to the stream, only BOM bytes are skipped.
     */
    protected void init() throws IOException {
        if (isInited) {
            return;
        }
        byte bom[] = new byte[BOM_SIZE];
        int n, unread;
        n = internalIn.read(bom, 0, bom.length);

        //logger.info("文件的前四个字符==="+bom[0]+","+bom[1]+","+bom[2]+","+bom[3]);

        if ( (bom[0] == (byte)0x00) && (bom[1] == (byte)0x00) &&
                (bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF) ) {
            encoding = "UTF-32BE";
            unread = n - 4;
        } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) &&
                (bom[2] == (byte)0x00) && (bom[3] == (byte)0x00) ) {
            encoding = "UTF-32LE";
            unread = n - 4;
        } else if (  (bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB) &&
                (bom[2] == (byte)0xBF) ) {
            encoding = "UTF-8";
            unread = n - 3;
        } else if ( (bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF) ) {
            encoding = "UTF-16BE";
            unread = n - 2;
        } else if ( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) ) {
            encoding = "UTF-16LE";
            unread = n - 2;
        } else {
            // Unicode BOM mark not found, unread all bytes
            encoding = defaultEnc;
            unread = n;
        }
        //System.out.println("read=" + n + ", unread=" + unread);
        if (unread > 0) {
            internalIn.unread(bom, (n - unread), unread);
        }
        isInited = true;
    }

    @Override
    public void close() throws IOException {
        //init();
        isInited = true;
        internalIn.close();
    }

    @Override
    public int read() throws IOException {
        //init();
        isInited = true;
        return internalIn.read();
    }
}