java多线程文件下载器的实现

Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1

ScheduledExecutorService s = Executors.newScheduledThreadPool(1);

/**
 * Description: 存放项目常量
 *
 * @Author 狐狸半面添
 * @Create 2023/11/6 1:22
 * @Version 1.0
 */
public class Constant {
    /**
     * 指定下载目录的存放位置
     */
    public static final String PATH = "D:\\download\\";

    public static final double MB = 1024d * 1024d;
    public static final double KB = 1024d;

    /**
     * 每次读取的字节大小
     */
    public static final int BYTE_SIZE = 1024 * 100;

    /**
     * 块文件（临时文件）的后缀
     */
    public static final String PART_FILE_SUFFIX = ".temp";

    /**
     * 线程数量
     */
    public static final int THREAD_NUM = 5;

    // 创建存放位置的代码
    // public static void main(String[] args) {
    //     File file = new File("D:\\download");
    //     if (!file.exists()) {
    //         file.mkdir();
    //     }
    // }
}

/**
 * Description: 文件相关工具
 *
 * @Author 狐狸半面添
 * @Create 2023/11/6 11:46
 * @Version 1.0
 */
public class FileUtils {
    /**
     * 获取本地文件的大小
     *
     * @param path 文件路径
     * @return 文件大小
     */
    public static long getFileContentLength(String path) {
        File file = new File(path);
        return file.exists() && file.isFile() ? file.length() : 0;
    }
}

/**
 * Description: Http 相关工具类
 *
 * @Author 狐狸半面添
 * @Create 2023/11/6 1:06
 * @Version 1.0
 */
public class HttpUtils {

    private static long id = System.currentTimeMillis();

    public static void change() {
        id = System.currentTimeMillis();
    }

    /**
     * 获取下载的文件大小
     *
     * @param url 下载文件链接
     * @return 文件大小
     * @throws IOException
     */
    public static long getHttpFileContentLength(String url) throws IOException {
        int contentLength;
        HttpURLConnection httpURLConnection = null;
        try {
            httpURLConnection = getHttpURLConnection(url);
            contentLength = httpURLConnection.getContentLength();
        } finally {
            if (httpURLConnection != null) {
                httpURLConnection.disconnect();
            }
        }
        return contentLength;
    }

    /**
     * 分块下载
     *
     * @param url      下载地址
     * @param startPos 下载文件起始位置
     * @param endPos   下载文件结束位置
     * @return 连接对象
     */
    public static HttpURLConnection getHttpURLConnection(String url, long startPos, long endPos) throws IOException {
        HttpURLConnection httpURLConnection = getHttpURLConnection(url);
        LogUtils.info("下载的区间是：{}-{}", startPos, endPos);

        if (endPos != 0) {
            httpURLConnection.setRequestProperty("RANGE", "bytes=" + startPos + "-" + endPos);
        } else {
            httpURLConnection.setRequestProperty("RANGE", "bytes=" + startPos + "-");
        }

        return httpURLConnection;
    }

    /**
     * 获取 HttpURLConnection 连接对象
     *
     * @param url 文件的地址
     * @return HttpURLConnection 连接对象
     */
    public static HttpURLConnection getHttpURLConnection(String url) throws IOException {
        URL httpUrl = new URL(url);
        HttpURLConnection httpURLConnection = (HttpURLConnection) httpUrl.openConnection();
        // 向文件所在的服务器发送标识信息
        httpURLConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1");
        return httpURLConnection;
    }

    /**
     * 获取下载文件的名字
     *
     * @param url 下载地址
     * @return 文件名
     */
    public static String getHttpFileName(String url) {

        String fileName;

        int startIndex = url.lastIndexOf("/");
        int endIndex = url.lastIndexOf("?");
        if (endIndex == -1) {
            fileName = url.substring(startIndex + 1);
        } else {
            fileName = url.substring(startIndex + 1, endIndex);
        }

        int pointIndex = fileName.lastIndexOf(".");

        return fileName.substring(0, fileName.lastIndexOf(".")) + "-" + id + fileName.substring(pointIndex);
    }

}

/**
 * Description: 日志工具类
 *
 * @Author 狐狸半面添
 * @Create 2023/11/6 1:41
 * @Version 1.0
 */
public class LogUtils {
    private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("hh:mm:ss");

    public static void info(String msg, Object... args) {
        print(msg, "-info-", args);
    }

    public static void error(String msg, Object... args) {
        print(msg, "-error-", args);
    }

    private static void print(String msg, String level, Object... args) {
        if (args != null && args.length > 0) {
            msg = String.format(msg.replace("{}", "%s"), args);
        }
        String threadName = Thread.currentThread().getName();
        System.out.println(LocalTime.now().format(FORMATTER) + "  " + threadName + level + msg);
    }
}

/**
 * Description: 展示下载信息
 *
 * @Author 狐狸半面添
 * @Create 2023/11/6 2:07
 * @Version 1.0
 */
@SuppressWarnings("AlibabaUndefineMagicConstant")
public class DownloadInfoThread implements Runnable {
    /**
     * 下载文件总大小
     */
    private final long httpFileContentLength;


    /**
     * 本次累计下载的大小
     */
    public static volatile LongAdder downSize = new LongAdder();

    /**
     * 前一次下载的大小
     */
    public double prevSize;

    public DownloadInfoThread(long httpFileContentLength) {
        this.httpFileContentLength = httpFileContentLength;
    }

    @Override
    public void run() {
        // 计算文件总大小 单位是 MB
        String httpFileSize = String.format("%.2f", httpFileContentLength / Constant.MB);

        // 计算每秒下载速度 kb
        int speed = (int) ((downSize.doubleValue() - prevSize) / Constant.KB);

        prevSize = downSize.doubleValue();

        // 剩余文件的大小
        double remainSize = httpFileContentLength - downSize.doubleValue();

        // 计算剩余时间
        String remainTime = String.format("%.1f", remainSize / Constant.KB / speed);

        if ("Infinity".equalsIgnoreCase(remainTime)) {
            remainTime = "-";
        }

        // 已下载大小
        String currentFileSize = String.format("%.1f", downSize.doubleValue() / Constant.MB);

        String speedInfo = String.format("已下载 %smb/%smb，速度 %skb/s，剩余时间 %ss", currentFileSize, httpFileSize, speed, remainTime);

        System.out.print("\r");
        System.out.print(speedInfo);

    }
}

/**
 * Description: 分块下载任务
 *
 * @Author 狐狸半面添
 * @Create 2023/11/7 0:58
 * @Version 1.0
 */
public class DownloaderTask implements Callable<Boolean> {

    private final String url;

    /**
     * 下载起始位置
     */
    private final long startPos;

    /**
     * 下载结束位置
     */
    private final long endPos;

    /**
     * 标识当前是哪一部分
     */
    private final int part;

    private final CountDownLatch countDownLatch;

    public DownloaderTask(String url, long startPos, long endPos, int part, CountDownLatch countDownLatch) {
        this.url = url;
        this.startPos = startPos;
        this.endPos = endPos;
        this.part = part;
        this.countDownLatch = countDownLatch;
    }

    @Override
    public Boolean call() throws Exception {
        // 获取文件名
        String httpFileName = HttpUtils.getHttpFileName(url);
        // 分块的文件名
        httpFileName = httpFileName + Constant.PART_FILE_SUFFIX + part;
        // 下载路径
        httpFileName = Constant.PATH + httpFileName;

        // 获取分块下载的连接
        HttpURLConnection httpURLConnection = HttpUtils.getHttpURLConnection(url, startPos, endPos);

        try (
                InputStream input = httpURLConnection.getInputStream();
                BufferedInputStream bis = new BufferedInputStream(input);
                RandomAccessFile accessFile = new RandomAccessFile(httpFileName, "rw");
        ) {
            byte[] buffer = new byte[Constant.BYTE_SIZE];
            int len;
            // 循环读取数据
            while ((len = bis.read(buffer)) != -1) {
                // 1s 内下载的数据，通过原子类下载
                DownloadInfoThread.downSize.add(len);
                accessFile.write(buffer, 0, len);
            }
        } catch (FileNotFoundException e) {
            LogUtils.error("下载文件不存在 {}", url);
            return false;
        } catch (Exception e) {
            LogUtils.error("下载出现异常");
            return false;
        } finally {
            httpURLConnection.disconnect();
            countDownLatch.countDown();
        }

        return true;
    }

}

/**
 * Description: 下载器
 *
 * @Author 狐狸半面添
 * @Create 2023/11/6 1:21
 * @Version 1.0
 */
public class Downloader {

    private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1);

    public ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(Constant.THREAD_NUM,
            Constant.THREAD_NUM,
            0,
            TimeUnit.SECONDS,
            new ArrayBlockingQueue<>(5));

    private CountDownLatch countDownLatch = new CountDownLatch(Constant.THREAD_NUM);

    public void download(String url) {
        // 获取文件名
        String httpFileName = HttpUtils.getHttpFileName(url);
        // 文件下载路径
        httpFileName = Constant.PATH + httpFileName;
        // 获取本地文件的大小
        long localFileLength = FileUtils.getFileContentLength(httpFileName);


        HttpURLConnection httpURLConnection = null;
        DownloadInfoThread downloadInfoThread;
        try {
            // 获取连接对象
            httpURLConnection = HttpUtils.getHttpURLConnection(url);

            // 获取下载文件的总大小
            int contentLength = httpURLConnection.getContentLength();

            // 判断文件是否已下载过
            if (localFileLength >= contentLength) {
                LogUtils.info("{} 已下载完毕，无需重新下载", httpFileName);
                // 关闭连接对象
                httpURLConnection.disconnect();
                // 关闭线程池
                scheduledExecutorService.shutdownNow();
                poolExecutor.shutdown();

                return;
            }

            // 创建获取下载信息的任务对象
            downloadInfoThread = new DownloadInfoThread(contentLength);

            // 将任务交给线程执行，每隔 1s 打印一次
            scheduledExecutorService.scheduleAtFixedRate(downloadInfoThread, 1, 1, TimeUnit.SECONDS);

            // 切分任务
            ArrayList<Future> list = new ArrayList<>();
            split(url, list);

            countDownLatch.await();

            System.out.print("\r");
            System.out.println("分块文件下载完成");

            // 合并文件
            if (merge(httpFileName)) {
                // 清除临时文件
                clearTemp(httpFileName);
            }


        } catch (IOException | InterruptedException e) {
            e.printStackTrace();
        } finally {
            System.out.println("本次执行完成");

            // 关闭连接对象
            if (httpURLConnection != null) {
                httpURLConnection.disconnect();
            }

            // 关闭线程池
            scheduledExecutorService.shutdownNow();
            poolExecutor.shutdown();
        }
    }

    /**
     * 文件切分
     *
     * @param url        文件链接
     * @param futureList 任务集合
     */
    public void split(String url, ArrayList<Future> futureList) {
        try {
            // 获取下载文件大小
            long contentLength = HttpUtils.getHttpFileContentLength(url);

            // 计算切分后的文件大小
            long size = contentLength / Constant.THREAD_NUM;

            // 计算分块个数
            for (int i = 0; i < Constant.THREAD_NUM; i++) {
                // 计算下载起始位置
                long startPos = i * size;

                // 计算结束位置
                long endPos;
                if (i == Constant.THREAD_NUM - 1) {
                    // 下载最后一块
                    endPos = 0;
                } else {
                    endPos = startPos + size - 1;
                }

                // 创建任务对象
                DownloaderTask downloaderTask = new DownloaderTask(url, startPos, endPos, i, countDownLatch);
                // 将任务提交到线程池
                Future<Boolean> future = poolExecutor.submit(downloaderTask);

                futureList.add(future);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * 文件合并
     *
     * @param fileName 文件名
     * @return 是否合并成功
     */
    public boolean merge(String fileName) {
        LogUtils.info("开始合并文件 {}", fileName);
        byte[] buffer = new byte[Constant.BYTE_SIZE];
        int len;
        try (
                RandomAccessFile accessFile = new RandomAccessFile(fileName, "rw")
        ) {
            for (int i = 0; i < Constant.THREAD_NUM; i++) {
                try (
                        BufferedInputStream bis = new BufferedInputStream(new FileInputStream(fileName + Constant.PART_FILE_SUFFIX + i))
                ) {
                    while ((len = bis.read(buffer)) != -1) {
                        accessFile.write(buffer, 0, len);
                    }

                }
            }

            LogUtils.info("文件合并完毕 {}", fileName);

        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }

        return true;
    }

    /**
     * 清除临时文件
     *
     * @param fileName 文件名
     */
    public void clearTemp(String fileName) {
        LogUtils.info("清理分块文件");
        for (int i = 0; i < Constant.THREAD_NUM; i++) {
            String name = fileName + Constant.PART_FILE_SUFFIX + i;
            File file = new File(name);
            file.delete();
        }
        LogUtils.info("分块清除完毕");
    }
}

public class Main {
    public static void main(String[] args) {
        // 下载地址
        String url = null;

        if (args == null || args.length == 0) {
            while (url == null || url.trim().isEmpty()) {
                System.out.print("请输入下载链接：");
                Scanner scanner = new Scanner(System.in);
                url = scanner.next();
            }
        } else {
            url = args[0];
        }

        Downloader downloader = new Downloader();
        downloader.download(url);

    }
}