0

java多线程网页下载代码

 2 years ago
source link: https://byronhe.com/post/2011/11/15/java-multithread-download-webpage/
Go to the source link to view the article. You can view the picture content, updated content and better typesetting reading experience. If the link is broken, please click the button below to view the snapshot at that time.

java多线程网页下载代码

2011-11-15

小项目,练手的。

1.使用了java.util.concurrent包里的线程池,可以飙升到满带宽,在100M带宽上,可以达到10MB/s。

2.使用了java.nio里的channels,性能比自己缓冲有一些提高。

import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.util.Calendar;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class HttpDownloader implements Callable {
    URLConnection connection;
    FileChannel outputChann;
    public static volatile int count = 0;

    public static void main(String[] args) throws Exception {

        ExecutorService poll = Executors.newFixedThreadPool(100);

        for (int i = 0; i < 100; i++) {
            Calendar now = Calendar.getInstance();
            String fileName = "../data/" + now.get(Calendar.YEAR) + "年"
                    + (now.get(Calendar.MONTH) + 1) + "月"
                    + now.get(Calendar.DAY_OF_MONTH) + "日--" + i + ".txt";
            poll.submit(new HttpDownloader("http://www.sina.com",
                    (new FileOutputStream(fileName)).getChannel()));
        }

        poll.shutdown();

        long start = System.currentTimeMillis();
        while (!poll.isTerminated()) {
            Thread.sleep(1000);
            System.out.println("已运行"
                    + ((System.currentTimeMillis() - start) / 1000) + "秒,"
                    + HttpDownloader.count + "个任务还在运行");
        }
    }

    public HttpDownloader(String url, FileChannel fileChannel) throws Exception {
        synchronized (HttpDownloader.class) {
            count++;
        }
        connection = (new URL(url)).openConnection();
        this.outputChann = fileChannel;
    }

    @Override
    public String call() throws Exception {
        connection.connect();
        InputStream inputStream = connection.getInputStream();
        ReadableByteChannel rChannel = Channels.newChannel(inputStream);
        outputChann.transferFrom(rChannel, 0, Integer.MAX_VALUE);
        // System.out.println(Thread.currentThread().getName() + " completed!");
        inputStream.close();
        outputChann.close();
        synchronized (HttpDownloader.class) {
            count--;
        }
        return null;
    }
}

About Joyk


Aggregate valuable and interesting links.
Joyk means Joy of geeK