6

利用github给国外文件下载加速

 3 years ago
source link: https://www.daqianduan.com/20837.html
Go to the source link to view the article. You can view the picture content, updated content and better typesetting reading experience. If the link is broken, please click the button below to view the snapshot at that time.

前言

作为一名程序员，经常需要下载一些编程相关的环境，而国内的网络环境大家都知道，有的文件用浏览器是下载不动的，于是我有了利用github下载文件的想法。

我的demo项目地址： https://github.com/bobowire/wireboy.remote.download

参考资料：

  1. NodeJS使用node-fetch下载文件并显示下载进度示例： https://www.jianshu.com/p/4b58711cb72a
  2. nodejs——发送邮件（带附件）： https://www.cnblogs.com/yourstars/p/6728931.html

觉得好玩的，大家可以点个赞~

下载加速原理

使用github的Action远程执行文件下载（下载qt环境速度可以达到3mb/s），然后将下载的文件进行分片，每片15mb，分片后的文件以邮件附件的方式发送到国内邮箱，我们通过下载邮箱中的附件，将分片的附件合并成完整文件，从而实现不翻墙、不用下载器也能下载国外文件的目的。

简单点说

  1. github远程下载
  2. 文件分片
  3. 通过邮箱发到国内
  4. 对附件进行合并

使用方法

  1. 新建github项目
  2. 创建js文件download.js文件，内容请查阅后文
  3. 创建workflows/RunDownload.yml文件，内容请查阅后文
  4. 修改download.js中的fileURL 变量值，此为文件url地址
  5. 在项目github->settings->Secrets中，点击右上方“new responsitory secret”按钮，添加”EMAILPASS”,”SENDEMAIL”,”TOEMAIL”变量（授权码、发送邮箱、目标邮箱）
  6. 以上全部完成后，我们每次修改download.js文件的fileURL地址，github都会自动进行一次下载。原理请自行百度“github action”。

注意：

  1. 授权码（EMAILPASS）是指“邮箱第三方登录授权码”，如何获取授权码，以QQ邮箱为例，请点击： http://jingyan.baidu.com/article/fedf0737af2b4035ac8977ea.html

github Action文件（RunDownload.yml）

name: Github wireboy.remote.download
 
on:
    push:
        branches:
            - main
    schedule:
     - cron: '* * * * *'
jobs:
    build:
        runs-on: ubuntu-latest
 
        steps:
            - name: Checkout codes
              uses: actions/checkout@v2
            - name: Use Node.js
              uses: actions/setup-node@v1
              with:
                node-version: '12.x'
            - name: Run
              run: npm install
              
            - run: node download.js
              
              env:
                EMAILPASS: ${{ secrets.EMAILPASS }}
                SENDEMAIL: ${{ secrets.SENDEMAIL }}
                TOEMAIL: ${{ secrets.TOEMAIL }}

源代码（download.js）

const fetch = require("node-fetch");
const fs = require("fs");
const path = require("path");
const progressStream = require('progress-stream');
const nodemailer = require('nodemailer');


//下载 的文件 地址 （https://nodejs.org/dist/v12.18.3/node-v12.18.3-x64.msi）
let fileURL = 'https://nodejs.org/dist/v12.18.3/node-v12.18.3-x64.msi';


//下载保存的文件路径
let fileSavePath = path.join(__dirname, path.basename(fileURL));
//缓存文件路径
let tmpFileSavePath = fileSavePath + ".tmp";
//创建写入流
const fileStream = fs.createWriteStream(tmpFileSavePath).on('error', function (e) {
    console.error('error==>', e)
}).on('ready', function () {
    console.log("开始下载:", fileURL);
}).on('finish', function () {
    //下载完成后重命名文件
    fs.renameSync(tmpFileSavePath, fileSavePath);
    console.log('文件下载完成:', fileSavePath);
    const readstream = fs.createReadStream(fileSavePath);
	let i = 0;
	console.time('readtime');
    let patchIndex = 0;
	readstream.on('readable', () => {
        {
            console.log('start read');
            let chunk = readstream.read(1024 * 1024 * 15);
            while (null !== chunk) {
                patchIndex = patchIndex + 1;
                console.log('read times:'+patchIndex)
                console.log(fileSavePath+'.email_'+patchIndex);
                let emailFile = fs.createWriteStream(fileSavePath+'.email_'+patchIndex).on('finish',function(){
    
                })
                emailFile.write(chunk);
                emailFile.end();
                let msg = createEmailMessage(patchIndex+'_'+path.basename(fileURL),fileSavePath+'.email_'+patchIndex,path.basename(fileURL) + '(' + patchIndex + ')');
                console.log('Send Mail ' + patchIndex + ' times');
                console.log(path.basename(fileURL));
                var transporter = createTransporter();
                transporter.sendMail(msg, (error, info) => {
                    if (error) {
                        console.log('Error occurred');
                        console.log(error.message);
                        return;
                    }
                    console.log('Message sent successfully!');
                    console.log('Server responded with "%s"', info.response);
                    transporter.close();
                });
    
                chunk = readstream.read(1024 * 1024 * 10);
            }
            console.log('end read');
        }
	});
	readstream.on('close', () => {
		console.timeEnd('readtime');
    });
    
});
//请求文件
fetch(fileURL, {
    method: 'GET',
    headers: { 'Content-Type': 'application/octet-stream' },
    // timeout: 100,    
}).then(res => {
    //获取请求头中的文件大小数据
    let fsize = res.headers.get("content-length");
    //创建进度
    let str = progressStream({
        length: fsize,
        time: 100 /* ms */
    });
    // 下载进度 
    str.on('progress', function (progressData) {
        //不换行输出
        let percentage = Math.round(progressData.percentage) + '%';
        console.log(percentage);
        // process.stdout.write('\033[2J'+);
        // console.log(progress);
        /*
        {
            percentage: 9.05,
            transferred: 949624,
            length: 10485760,
            remaining: 9536136,
            eta: 42,
            runtime: 3,
            delta: 295396,
            speed: 949624
        }
        */
    });
    res.body.pipe(str).pipe(fileStream);
}).catch(e => {
    //自定义异常处理
    console.log(e);
});



var createTransporter = function(){
    return nodemailer.createTransport({
        service: 'qq',
        auth: {
            user: process.env.SENDEMAIL,//发送者邮箱
            pass: process.env.EMAILPASS //邮箱第三方登录授权码
        },
       //  logger: bunyan.createLogger({
       //      name: 'nodemailer'
       //  }),//打印日志
        debug: true
    },{
        from: process.env.SENDEMAIL,//发送者邮箱
        headers: {
            'X-Laziness-level': 1000
        }
    });
} 

console.log('SMTP Configured');

var createEmailMessage = function(filename,filepath,subject){
    var message = {
        // Comma separated lsit of recipients 收件人用逗号间隔
        to: process.env.TOEMAIL,
    
        // Subject of the message 信息主题
        subject:  subject,
    
        // plaintext body
        text: '请查阅附件',
    
        // Html body
        html: '<p>下载文件成功</p>',
    
        // Apple Watch specific HTML body 苹果手表指定HTML格式
        watchHtml: '<b>Hello</b> to myself',
    
        // An array of attachments 附件
        attachments: [
            // String attachment
           //  {
           //      filename: 'notes.txt',
           //      content: 'Some notes about this e-mail',
           //      contentType: 'text/plain' // optional,would be detected from the filename 可选的，会检测文件名
           //  },
           //  // Binary Buffer attchment
           //  {
           //      filename: 'image.png',
           //      content: Buffer.from('iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD/' +
           //         '//+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4U' +
           //         'g9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC', 'base64'),
           //      cid: '00001'  // should be as unique as possible 尽可能唯一
           //  },
            // File Stream attachment
            {
                filename: filename,
                path: filepath,
                // cid: '00002'  // should be as unique as possible 尽可能唯一
             }
        ]
    
    };
    return message;
};

效果图

收件箱

2A3u2m.png

BF3mmm.png

github的action运行日志

uQbuem.png

#感谢您访问本站#
#本文转载自互联网，若侵权，请联系删除，谢谢！657271#qq.com#

report

Recommend

About Joyk


Aggregate valuable and interesting links.
Joyk means Joy of geeK