nodejs爬虫下载文件(arcgis api for js sdk包)

已被阅读 1497 次 | 文章分类:nodejs | 2021-06-21 23:42

做过arcgis webgis二次开发的同学应该深有体会,arcgis api for js下载一波三折,首先需要到官网注册账号,然后下载所需版本api,但是该网站对国内网速不友好,有时候几十m文件下载很慢,所以我写脚本爬取下载使用,下面是全部的nodejs代码,本站也将下载的文件提供出来,方便大家下载,arcgis api for js所有版本

1 下载主文件脚本-index.js

新建文件夹,然后新建index.js,键入如下代码

                                            
const request = require('request');
const fs = require('fs');
const path = require('path');
const ApiPromise = require('./getAPiList');
const config = require('./config');
// 创建文件夹目录
var dirPath = path.join(__dirname, 'file');
let token = config.token;
ApiPromise.then(list => {
  debugger;
  // 已经存在的文件
  var existedApis = fs.readdirSync(dirPath);
  for (let i = 0; i < list.length - 1; i++) {
    // 不存在则下载
    let item = list[i];
    if (existedApis.indexOf(item.name) == -1) {
      let url = `https://downloads.arcgis.com/dms/rest/download/secured/${item.name}?token=${token}&folder=${item.folder}`;
      request(url, function(error, response, body) {
        if (!error && response.statusCode == 200) {
          // 获取下载路径
          let zipUrl = JSON.parse(response.body).url.toString();
          console.log(zipUrl);
          let startTime = new Date().getTime();
          let stream = fs.createWriteStream(path.join(dirPath, `${item.name}`));
          request(zipUrl)
            .pipe(stream)
            .on('close', function(err) {
              let total = parseFloat((new Date().getTime() - startTime) / 1000);
              console.log(`${item.name}下载完毕,耗时${total}s`);
              process.stdout.write('\x07');
            });
        }
      });
    } else {
      console.log('已經存在');
    }
  }
});
                                            
                                        

本文件中,依赖包有以下

path,fs 是nodejs内置的包

request包,是我们用来爬虫,请求接口的包,需要自己安装,在根目录执行 npm install --save request即可,会自动生成package.json文件;

config是将登陆后获取的token配置下来;getAPiList是获取所有sdk的zip下载链接

2 config.js

这里的token,是我注册账号后,获取的token,大家有兴趣可以自己注册后,替换为自己的token值

                                            
const config = {
  token:
    '-vnDPqZacQU_2I97Ww2AQWr_aajbRot-T9ET9xjkJLszN0VYbJPlHFpjcMjhpA8StucCP0uKgWugDCfEDDa6jYEOdT_2HzFiccplMpGjh8f_Lf7Ta1AggLjnSd6iuLCnXPeqTK6D6Fq07vCAXWtmfZgHB471fnvkwu0IVktCgJaW2vmpuDrnLBHjxYbKUKpX'
};
module.exports = config;
                                            
                                        

3 getAPiList.js

                                            
var request = require('request');
let url = `https://developers.arcgis.com/data/downloads.json`;
let result;
let infoArr = [];
module.exports = new Promise(function(resolve, reject) {
  request(url, function(error, response, body) {
    debugger;
    if (!error && response.statusCode == 200) {
      result = JSON.parse(response.body);
      result = result.filter(item => {
        return item.product_name == 'ArcGIS API for JavaScript';
      })[0];
      let allVersions = result.downloads;
      let versionkeys = Object.keys(allVersions);
      versionkeys.forEach(version => {
        let version_number = allVersions[version].version_number;
        let apifile = allVersions[version].files[0];
        let obj = {};
        obj['name'] = apifile.filename;
        obj['folder'] = apifile.folder;
        infoArr.push(obj);
      });
      resolve(infoArr);
    }
  });
});
                                            
                                        

/net/upload/image/20210621/ac0a7ed6-866b-4618-86f6-6efbc36ef18d.jpg

QQ:3410192267 | 技术支持 微信:popstarqqsmall

Copyright ©2017 xiaobaigis.com . 版权所有 鲁ICP备17027716号