har.js
const puppeteer = require(‘puppeteer‘);const PuppeteerHar = require(‘puppeteer-har‘);/*(async () => { ?const browser = await puppeteer.launch(); ?const page = await browser.newPage(); ?const har = new PuppeteerHar(page); ?await har.start({ path: ‘results.har‘ }); ?await page.goto(‘http://example.com‘); ?await har.stop(); ?await browser.close();})();*/async function saveHarlog(url,filename){ ?//启动浏览器实例 [puppeteer.createBrowserFetcher([options])] ?let browser = await puppeteer.launch({ ???// 若是手动下载的chromium需要指定chromium地址, 默认引用地址为 /项目目录/node_modules/puppeteer/.local-chromium/ ???//executablePath: ‘/Users/huqiyang/Documents/project/z/chromium/Chromium.app/Contents/MacOS/Chromium‘, ???//设置超时时间 ???timeout: 15000, ???//如果是访问https页面 此属性会忽略https错误 ???ignoreHTTPSErrors: true, ???// 关闭headless模式, 不会打开浏览器 ???headless: false, ???args:["--disk-cache-size=0","--disable-cache",‘--disable-infobars‘], ???//是否为每个选项卡自动打开DevTools面板。 如果此选项为true，则headless选项将设置为false。 ???devtools:false ?}); ?//创建一个新页面 ?let page = await browser.newPage(); ?//Puppeteer 初始化的屏幕大小默认为 800px x 600px。但是这个尺寸可以通过 Page.setViewport() 设置。 ?await page.setViewport({ ???????width: 800, ???????height: 600 ?}); ?let har = new PuppeteerHar(page); ?await har.start({ path: (filename +‘.har‘) }); ?await page.goto(url); ?// Get the "viewport" of the page, as reported by the page. ?let dimensions = await page.evaluate(() => { ???return { ?????width: document.documentElement.clientWidth, ?????height: document.documentElement.clientHeight, ?????deviceScaleFactor: window.devicePixelRatio ???}; ?}); ?await har.stop(); ?await browser.close();}exports.saveHarlog = saveHarlog;cvsresovler.js
const fs = require("fs");const path = require("path");const csv =require(‘csv‘);const parse = require(‘csv-parse/lib/sync‘)const iconv = require(‘iconv-lite‘); ??/*npm install iconv-lite */function readUrlRecord(csvpath){ ???console.log(‘开始解析文件:‘ + csvpath) ; ???//读取文件 ???const input = fs.readFileSync(csvpath,‘utf8‘) ; ???/* ???解析文件,生成JSON格式 ???{ ‘ ??‘: ‘142‘, ???AREA_NAME: ‘湖北‘, ???SITE_LINK: ‘www.banggo.com‘, ???BEARING_MODE: ‘移动接入‘, ???SITE_NAME: ‘邦购‘, ???MENU_TYPE: ‘二级‘ } ???*/ ???const records = parse(input, { ?????columns: true, ?????skip_empty_lines: true, ?????delimiter: ‘,‘, ???}) ; ???return records ;}//readUrlRecord(‘../top300.csv‘) ;exports.readUrlRecord = readUrlRecord;main.js
const fs = require("fs");const path = require("path");const moment = require("moment");const schedule = require(‘node-schedule‘);const cvsresovler=require("./module/cvsresovler");const mhar=require("./module/har");/*cnpm install --save momentcnpm install --save csvcnpm install --save node-schedulecnpm install --save puppeteercnpm install --save puppeteer-harcnpm install --save iconv-lite*/ ?function init(){ ???????console.log(‘初始化调度器‘) ; ???//每分钟的第30秒定时执行一次: ???schedule.scheduleJob(‘0 21 * * * *‘,()=>{ ???????let ftime = moment().format(‘YYYYMMDDHHmm‘); ???????console.log(‘当前调度时间为:‘ + ftime) ; ???????let dirPath = path.join(__dirname,‘harlogs‘,ftime) ; ???????console.log("创建目录:" + dirPath) ; ???????let isExist = false ; ???????if(fs.existsSync(dirPath)){ ???????????????//创建文件夹 ???????????????let stat = fs.lstatSync(dirPath); ???????????????if(stats.isDirectory()){ ???????????????????isExist = false ; ???????????????} ???????} ???????if(!isExist){ ???????????//创建文件夹 ???????????console.log("创建文件夹" + ftime) ; ???????????????fs.mkdirSync(dirPath); ???????} ??????????????//开始解析需要处理的URL ???????let dataArr = cvsresovler.readUrlRecord(path.join(__dirname,‘top300.csv‘)) ; ???????console.log("解析出URL共计" + dataArr.length + "条") ; ???????/* ???????????开始抓取HAR数据【同步的方式执行】。 ???????????注意：如果这里直接通过for循环遍历dataArr并调用saveHarlog方法，那么这将是一个异步的过程。 ???????*/ ???????(async function iterator(i){ ???????????let data = ?dataArr[i] ???????????let url = data[‘SITE_LINK‘] ; ???????????console.log("开始处理：" + url ) ; ???????????await mhar.saveHarlog(‘http://‘ + url,path.join(dirPath,url.replace(‘/‘,"-"))) ; ???????????if(i + 1 < dataArr.length){ ???????????????iterator(i+1) ; ???????????} ???})(0) ; ?}); ??console.log(‘应用程序启动完成‘) ;}//执行init();Nodejs中puppeteer抓取浏览器HAR数据
原文地址：https://blog.51cto.com/dengshuangfu/2365190