Javascript 如何分离嵌套异步函数并按所需顺序执行它们?
我有一个异步函数,它将URL列表作为解析csv文件的值。 在这个函数中,我使用Puppeter启动一个页面并导航到每个url。 我有两个功能-一个用于下载资源,另一个用于检查资源是否已下载Javascript 如何分离嵌套异步函数并按所需顺序执行它们?,javascript,node.js,module,async-await,puppeteer,Javascript,Node.js,Module,Async Await,Puppeteer,我有一个异步函数,它将URL列表作为解析csv文件的值。 在这个函数中,我使用Puppeter启动一个页面并导航到每个url。 我有两个功能-一个用于下载资源,另一个用于检查资源是否已下载 我想先运行scriptDownload(),在它的执行完成后,运行networkProfile()函数 我还想知道这些功能是否可以在单独的文件中隔离,并应用模块导出和导入的功能 我还没有初始化所附代码中的函数 此外,稍后在networkProfile()中添加的异步mongodb命令也很少 我试过用。然后没用
const puppeter=require('puppeter');
var fse=要求('fs-extra');
var Papa=需要(“papaparse”);
const path=require('path');
var scr=需要('./脚本');
异步函数脚本(x){
const browser=wait puppeter.launch({
无头:是的,
参数:['--enable features=NetworkService'],
ignoreHTTPSErrors:正确
});
const page=wait browser.newPage();
const client=wait page.target().createCDPSession();
等待client.send('Network.enable');
等待client.send('Network.clearBrowserCache');
等待client.send('Runtime.enable');
等待client.send('Debugger.enable');
异步函数scriptDownload(){
page.on('response',async resp=>{
constURL=新url(page.url()).host;
const url1=resp.request().url();
var url2=path.basename(url1);
url2=url2。替换(/\?/g,&q');
const isScript=resp.request().resourceType()='script';
const isCSS=resp.request().resourceType()=“样式表”;
const isXHR=resp.request().resourceType()='xhr';
如果(isScript){
让filePath=path.resolve(`./Responses/${url}/js/${url2}.js`);
wait fse.outputFile(文件路径,wait resp.text());
log(“脚本下载”);
}
if(isCSS){
让filePath=path.resolve(`./Responses/${url}/css/${url2}.css`);
wait fse.outputFileSync(文件路径,wait resp.text());
log('CSS下载');
}
if(isXHR){
让filePath=path.resolve(`./Responses/${url}/xhr/${url2}.xhr`);
wait fse.outputFileSync(文件路径,wait resp.text());
log('XHR下载');
}
});
}
//网络多米安
异步函数networkProfile(){
client.on('Network.requestWillBeSent',参数=>{
const request_url=parameters.request.url;
var reqname=path.basename(请求url);
var reqnamereg=reqname.replace(/\?/g,&q');
const urlloc=新URL(page.URL()).host;
试一试{
/**************************检查文件是否存在*************************************************************************************/
const jsFilePath=path.resolve(`./Responses/${urlloc}/js/${reqnamereg}.js`);
const jsPath=path.resolve(`./Responses/${urlloc}/js/${reqnamereg}`);
var locStorage=“null”;
异步函数localjs(f){
const existsjs=等待fse.pathExists(f)
如果(existsjs==true){
locStorage=jsPath;
日志(urlloc、reqnamereg、existsjs);
}
否则{
locStorage='Js不存在';
日志(urlloc、reqnamereg、existsjs);
}
}
异步函数本地(f){
const exists=等待fse.pathExists(f)
if(exists==true){
locStorage=jsFilePath;
log(urlloc,reqname,exists);
}
否则{
locStorage='文件不存在';
log(urlloc,reqname,exists);
}
}
if(reqname.endsWith('.js')=='true'){
localjs(jsPath)
}
否则{
本地(jsFilePath)
}
}捕获(错误){
console.log(错误);
}
});
}
对于(var j=0;j{
等待页面。关闭();
等待浏览器关闭();
}, 60000 * 2);
}
file=fse.createReadStream(uu dirname+'/test.csv',utf8')
函数parseData(文件、回调){
parse(文件{
分隔符:“”,//自动检测
换行:,//自动检测
标题:false,
下载:对,
完成:功能(结果){
回调(results.data);
}
});
}
解析数据(文件、脚本);
预期结果将是
下载的脚本
下载CSS
<
const puppeteer = require('puppeteer');
var fse = require('fs-extra');
var Papa= require("papaparse");
const path = require('path');
var scr = require('./scripts');
async function scripts(x){
const browser = await puppeteer.launch({
headless: true,
args: ['--enable-features=NetworkService'],
ignoreHTTPSErrors:true
});
const page = await browser.newPage();
const client = await page.target().createCDPSession();
await client.send('Network.enable');
await client.send('Network.clearBrowserCache');
await client.send('Runtime.enable');
await client.send('Debugger.enable');
async function scriptDownload() {
page.on('response', async resp => {
const url = new URL(page.url()).host;
const url1 = resp.request().url();
var url2 = path.basename(url1);
url2 = url2.replace(/\?/g, '&q');
const isScript = resp.request().resourceType() === 'script';
const isCSS = resp.request().resourceType() === 'stylesheet';
const isXHR = resp.request().resourceType() === 'xhr';
if (isScript) {
let filePath = path.resolve(`./Responses/${url}/js/${url2}.js`);
await fse.outputFile(filePath, await resp.text());
console.log('Scripts Downloaded');
}
if (isCSS) {
let filePath = path.resolve(`./Responses/${url}/css/${url2}.css`);
await fse.outputFileSync(filePath, await resp.text());
console.log('CSS Downloaded');
}
if (isXHR) {
let filePath = path.resolve(`./Responses/${url}/xhr/${url2}.xhr`);
await fse.outputFileSync(filePath, await resp.text());
console.log('XHR Downloaded');
}
});
}
//network domian
async function networkProfile() {
client.on('Network.requestWillBeSent', parameters => {
const request_url = parameters.request.url;
var reqname = path.basename(request_url);
var reqnamereg = reqname.replace(/\?/g, '&q');
const urlloc = new URL(page.url()).host;
try {
/**************************Check if file exists********************************************************** */
const jsFilePath = path.resolve(`./Responses/${urlloc}/js/${reqnamereg}.js`);
const jsPath = path.resolve(`./Responses/${urlloc}/js/${reqnamereg}`);
var locStorage = "null";
async function localjs(f) {
const existsjs = await fse.pathExists(f)
if (existsjs == true) {
locStorage = jsPath;
console.log(urlloc, reqnamereg, existsjs);
}
else {
locStorage = 'Js does not exist';
console.log(urlloc, reqnamereg, existsjs);
}
}
async function local(f) {
const exists = await fse.pathExists(f)
if (exists == true) {
locStorage = jsFilePath;
console.log(urlloc, reqname, exists);
}
else {
locStorage = 'File does not exist';
console.log(urlloc, reqname, exists);
}
}
if (reqname.endsWith('.js') == 'true') {
localjs(jsPath)
}
else {
local(jsFilePath)
}
} catch (error) {
console.log(error);
}
});
}
for (var j = 0; j < x.length; j++){
try{
await page.goto('https://www.' + x[j][1], { waitUntil: 'load' });
const url = new URL(page.url()).host;
let filePathhtml = path.resolve(`./Responses/${url}/html/${url}.html`);
await fse.outputFileSync(filePathhtml, await page.content());
}
catch(error){
if(error==='net::ERR_CONNECTION_TIMED_OUT'){
}
}
}
setTimeout(async () => {
await page.close();
await browser.close();
}, 60000 * 2);
}
file = fse.createReadStream(__dirname + '/test.csv', 'utf8')
function parseData(file, callBack) {
Papa.parse(file, {
delimiter: "", // auto-detect
newline: "", // auto-detect
header: false,
download: true,
complete: function(results) {
callBack(results.data);
}
});
}
parseData(file, scripts);
async function scriptx(){
await scriptDownload(page);
await networkProfile(client);
}
async function scriptDownload(page){
}
async function networkProfile(client){
}