Javascript 需要打开一个URL数组';幻影中的幻影
我已经用phantomJs创建了一个脚本。它所做的是,它从一个工作正常的特定页面获取一些元素 代码如下:Javascript 需要打开一个URL数组';幻影中的幻影,javascript,phantomjs,Javascript,Phantomjs,我已经用phantomJs创建了一个脚本。它所做的是,它从一个工作正常的特定页面获取一些元素 代码如下: var page = new WebPage(), testindex = 0, loadInProgress = false, fs = require('fs'), i = 0, j = 0, k = 0; page.onConsoleMessage = function(msg) { console.log(msg); }; page.onLoadStarted = funct
var page = new WebPage(), testindex = 0, loadInProgress = false, fs = require('fs'), i = 0, j = 0, k = 0;
page.onConsoleMessage = function(msg) { console.log(msg); };
page.onLoadStarted = function() { loadInProgress = true; console.log("load started"); };
page.onLoadFinished = function() { loadInProgress = false; console.log("load finished"); };
// Sets the User Agent
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.107 Safari/537.36';
// Enable/Disable Javascript
// page.settings.javascriptEnabled = false;
var steps = [
function() { //Load Page
page.open("http://www.example.com/mobiles/");
page.injectJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js");
},
function() { //Fetch Products
page.onCallback = function(result) {
var fs = require('fs');
fs.write('product-list.csv', result, 'w+');
};
page.evaluate(function() {
var arr_mainList = new Array();
var arr_innerList = new Array();
try {
for (i = 0; i < (document.getElementsByClassName("item_grid")[0].getElementsByTagName("ul").length); i++) {
arr_mainList.push(document.getElementsByClassName("lap_thu_box")[i]);
window.callPhantom(arr_mainList[i].getElementsByTagName("h3")[0].getElementsByTagName("a")[0].textContent + ", ");
//window.callPhantom(arr_mainList[i].getElementsByTagName("h3")[0].getElementsByTagName("a")[0].href + ", ");
var myWindow = window.open(arr_mainList[i].getElementsByTagName("h3")[0].getElementsByTagName("a")[0].href);
console.log(myWindow.getElementsByClassName("item_desc")[0].textContent);
myWindow.close();
if (arr_mainList[i].getElementsByTagName("h3")[0].getElementsByTagName("a")[0].href.length > 43) {
var innerURL = arr_mainList[i].getElementsByTagName("h3")[0].getElementsByTagName("a")[0].href;
console.log(innerURL);
}
window.callPhantom(arr_mainList[i].getElementsByTagName("img")[0].getAttribute("data-original") + ", ");
arr_innerList.push(arr_mainList[i]);
for (j = 0; j < (document.getElementsByClassName("lap_thu_box")[i].getElementsByTagName("ul")[0].getElementsByTagName("li").length); j++) {
if ((j+1) < document.getElementsByClassName("lap_thu_box")[i].getElementsByTagName("ul")[0].getElementsByTagName("li").length) {
window.callPhantom(arr_innerList[i].getElementsByTagName("li")[j].textContent.replace(/,/g, "") + " | ");
}
else {
window.callPhantom(arr_innerList[i].getElementsByTagName("li")[j].textContent.replace(/,/g, "") + ", ");
}
};
//window.callPhantom(", ");
window.callPhantom(arr_innerList[i].getElementsByClassName("cat_price")[0].textContent.replace(/,/g, ""));
window.callPhantom("\n");
};
loadInProgress = true;
console.log("Successful.");
}
catch(ex) {
console.log("Failed: " + ex);
}
});
}
];
interval = setInterval(function() {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
setTimeout(function() {
//fs.write('product-list.html', page.content, 'w');
console.log("test complete!");
phantom.exit();
}, 100);
}
}, 5000);
var page=new WebPage(),testindex=0,loadInProgress=false,fs=require('fs'),i=0,j=0,k=0;
page.onConsoleMessage=函数(msg){console.log(msg);};
page.onLoadStarted=function(){loadInProgress=true;console.log(“load started”);};
page.onLoadFinished=function(){loadInProgress=false;console.log(“load finished”);};
//设置用户代理
page.settings.userAgent='Mozilla/5.0(windowsnt 6.3;WOW64)AppleWebKit/537.36(KHTML,比如Gecko)Chrome/44.0.2403.107 Safari/537.36';
//启用/禁用Javascript
//page.settings.javascriptEnabled=false;
变量步骤=[
函数(){//加载页面
第页打开(“http://www.example.com/mobiles/");
page.js(“http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js");
},
函数(){//获取产品
page.onCallback=函数(结果){
var fs=需要('fs');
fs.write('product-list.csv',结果'w+');
};
page.evaluate(函数(){
var arr_mainList=新数组();
var arr_innerList=新数组();
试一试{
对于(i=0;i<(document.getElementsByClassName(“item_grid”)[0]。getElementsByTagName(“ul”).length);i++){
arr_mainList.push(document.getElementsByClassName(“lap_-thu-box”)[i]);
window.callPhantom(arr_mainList[i].getElementsByTagName(“h3”)[0].getElementsByTagName(“a”)[0].textContent+“,”;
//window.callPhantom(arr_mainList[i].getElementsByTagName(“h3”)[0].getElementsByTagName(“a”)[0].href+“,”;
var myWindow=window.open(arr_mainList[i].getElementsByTagName(“h3”)[0].getElementsByTagName(“a”)[0].href);
console.log(myWindow.getElementsByClassName(“item_desc”)[0].textContent);
myWindow.close();
if(arr_mainList[i].getElementsByTagName(“h3”)[0].getElementsByTagName(“a”)[0].href.length>43){
var innerURL=arr_mainList[i]。getElementsByTagName(“h3”)[0]。getElementsByTagName(“a”)[0]。href;
log(innerURL);
}
callPhantom(arr_mainList[i].getElementsByTagName(“img”)[0].getAttribute(“原始数据”)+“,”;
arr_innerList.push(arr_mainList[i]);
对于(j=0;j<(document.getElementsByClassName(“lap_-thu-box”)[i]。getElementsByTagName(“ul”)[0]。getElementsByTagName(“li”).length);j++{
if((j+1)
现在,如果我运行该程序,我将获得csv文件中的所有信息。除非转到window.open,否则phantomJs将停止。我知道我无法在page.evaluate中打开新页面。但我需要获取产品描述并将其添加到csv文件中,以代替产品链接。我已经找了好几个小时了,任何帮助都很好。
注意:我的限制是我必须使用phantomJs。我对您的脚本做了一些修改。所以现在你想做什么就做什么。请记住,不要让很多物品报废,否则会出现内存问题。所以,如果在使用过的网站中存在分页,请为其使用新功能。 在这段代码中,我假设您需要每个设备的描述,但也可以访问其他元素 注意:您可能知道跨域策略不允许我们使用javascript/jQuery访问iFrame,这将是一个巨大的缺陷。您必须添加 --网络安全=否 在cmd/terminal中执行脚本时的标志
var page = new WebPage(), innerPage = new WebPage(), testindex = 0, loadInProgress = false, fs = require('fs'), i = 0, j = 0, k = 0;
page.onConsoleMessage = function(msg) { console.log(msg); };
page.onLoadStarted = function() { loadInProgress = true; console.log("load started"); };
page.onLoadFinished = function() { loadInProgress = false; console.log("load finished"); };
// Sets the User Agent
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.107 Safari/537.36';
// Enable/Disable Javascript
//page.settings.javascriptEnabled = false;
//IMPORTANT FLAGS
//--web-security=yes/no
var steps = [
function() { //Load Page
page.open("http://www.example.com/mobiles-apple/", function() {
page.injectJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js");
page.evaluate(function() {
try {
$("#main1").append('<div id="inner-data_iframes"></div>');
for (i = 0; i < (document.getElementsByClassName("item_grid")[0].getElementsByTagName("ul").length); i++) {
var iFrameAdd = document.getElementsByClassName("lap_thu_box")[i].getElementsByTagName("h3")[0].getElementsByTagName("a")[0].href;
$("#inner-data_iframes").append('<iframe id="myIframe' + [i] + '" src="' + iFrameAdd + '"></iframe>');
window.document.body.scrollTop = document.body.scrollHeight;
}
console.log("Mission Successful.");
}
catch(ex) {
console.log("Failed to add iFrame.");
}
});
});
},
function() { //Fetch Products
page.onCallback = function(result) {
var fs = require('fs');
fs.write('product-list.csv', result, 'w+');
};
page.evaluate(function() {
var arr_mainList = new Array();
var arr_innerList = new Array();
try {
for (i = 0; i < (document.getElementsByClassName("item_grid")[0].getElementsByTagName("ul").length); i++) {
arr_mainList.push(document.getElementsByClassName("lap_thu_box")[i]);
window.callPhantom(arr_mainList[i].getElementsByTagName("h3")[0].getElementsByTagName("a")[0].textContent + ", ");
var desc = $("#myIframe" + [i]).contents().find(".item_desc").html();
desc = desc.replace(/,/g, "");
window.callPhantom(desc + ", ");
window.callPhantom(arr_mainList[i].getElementsByTagName("img")[0].getAttribute("data-original") + ", ");
arr_innerList.push(arr_mainList[i]);
for (j = 0; j < (document.getElementsByClassName("lap_thu_box")[i].getElementsByTagName("ul")[0].getElementsByTagName("li").length); j++) {
if ((j+1) < document.getElementsByClassName("lap_thu_box")[i].getElementsByTagName("ul")[0].getElementsByTagName("li").length) {
window.callPhantom(arr_innerList[i].getElementsByTagName("li")[j].textContent.replace(/,/g, "") + " | ");
}
else {
window.callPhantom(arr_innerList[i].getElementsByTagName("li")[j].textContent.replace(/,/g, "") + ", ");
}
}
window.callPhantom(arr_innerList[i].getElementsByClassName("cat_price")[0].textContent.replace(/,/g, ""));
window.callPhantom("\n");
}
loadInProgress = true;
console.log("Successful.");
}
catch(ex) {
console.log("Failed: " + ex);
}
});
}
];
interval = setInterval(function() {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
setTimeout(function(){
//fs.write('product-list.html', page.content, 'w');
console.log("test complete!");
phantom.exit();
}, 100);
}
}, 5000);
var page=new WebPage(),innerPage=new WebPage(),testindex=0,loadInProgress=false,fs=require('fs'),i=0,j=0,k=0;
page.onConsoleMessage=函数(msg){console.log(msg);};
page.onLoadStarted=function(){loadInProgress=true;console.log(“load started”);};
page.onLoadFinished=function(){loadInProgress=false;console.log(“load finished”);};
//设置用户代理
page.settings.userAgent='Mozilla/5.0(windowsnt 6.3;WOW64)AppleWebKit/537.36(KHTML,比如Gecko)Chrome/44.0.2403.107 Safari/537.36';
//启用/禁用Javascript
//page.settings.javascriptEnabled=false;
//重要旗帜
//--web安全性=是/否
变量步骤=[
函数(){//加载页面
第页打开(“http://www.example.com/mobiles-apple/“,函数(){
page.js(“http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js");
第e页