Javascript节点-一次请求过多?
我对请求有问题,因为大约有1-3个请求跑掉了,我不知道为什么。我放了两个变量:counter和check,它们应该相等,但通常不相等 如果我在小数据(例如1个服务)上检查此项,则它可以工作,但所有数据都无法工作Javascript节点-一次请求过多?,javascript,node.js,httprequest,Javascript,Node.js,Httprequest,我对请求有问题,因为大约有1-3个请求跑掉了,我不知道为什么。我放了两个变量:counter和check,它们应该相等,但通常不相等 如果我在小数据(例如1个服务)上检查此项,则它可以工作,但所有数据都无法工作 var request = require('request'); var UserAgent = 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0' var check = "" var c
var request = require('request');
var UserAgent = 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0'
var check = ""
var counter = ""
services = [
{
name: "otodom",
url: 'http://otodom.pl/',
css_selector: '.search__counter',
sell: {
individual: {
house: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=House',
terrain: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Terrain',
flat: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Flat',
},
totalUser: {
house: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=House',
terrain: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Terrain',
flat: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Flat',
}
},
rent: {
individual: {
house: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=rent&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=House',
terrain: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=rent&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Terrain',
flat: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=rent&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Flat',
room: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=rent&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Room',
},
totalUser: {
house: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=House',
terrain: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Terrain',
flat: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Flat',
room: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Room',
}
}
},
{
name: "olx",
url: 'http://olx.pl/nieruchomosci/',
css_selector: '.color-2',
sell: {
individual: {
house: 'http://olx.pl/nieruchomosci/domy/sprzedaz/',
terrain: 'http://olx.pl/nieruchomosci/dzialki/sprzedaz/',
flat: 'http://olx.pl/nieruchomosci/mieszkania/sprzedaz/',
},
totalUser: {
house: 'http://olx.pl/nieruchomosci/mieszkania/sprzedaz/',
terrain: 'http://olx.pl/nieruchomosci/dzialki/sprzedaz/',
flat: 'http://olx.pl/nieruchomosci/mieszkania/sprzedaz/',
}
},
rent: {
individual: {
house: 'http://olx.pl/nieruchomosci/domy/wynajem/',
terrain: 'http://olx.pl/nieruchomosci/dzialki/wynajem/',
room: 'http://olx.pl/nieruchomosci/stancje-pokoje/',
flat: 'http://olx.pl/nieruchomosci/mieszkania/wynajem/',
},
totalUser: {
house: 'http://olx.pl/nieruchomosci/mieszkania/wynajem/',
terrain: 'http://olx.pl/nieruchomosci/dzialki/wynajem/',
room: 'http://olx.pl/nieruchomosci/stancje-pokoje/',
flat: 'http://olx.pl/nieruchomosci/mieszkania/wynajem/',
}
}
},
{
name: "gratka.pl",
url: 'http://dom.gartka.pl/',
css_selector: '.ilosc',
sell: {
individual: {
house: 'http://dom.gratka.pl/domy-sprzedam/lista/,,on,zi.html',
terrain: 'http://dom.gratka.pl/dzialki-grunty-sprzedam/lista/,,on,zi.html',
flat: 'http://dom.gratka.pl/mieszkania-sprzedam/lista/,,on,zi.html',
},
totalUser: {
house: 'http://dom.gratka.pl/domy-sprzedam/lista/',
terrain: 'http://dom.gratka.pl/dzialki-grunty-sprzedam/lista/',
flat: 'http://dom.gratka.pl/mieszkania-sprzedam/lista/',
}
},
rent: {
individual: {
house: 'http://dom.gratka.pl/domy-do-wynajecia/lista/,,on,zi.html',
terrain: 'http://dom.gratka.pl/dzialki-grunty-do-wydzierzawienia/lista/,,on,zi.html',
room: 'http://dom.gratka.pl/pokoje-do-wynajecia/lista/,,on,zi.html',
flat: 'http://dom.gratka.pl/mieszkania-do-wynajecia/lista/,,on,zi.html',
},
totalUser: {
house: 'http://dom.gratka.pl/domy-do-wynajecia/lista/',
terrain: 'http://dom.gratka.pl/dzialki-grunty-do-wydzierzawienia/lista/',
room: 'http://dom.gratka.pl/pokoje-do-wynajecia/lista/',
flat: 'http://dom.gratka.pl/mieszkania-do-wynajecia/lista/',
}
}
}
]
这是一个简短的脚本
function do_get (page, category, typ, user, service) {
counter += '1'
console.log(counter.length)
request({url: page, headers: {'User-Agent': UserAgent}}, function(buffer, res, body) {
check += '1'
console.log(check.length)
});
}
功能执行(服务){
for(服务中的var用户.sell){
var users=service.sell[user];
for(用户中的var类别){
do__get(用户[类别]、类别“销售”、用户、服务);
}
}
for(服务中的var用户。租金){
var users=service.rent[user];
for(用户中的var类别){
do__get(用户[类别]、类别“租金”、用户、服务);
}
}
}
对于(var i=0;i
每当一个或多个请求运行时,计数器和检查将不相同。除此之外,我不明白问题是什么。我不知道为什么我没有得到所有URL的响应。您是否意识到您正在同时启动所有请求,以便它们将快速连续地命中服务器,然后您正在等待所有结果。很可能您超过了node.js套接字的限制。是的,我知道所有这些请求都是一次启动的,但我不知道如何一步一步地启动。我试着用回调函数编写函数,但仍然无法实现我的目标。
function do_that (service) {
for(var user in service.sell) {
var users = service.sell[user];
for (var category in users) {
do_get(users[category], category, "sell", user, service);
}
}
for(var user in service.rent) {
var users = service.rent[user];
for (var category in users) {
do_get(users[category], category, "rent", user, service);
}
}
}
for(var i = 0; i < services.length; i++) {
do_that(services[i]);
}