Warning: file_get_contents(/data/phpspider/zhask/data//catemap/9/javascript/428.json): failed to open stream: No such file or directory in /data/phpspider/zhask/libs/function.php on line 167

Warning: Invalid argument supplied for foreach() in /data/phpspider/zhask/libs/tag.function.php on line 1116

Notice: Undefined index: in /data/phpspider/zhask/libs/function.php on line 180

Warning: array_chunk() expects parameter 1 to be array, null given in /data/phpspider/zhask/libs/function.php on line 181
Javascript节点-一次请求过多?_Javascript_Node.js_Httprequest - Fatal编程技术网

Javascript节点-一次请求过多?

Javascript节点-一次请求过多?,javascript,node.js,httprequest,Javascript,Node.js,Httprequest,我对请求有问题,因为大约有1-3个请求跑掉了,我不知道为什么。我放了两个变量:counter和check,它们应该相等,但通常不相等 如果我在小数据(例如1个服务)上检查此项,则它可以工作,但所有数据都无法工作 var request = require('request'); var UserAgent = 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0' var check = "" var c

我对请求有问题,因为大约有1-3个请求跑掉了,我不知道为什么。我放了两个变量:counter和check,它们应该相等,但通常不相等

如果我在小数据(例如1个服务)上检查此项,则它可以工作,但所有数据都无法工作

var request = require('request');

var UserAgent = 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101     Firefox/36.0'

var check = ""
var counter = ""

services = [
{
    name: "otodom",
    url: 'http://otodom.pl/',
    css_selector: '.search__counter',
    sell: {
        individual: {
            house: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=House',
            terrain: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Terrain',
            flat: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Flat',
        },
        totalUser: {
            house: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=House',
            terrain: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Terrain',
            flat: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Flat',
        }
    },
    rent: {
        individual: {
            house: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=rent&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=House',
            terrain: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=rent&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Terrain',
            flat: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=rent&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Flat',
            room: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=rent&objSearchQuery.OnlyFromRegularUser=y&objSearchQuery.ObjectName=Room',
        },
        totalUser: {
            house: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=House',
            terrain: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Terrain',
            flat: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Flat',
            room: 'http://otodom.pl/index.php?mod=listing&source=context&objSearchQuery.OfferType=sell&objSearchQuery.ObjectName=Room',
        }
    }
},
{
    name: "olx",
    url: 'http://olx.pl/nieruchomosci/',
    css_selector: '.color-2',
    sell: {
        individual: {
            house: 'http://olx.pl/nieruchomosci/domy/sprzedaz/',
            terrain: 'http://olx.pl/nieruchomosci/dzialki/sprzedaz/',
            flat: 'http://olx.pl/nieruchomosci/mieszkania/sprzedaz/',
        },
        totalUser: {
            house: 'http://olx.pl/nieruchomosci/mieszkania/sprzedaz/',
            terrain: 'http://olx.pl/nieruchomosci/dzialki/sprzedaz/',
            flat: 'http://olx.pl/nieruchomosci/mieszkania/sprzedaz/',
        }
    },
    rent: {
        individual: {
            house: 'http://olx.pl/nieruchomosci/domy/wynajem/',
            terrain: 'http://olx.pl/nieruchomosci/dzialki/wynajem/',
            room: 'http://olx.pl/nieruchomosci/stancje-pokoje/',
            flat: 'http://olx.pl/nieruchomosci/mieszkania/wynajem/',
        },
        totalUser: {
            house: 'http://olx.pl/nieruchomosci/mieszkania/wynajem/',
            terrain: 'http://olx.pl/nieruchomosci/dzialki/wynajem/',
            room: 'http://olx.pl/nieruchomosci/stancje-pokoje/',
            flat: 'http://olx.pl/nieruchomosci/mieszkania/wynajem/',
        }
    }
},
{
    name: "gratka.pl",
    url: 'http://dom.gartka.pl/',
    css_selector: '.ilosc',
    sell: {
        individual: {
            house: 'http://dom.gratka.pl/domy-sprzedam/lista/,,on,zi.html',
            terrain: 'http://dom.gratka.pl/dzialki-grunty-sprzedam/lista/,,on,zi.html',
            flat: 'http://dom.gratka.pl/mieszkania-sprzedam/lista/,,on,zi.html',
        },
        totalUser: {
            house: 'http://dom.gratka.pl/domy-sprzedam/lista/',
            terrain: 'http://dom.gratka.pl/dzialki-grunty-sprzedam/lista/',
            flat: 'http://dom.gratka.pl/mieszkania-sprzedam/lista/',
        }
    },
    rent: {
        individual: {
            house: 'http://dom.gratka.pl/domy-do-wynajecia/lista/,,on,zi.html',
            terrain: 'http://dom.gratka.pl/dzialki-grunty-do-wydzierzawienia/lista/,,on,zi.html',
            room: 'http://dom.gratka.pl/pokoje-do-wynajecia/lista/,,on,zi.html',
            flat: 'http://dom.gratka.pl/mieszkania-do-wynajecia/lista/,,on,zi.html',
        },
        totalUser: {
            house: 'http://dom.gratka.pl/domy-do-wynajecia/lista/',
            terrain: 'http://dom.gratka.pl/dzialki-grunty-do-wydzierzawienia/lista/',
            room: 'http://dom.gratka.pl/pokoje-do-wynajecia/lista/',
            flat: 'http://dom.gratka.pl/mieszkania-do-wynajecia/lista/',
        }
    }
}
]

这是一个简短的脚本

function do_get (page, category, typ, user, service) {
   counter += '1'
   console.log(counter.length)
   request({url: page, headers: {'User-Agent': UserAgent}}, function(buffer,   res, body) {
     check += '1'
     console.log(check.length)
});
}

功能执行(服务){
for(服务中的var用户.sell){
var users=service.sell[user];
for(用户中的var类别){
do__get(用户[类别]、类别“销售”、用户、服务);
}
}
for(服务中的var用户。租金){
var users=service.rent[user];
for(用户中的var类别){
do__get(用户[类别]、类别“租金”、用户、服务);
}
}
}
对于(var i=0;i
每当一个或多个请求运行时,
计数器和
检查将不相同。除此之外,我不明白问题是什么。我不知道为什么我没有得到所有URL的响应。您是否意识到您正在同时启动所有请求,以便它们将快速连续地命中服务器,然后您正在等待所有结果。很可能您超过了node.js套接字的限制。是的,我知道所有这些请求都是一次启动的,但我不知道如何一步一步地启动。我试着用回调函数编写函数,但仍然无法实现我的目标。
function do_that (service) {
    for(var user in service.sell) {
        var users = service.sell[user];
        for (var category in users) {
            do_get(users[category], category, "sell", user, service);
       }
    }
    for(var user in service.rent) {
         var users = service.rent[user];
         for (var category in users) {
            do_get(users[category], category, "rent", user, service);
         }
    }
}

for(var i = 0; i < services.length; i++) {
    do_that(services[i]);
}