Node.js findOneAndUpdate猫鼬与';只有';20k异步更新
我需要从Google Sheets读取csv文件并更新mongodb数据库。该图纸有20k到25k行(每行约30列)。如果我将工作表的读取限制在10k行以下,我会做得很好,但是如果我超过了10k行,我会开始表现出奇怪的行为,并且需要很长时间才能最终运行所有异步FindOneAndUpdates。下面是我正在使用的代码(我创建了一个回调到console.log的计数器,当它应该完成时):Node.js findOneAndUpdate猫鼬与';只有';20k异步更新,node.js,mongodb,mongoose,Node.js,Mongodb,Mongoose,我需要从Google Sheets读取csv文件并更新mongodb数据库。该图纸有20k到25k行(每行约30列)。如果我将工作表的读取限制在10k行以下,我会做得很好,但是如果我超过了10k行,我会开始表现出奇怪的行为,并且需要很长时间才能最终运行所有异步FindOneAndUpdates。下面是我正在使用的代码(我创建了一个回调到console.log的计数器,当它应该完成时): 你找到解决办法了吗? const mongooseDeals = require('mongoose'); c
你找到解决办法了吗?
const mongooseDeals = require('mongoose');
const dotenv = require('dotenv'); dotenv.load();
const googleAuth = require('./gAuth');
mongooseDeals.connect(process.env.MONGODB_URI_DEALS, {
useMongoClient: true,
socketTimeoutMS: 0,
reconnectTries: 30
});
const fs = require('fs');
const readline = require('readline');
const google = require('googleapis');
//Models
const Deals = require('./api/models/deals');
googleAuth.authorize()
.then(readDeals)
.catch((err) => {
console.log('auth error', err);
});
function readDeals(auth) {
var sheets = google.sheets('v4');
sheets.spreadsheets.values.get({
auth: auth,
spreadsheetId: process.env.SPREADSHEET_ID,
range: 'A2:AB',
}, function(err, response) {
if (err) {
console.log('The API returned an error: ' + err);
throw err;
}
var rows = response.values;
console.log(rows.length + ' records loaded');
if (rows.length == 0) {
console.log('No data found.');
throw err;
} else {
for (var i = 0; i < rows.length; i++) {
copyValues(rows[i]);
}
}
});
}
var counter = 0;
function copyValues(fila){
var row = fila;
counter += 1;
var speed = 0.0;
var lat = 0.0;
var lng = 0.0;
var nl_title = '';
var template_id = '';
var deal_id = '';
var country_code = '';
var merchant_name = '';
var cat1 = '';
speed = parseFloat(row[10])* parseFloat(row[19]);
lat = parseFloat(row[24])/10;
lng = parseFloat(row[25])/10;
nl_title = row[5];
template_id = parseInt(row[3]);
deal_id = parseInt(row[4]);
country_code = row[0];
merchant_name = row[6];
cat1 = row[16];
var aux ={
template_id: template_id,
deal_id: deal_id,
country_code: country_code,
position:{
type: "Point",
coordinates: [lng,lat]
},
sellSpeed: speed,
nl_title: nl_title,
merchant_name: merchant_name,
cat1: cat1
};
var query = {
deal_id: deal_id
};
var data = {
$set: aux
};
Deals.findOneAndUpdate(query, data, { upsert: true }, function(err, object) {
if (err) throw err;
counter -= 1;
if (!counter){
console.log('termine!');
}
});
}
const mongooseDeals = require('mongoose');
const GeoJSON = require('mongoose-geojson-schema');
const DealSchema = new mongooseDeals.Schema({
country_code: String,
product_type: String,
city: String,
template_id: Number,
deal_id: {
type: Number,
required: true,
unique: true
},
nl_title: String,
merchant_name: String,
picture: String,
value: Number,
final_price: Number,
start_date: Date,
due_date: Date,
cat_id: Number,
cat3: String,
cat2: String,
cat1: String,
url: String,
stock: Number,
sold: Number,
position: mongooseDeals.Schema.Types.Point,
merchant_id: Number,
country_id: Number,
sellSpeed: Number,
idioma: String
},{
timestamps: true
});
DealSchema.index({ deal_id: 1 });
DealSchema.index({ position: "2dsphere" });
DealSchema.index(
{
merchant_name: "text",
nl_title: "text",
cat1: "text"
},
{
weights: {
merchant_name: 10,
nl_title: 5,
cat1: 1
},
default_language: "spanish",
language_override: "idioma",
name: "TextIndex"
}
);
const Deal = module.exports = mongooseDeals.model('Deal', DealSchema);