Node.js findOneAndUpdate猫鼬与';只有';20k异步更新

Node.js findOneAndUpdate猫鼬与';只有';20k异步更新,node.js,mongodb,mongoose,Node.js,Mongodb,Mongoose,我需要从Google Sheets读取csv文件并更新mongodb数据库。该图纸有20k到25k行(每行约30列)。如果我将工作表的读取限制在10k行以下,我会做得很好,但是如果我超过了10k行,我会开始表现出奇怪的行为,并且需要很长时间才能最终运行所有异步FindOneAndUpdates。下面是我正在使用的代码(我创建了一个回调到console.log的计数器,当它应该完成时): 你找到解决办法了吗? const mongooseDeals = require('mongoose'); c

我需要从Google Sheets读取csv文件并更新mongodb数据库。该图纸有20k到25k行(每行约30列)。如果我将工作表的读取限制在10k行以下,我会做得很好,但是如果我超过了10k行,我会开始表现出奇怪的行为,并且需要很长时间才能最终运行所有异步FindOneAndUpdates。下面是我正在使用的代码(我创建了一个回调到console.log的计数器,当它应该完成时):


你找到解决办法了吗?
const mongooseDeals = require('mongoose');
const dotenv = require('dotenv'); dotenv.load();
const googleAuth = require('./gAuth');

mongooseDeals.connect(process.env.MONGODB_URI_DEALS, {
  useMongoClient: true,
  socketTimeoutMS: 0,
  reconnectTries: 30
});

const fs = require('fs');
const readline = require('readline');
const google = require('googleapis');

//Models
const Deals = require('./api/models/deals');

googleAuth.authorize()
  .then(readDeals)
  .catch((err) => {
    console.log('auth error', err);
  });

function readDeals(auth) {
  var sheets = google.sheets('v4');
  sheets.spreadsheets.values.get({
    auth: auth,
    spreadsheetId: process.env.SPREADSHEET_ID,
    range: 'A2:AB',
  }, function(err, response) {
    if (err) {
      console.log('The API returned an error: ' + err);
      throw err;
    }
    var rows = response.values;
    console.log(rows.length + ' records loaded');
    if (rows.length == 0) {
      console.log('No data found.');
      throw err;
    } else {
      for (var i = 0; i < rows.length; i++) {
        copyValues(rows[i]);
      }
    }
  });
}

var counter = 0;

function copyValues(fila){
  var row = fila;
  counter += 1;

  var speed = 0.0;
  var lat = 0.0;
  var lng = 0.0;
  var nl_title = '';    
  var template_id = '';
  var deal_id = '';
  var country_code = '';
  var merchant_name = '';
  var cat1 = '';

  speed = parseFloat(row[10])* parseFloat(row[19]);
  lat = parseFloat(row[24])/10;
  lng = parseFloat(row[25])/10;
  nl_title = row[5];
  template_id = parseInt(row[3]);
  deal_id = parseInt(row[4]);
  country_code = row[0];
  merchant_name = row[6];
  cat1 = row[16];

  var aux ={
    template_id: template_id,
    deal_id: deal_id,
    country_code: country_code,
    position:{
      type: "Point",
      coordinates: [lng,lat]          
    },
    sellSpeed: speed,
    nl_title: nl_title,
    merchant_name: merchant_name,
    cat1: cat1
  };

  var query = {
    deal_id: deal_id
  };

  var data = {
    $set: aux
  };

  Deals.findOneAndUpdate(query, data, { upsert: true }, function(err, object) {
    if (err) throw err;
    counter -= 1;
    if (!counter){
      console.log('termine!');
    }
  });
}
const mongooseDeals = require('mongoose');
const GeoJSON = require('mongoose-geojson-schema');

const DealSchema = new mongooseDeals.Schema({
  country_code: String,
  product_type: String,
  city: String,
  template_id: Number,
  deal_id: {
        type: Number,
        required: true,
        unique: true
    },
  nl_title: String,
  merchant_name: String,
  picture: String,
  value: Number,
  final_price: Number,
  start_date: Date,
  due_date: Date,
  cat_id: Number,
  cat3: String,
  cat2: String,
  cat1: String,
  url: String,
  stock: Number,
  sold: Number,
  position: mongooseDeals.Schema.Types.Point,
  merchant_id: Number,
  country_id: Number,
  sellSpeed: Number,
  idioma: String
},{
    timestamps: true
});

DealSchema.index({ deal_id: 1 });

DealSchema.index({ position: "2dsphere" });

DealSchema.index(
   {
      merchant_name: "text",
      nl_title: "text",
      cat1: "text"
   },
   {
    weights: {
      merchant_name: 10,
      nl_title: 5,
      cat1: 1
    },
    default_language: "spanish",
    language_override: "idioma",
    name: "TextIndex"
   }
 );

const Deal = module.exports = mongooseDeals.model('Deal', DealSchema);