Node.js 使用批处理递归更新文档只适用于小集合
我有一个团队,其中包含大约80000份文档。每周一我都会使用firebase云功能重置每个团队的分数。这是我的职责:Node.js 使用批处理递归更新文档只适用于小集合,node.js,firebase,google-cloud-functions,Node.js,Firebase,Google Cloud Functions,我有一个团队,其中包含大约80000份文档。每周一我都会使用firebase云功能重置每个团队的分数。这是我的职责: exports.resetOrgScore = functions.runWith(runtimeOpts).pubsub.schedule("every monday 00:00").timeZone("Europe/Oslo").onRun(async (context) => { let batch = admin.fires
exports.resetOrgScore = functions.runWith(runtimeOpts).pubsub.schedule("every monday 00:00").timeZone("Europe/Oslo").onRun(async (context) => {
let batch = admin.firestore().batch();
let count = 0;
let overallCount = 0;
const orgDocs = await admin.firestore().collection("teams").get();
orgDocs.forEach(async(doc) => {
batch.update(doc.ref, {score:0.0});
if (++count >= 500 || ++overallCount >= orgDocs.docs.length) {
await batch.commit();
batch = admin.firestore().batch();
count = 0;
}
});
}))
我尝试在10个文档的较小集合中运行该函数,它工作正常,但在“teams”集合中运行该函数时,它返回“无法修改已提交的写回”。我试着像这样返回承诺(下面的代码),但这并不能解决问题。提前感谢:)
代码中有三个问题:
async/await
与forEach()
一起使用:问题是传递给forEach()
的回调未被等待,请参阅更多解释或说明wait batch.commit();batch=admin.firestore().batch()代码>这正是你正在做的
dateUpdated
标志为每个新批次选择文档:使用原始代码,文档被删除,因此不需要标志
const runtimeOpts = {
timeoutSeconds: 540,
memory: '1GB',
};
exports.resetOrgScore = functions
.runWith(runtimeOpts)
.pubsub
.schedule("every monday 00:00")
.timeZone("Europe/Oslo")
.onRun((context) => {
return new Promise((resolve, reject) => {
deleteQueryBatch(resolve).catch(reject);
});
});
async function deleteQueryBatch(resolve) {
const db = admin.firestore();
const snapshot = await db
.collection('teams')
.where('dateUpdated', '==', "20210302")
.orderBy('__name__')
.limit(499)
.get();
const batchSize = snapshot.size;
if (batchSize === 0) {
// When there are no documents left, we are done
resolve();
return;
}
// Delete documents in a batch
const batch = db.batch();
snapshot.docs.forEach((doc) => {
batch.update(doc.ref, { score:0.0, dateUpdated: "20210303" });
});
await batch.commit();
// Recurse on the next process tick, to avoid
// exploding the stack.
process.nextTick(() => {
deleteQueryBatch(resolve);
});
}
请注意,上述云功能用于超时,即9分钟
如果您的所有文档似乎无法在9分钟内更新,您需要找到另一种方法,例如使用某台服务器上的Admin SDK,或者将工作分成几部分并运行CF数次。如果您更改为
++count>=250
,这有区别吗?@FrankvanPuffelen谢谢你的建议,但我还是犯了同样的错误
const runtimeOpts = {
timeoutSeconds: 540,
memory: '1GB',
};
exports.resetOrgScore = functions
.runWith(runtimeOpts)
.pubsub
.schedule("every monday 00:00")
.timeZone("Europe/Oslo")
.onRun((context) => {
return new Promise((resolve, reject) => {
deleteQueryBatch(resolve).catch(reject);
});
});
async function deleteQueryBatch(resolve) {
const db = admin.firestore();
const snapshot = await db
.collection('teams')
.where('dateUpdated', '==', "20210302")
.orderBy('__name__')
.limit(499)
.get();
const batchSize = snapshot.size;
if (batchSize === 0) {
// When there are no documents left, we are done
resolve();
return;
}
// Delete documents in a batch
const batch = db.batch();
snapshot.docs.forEach((doc) => {
batch.update(doc.ref, { score:0.0, dateUpdated: "20210303" });
});
await batch.commit();
// Recurse on the next process tick, to avoid
// exploding the stack.
process.nextTick(() => {
deleteQueryBatch(resolve);
});
}