Fatal编程技术网
  • mongodb/
  • MongoDB：将多个集合中的数据合并到一个集合中..如何？

MongoDB：将多个集合中的数据合并到一个集合中..如何？

mongodb

MongoDB：将多个集合中的数据合并到一个集合中..如何？,mongodb,mongodb-query,aggregation-framework,Mongodb,Mongodb Query,Aggregation Framework,如何（在MongoDB中）将多个集合中的数据合并到一个集合中 我可以使用map reduce吗？如果可以，如何使用 我是一个新手，如果能举个例子，我将不胜感激 您必须在应用程序层中这样做。如果您使用的是ORM，它可以使用注释（或类似的东西）来提取其他集合中存在的引用。我只使用过，注释在查询时会获取引用的实体，因此我可以避免自己在代码中这样做。虽然您不能实时执行此操作，但您可以多次运行map reduce，通过使用MongoDB 1.8+map/reduce中的“reduce”out选项将数据合并

如何（在MongoDB中）将多个集合中的数据合并到一个集合中

我可以使用map reduce吗？如果可以，如何使用

我是一个新手，如果能举个例子，我将不胜感激

您必须在应用程序层中这样做。如果您使用的是ORM，它可以使用注释（或类似的东西）来提取其他集合中存在的引用。我只使用过，注释在查询时会获取引用的实体，因此我可以避免自己在代码中这样做。

虽然您不能实时执行此操作，但您可以多次运行map reduce，通过使用MongoDB 1.8+map/reduce中的“reduce”out选项将数据合并在一起（请参阅）。两个集合中都需要有一些密钥，可以用作_id

例如，假设您有一个
users
集合和一个
comments
集合，并且您希望有一个新集合，其中包含每个评论的一些用户统计信息

假设
users
集合具有以下字段：

  • _身份证
  • 名字
  • 姓氏
  • 国家
  • 性别
  • 年龄
然后，
comments
集合具有以下字段：

  • _身份证
  • 用户ID
  • 评论
  • 创造
您将执行以下映射/减少操作：

var-mapUsers、mapcoments、reduce；
db.users_comments.remove（）；
//设置示例数据-不会在生产中实际使用此数据
db.users.remove（）；
db.comments.remove（）；
db.users.save（{姓氏：“Rich”，姓氏：“S”，性别：“M”，国家：“CA”，年龄：“18”}）；
db.users.save（{名字：“Rob”，姓氏：“M”，性别：“M”，国家：“美国”，年龄：“25”}）；
db.users.save（{姓：“莎拉”，姓：“T”，性别：“F”，国家：“美国”，年龄：“13”}）；
var users=db.users.find（）；
save（{userId:users[0]。\u id，“comment”：“嘿，怎么了？”，创建：new ISODate（）}）；
save（{userId:users[1]。\u id，“comment”：“Not much”，created:new ISODate（）}）；
save（{userId:users[0]。\u id，“comment”：“Cool”，created:new ISODate（）}）；
//结束样本数据设置
mapUsers=函数（）{
var值={
国家：这个国家，
性别：这个，性别，
年龄：这个年龄
};
发射（此._id，值）；
};
mapComments=函数（）{
var值={
commentId:这个，
注释：this.comment，
创建的：这个。创建的
};
emit（this.userId，值）；
};
reduce=函数（k，值）{
var result={}，commentFields={
“commentId”：“”，
“注释”：“”，
“已创建”：”
};
values.forEach（函数（值）{
var场；
如果（值中的“注释”）{
如果（！（“结果中的注释”）{
result.comments=[]；
}
结果.注释.推送（值）；
}else if（值中的“注释”）{
如果（！（“结果中的注释”）{
result.comments=[]；
}
result.comments.push.apply（result.comments，value.comments）；
}
用于（值中的字段）{
if（value.hasOwnProperty（字段）和&！（commentFields中的字段））{
结果[字段]=值[字段]；
}
}
});
返回结果；
};
mapReduce（mapUsers，reduce，{“out”：{“reduce”：“users_comments”}）；
mapReduce（mapComments，reduce，{“out”：{“reduce”：“users_comments”}}）；
db.users_comments.find（）.pretty（）；//查看生成的集合
此时，您将拥有一个名为
users\u comments
的新集合，其中包含合并的数据，您现在可以使用它了。这些缩减的集合都有
\u id
，这是您在映射函数中发出的键，然后所有值都是
值
键内的子对象-这些值不在这些缩减文档的顶层

这是一个有点简单的例子。您可以对更多集合重复此操作，只要您想继续构建减少的集合即可。您还可以在此过程中对数据进行汇总和聚合。由于聚合和保存现有字段的逻辑变得越来越复杂，您可能会定义多个reduce函数

您还将注意到，现在每个用户都有一个文档，其中包含该用户在一个数组中的所有注释。如果我们要合并具有一对一关系而不是一对多关系的数据，它将是平面的，您可以简单地使用如下reduce函数：

{
    Collection1: [...],
    Collection2: [...],
    Collection3: [...]
}

reduce=函数（k，值）{
var result={}；
values.forEach（函数（值）{
var场；
用于（值中的字段）{
if（value.hasOwnProperty（字段））{
结果[字段]=值[字段]；
}
}
});
返回结果；
};
如果要展平
users\u comments
集合，使其成为每个评论一个文档，请另外运行以下命令：

var映射，减少；
map=函数（）{
变量调试=函数（值）{
var场；
用于（值中的字段）{
打印（字段+”：“+值[字段]）；
}
};
调试（这个）；
var=这个；
如果（此.value中的“注释”）{
this.value.comments.forEach（函数（值）{
发出（value.commentId{
userId:that.\u id，
国家：那。价值。国家，
年龄：那。价值。年龄，
评论：value.comment，
创建：value.created，
});
});
}
};
reduce=函数（k，值）{
var result={}；
values.forEach（函数（值）{
var场；
用于（值中的字段）{
if（value.hasOwnProperty（字段））{
结果[字段]=值[字段]；
}
}
});
返回结果；
};
db.users\u comments.mapReduce（map，reduce，{“out”：“comments\u with\u demographics”}）；
这项技术绝对不应该在f 
db.small_collection.find().forEach(function(obj){ 
   db.big_collection.insert(obj)
});

function shangMergeCol() {
  tcol= db.getCollection(arguments[0]);
  for (var i=1; i<arguments.length; i++){
    scol= db.getCollection(arguments[i]);
    scol.find().forEach(
        function (d) {
            tcol.insert(d);
        }
    )
  }
}

{
    "isbn": "978-3-16-148410-0",
    "title": "Some cool book",
    "author": "John Doe"
}
{
    "isbn": "978-3-16-148999-9",
    "title": "Another awesome book",
    "author": "Jane Roe"
}

{
    "_id": ObjectId("56e31bcf76cdf52e541d9d26"),
    "isbn": "978-3-16-148410-0",
    "copies_sold": 12500
}
{
    "_id": ObjectId("56e31ce076cdf52e541d9d28"),
    "isbn": "978-3-16-148999-9",
    "copies_sold": 720050
}
{
    "_id": ObjectId("56e31ce076cdf52e541d9d29"),
    "isbn": "978-3-16-148999-9",
    "copies_sold": 1000
}

db.books.aggregate([{
    $lookup: {
            from: "books_selling_data",
            localField: "isbn",
            foreignField: "isbn",
            as: "copies_sold"
        }
}])

{
    "isbn": "978-3-16-148410-0",
    "title": "Some cool book",
    "author": "John Doe",
    "copies_sold": [
        {
            "_id": ObjectId("56e31bcf76cdf52e541d9d26"),
            "isbn": "978-3-16-148410-0",
            "copies_sold": 12500
        }
    ]
}
{
    "isbn": "978-3-16-148999-9",
    "title": "Another awesome book",
    "author": "Jane Roe",
    "copies_sold": [
        {
            "_id": ObjectId("56e31ce076cdf52e541d9d28"),
            "isbn": "978-3-16-148999-9",
            "copies_sold": 720050
        },
        {
            "_id": ObjectId("56e31ce076cdf52e541d9d28"),
            "isbn": "978-3-16-148999-9",
            "copies_sold": 1000
        }
    ]
}

db.getCollection('users').aggregate([
    {
        $lookup: {
            from: "userinfo",
            localField: "userId",
            foreignField: "userId",
            as: "userInfoData"
        }
    },
    {
        $lookup: {
            from: "userrole",
            localField: "userId",
            foreignField: "userId",
            as: "userRoleData"
        }
    },
    { $unwind: { path: "$userInfoData", preserveNullAndEmptyArrays: true }},
    { $unwind: { path: "$userRoleData", preserveNullAndEmptyArrays: true }}
])

 { $unwind: { path: "$userInfoData", preserveNullAndEmptyArrays: true }}, 
 { $unwind: { path: "$userRoleData", preserveNullAndEmptyArrays: true }}

{ $unwind:"$userRoleData"} 
{ $unwind:"$userRoleData"}

db.getCollection('servicelocations').aggregate([
  {
    $match: {
      serviceLocationId: {
        $in: ["36728"]
      }
    }
  },
  {
    $lookup: {
      from: "orders",
      localField: "serviceLocationId",
      foreignField: "serviceLocationId",
      as: "orders"
    }
  },
  {
    $lookup: {
      from: "timewindowtypes",
      localField: "timeWindow.timeWindowTypeId",
      foreignField: "timeWindowTypeId",
      as: "timeWindow"
    }
  },
  {
    $lookup: {
      from: "servicetimetypes",
      localField: "serviceTimeTypeId",
      foreignField: "serviceTimeTypeId",
      as: "serviceTime"
    }
  },
  {
    $unwind: "$orders"
  },
  {
    $unwind: "$serviceTime"
  },
  {
    $limit: 14
  }
])

{
    "_id" : ObjectId("59c3ac4bb7799c90ebb3279b"),
    "serviceLocationId" : "36728",
    "regionId" : 1.0,
    "zoneId" : "DXBZONE1",
    "description" : "AL HALLAB REST EMIRATES MALL",
    "locationPriority" : 1.0,
    "accountTypeId" : 1.0,
    "locationType" : "SERVICELOCATION",
    "location" : {
        "makani" : "",
        "lat" : 25.119035,
        "lng" : 55.198694
    },
    "deliveryDays" : "MTWRFSU",
    "timeWindow" : [ 
        {
            "_id" : ObjectId("59c3b0a3b7799c90ebb32cde"),
            "timeWindowTypeId" : "1",
            "Description" : "MORNING",
            "timeWindow" : {
                "openTime" : "06:00",
                "closeTime" : "08:00"
            },
            "accountId" : 1.0
        }, 
        {
            "_id" : ObjectId("59c3b0a3b7799c90ebb32cdf"),
            "timeWindowTypeId" : "1",
            "Description" : "MORNING",
            "timeWindow" : {
                "openTime" : "09:00",
                "closeTime" : "10:00"
            },
            "accountId" : 1.0
        }, 
        {
            "_id" : ObjectId("59c3b0a3b7799c90ebb32ce0"),
            "timeWindowTypeId" : "1",
            "Description" : "MORNING",
            "timeWindow" : {
                "openTime" : "10:30",
                "closeTime" : "11:30"
            },
            "accountId" : 1.0
        }
    ],
    "address1" : "",
    "address2" : "",
    "phone" : "",
    "city" : "",
    "county" : "",
    "state" : "",
    "country" : "",
    "zipcode" : "",
    "imageUrl" : "",
    "contact" : {
        "name" : "",
        "email" : ""
    },
    "status" : "ACTIVE",
    "createdBy" : "",
    "updatedBy" : "",
    "updateDate" : "",
    "accountId" : 1.0,
    "serviceTimeTypeId" : "1",
    "orders" : [ 
        {
            "_id" : ObjectId("59c3b291f251c77f15790f92"),
            "orderId" : "AQ18O1704264",
            "serviceLocationId" : "36728",
            "orderNo" : "AQ18O1704264",
            "orderDate" : "18-Sep-17",
            "description" : "AQ18O1704264",
            "serviceType" : "Delivery",
            "orderSource" : "Import",
            "takenBy" : "KARIM",
            "plannedDeliveryDate" : ISODate("2017-08-26T00:00:00.000Z"),
            "plannedDeliveryTime" : "",
            "actualDeliveryDate" : "",
            "actualDeliveryTime" : "",
            "deliveredBy" : "",
            "size1" : 296.0,
            "size2" : 3573.355,
            "size3" : 240.811,
            "jobPriority" : 1.0,
            "cancelReason" : "",
            "cancelDate" : "",
            "cancelBy" : "",
            "reasonCode" : "",
            "reasonText" : "",
            "status" : "",
            "lineItems" : [ 
                {
                    "ItemId" : "BNWB020",
                    "size1" : 15.0,
                    "size2" : 78.6,
                    "size3" : 6.0
                }, 
                {
                    "ItemId" : "BNWB021",
                    "size1" : 20.0,
                    "size2" : 252.0,
                    "size3" : 11.538
                }, 
                {
                    "ItemId" : "BNWB023",
                    "size1" : 15.0,
                    "size2" : 285.0,
                    "size3" : 16.071
                }, 
                {
                    "ItemId" : "CPMW112",
                    "size1" : 3.0,
                    "size2" : 25.38,
                    "size3" : 1.731
                }, 
                {
                    "ItemId" : "MMGW001",
                    "size1" : 25.0,
                    "size2" : 464.375,
                    "size3" : 46.875
                }, 
                {
                    "ItemId" : "MMNB218",
                    "size1" : 50.0,
                    "size2" : 920.0,
                    "size3" : 60.0
                }, 
                {
                    "ItemId" : "MMNB219",
                    "size1" : 50.0,
                    "size2" : 630.0,
                    "size3" : 40.0
                }, 
                {
                    "ItemId" : "MMNB220",
                    "size1" : 50.0,
                    "size2" : 416.0,
                    "size3" : 28.846
                }, 
                {
                    "ItemId" : "MMNB270",
                    "size1" : 50.0,
                    "size2" : 262.0,
                    "size3" : 20.0
                }, 
                {
                    "ItemId" : "MMNB302",
                    "size1" : 15.0,
                    "size2" : 195.0,
                    "size3" : 6.0
                }, 
                {
                    "ItemId" : "MMNB373",
                    "size1" : 3.0,
                    "size2" : 45.0,
                    "size3" : 3.75
                }
            ],
            "accountId" : 1.0
        }, 
        {
            "_id" : ObjectId("59c3b291f251c77f15790f9d"),
            "orderId" : "AQ137O1701240",
            "serviceLocationId" : "36728",
            "orderNo" : "AQ137O1701240",
            "orderDate" : "18-Sep-17",
            "description" : "AQ137O1701240",
            "serviceType" : "Delivery",
            "orderSource" : "Import",
            "takenBy" : "KARIM",
            "plannedDeliveryDate" : ISODate("2017-08-26T00:00:00.000Z"),
            "plannedDeliveryTime" : "",
            "actualDeliveryDate" : "",
            "actualDeliveryTime" : "",
            "deliveredBy" : "",
            "size1" : 28.0,
            "size2" : 520.11,
            "size3" : 52.5,
            "jobPriority" : 1.0,
            "cancelReason" : "",
            "cancelDate" : "",
            "cancelBy" : "",
            "reasonCode" : "",
            "reasonText" : "",
            "status" : "",
            "lineItems" : [ 
                {
                    "ItemId" : "MMGW001",
                    "size1" : 25.0,
                    "size2" : 464.38,
                    "size3" : 46.875
                }, 
                {
                    "ItemId" : "MMGW001-F1",
                    "size1" : 3.0,
                    "size2" : 55.73,
                    "size3" : 5.625
                }
            ],
            "accountId" : 1.0
        }, 
        {
            "_id" : ObjectId("59c3b291f251c77f15790fd8"),
            "orderId" : "AQ110O1705036",
            "serviceLocationId" : "36728",
            "orderNo" : "AQ110O1705036",
            "orderDate" : "18-Sep-17",
            "description" : "AQ110O1705036",
            "serviceType" : "Delivery",
            "orderSource" : "Import",
            "takenBy" : "KARIM",
            "plannedDeliveryDate" : ISODate("2017-08-26T00:00:00.000Z"),
            "plannedDeliveryTime" : "",
            "actualDeliveryDate" : "",
            "actualDeliveryTime" : "",
            "deliveredBy" : "",
            "size1" : 60.0,
            "size2" : 1046.0,
            "size3" : 68.0,
            "jobPriority" : 1.0,
            "cancelReason" : "",
            "cancelDate" : "",
            "cancelBy" : "",
            "reasonCode" : "",
            "reasonText" : "",
            "status" : "",
            "lineItems" : [ 
                {
                    "ItemId" : "MMNB218",
                    "size1" : 50.0,
                    "size2" : 920.0,
                    "size3" : 60.0
                }, 
                {
                    "ItemId" : "MMNB219",
                    "size1" : 10.0,
                    "size2" : 126.0,
                    "size3" : 8.0
                }
            ],
            "accountId" : 1.0
        }
    ],
    "serviceTime" : {
        "_id" : ObjectId("59c3b07cb7799c90ebb32cdc"),
        "serviceTimeTypeId" : "1",
        "serviceTimeType" : "nohelper",
        "description" : "",
        "fixedTime" : 30.0,
        "variableTime" : 0.0,
        "accountId" : 1.0
    }
}

// Create employees data for testing the union.
db.getCollection('employees').insert({ name: "John", type: "employee", department: "sales" });
db.getCollection('employees').insert({ name: "Martha", type: "employee", department: "accounting" });
db.getCollection('employees').insert({ name: "Amy", type: "employee", department: "warehouse" });
db.getCollection('employees').insert({ name: "Mike", type: "employee", department: "warehouse"  });

// Create freelancers data for testing the union.
db.getCollection('freelancers').insert({ name: "Stephany", type: "freelancer", department: "accounting" });
db.getCollection('freelancers').insert({ name: "Martin", type: "freelancer", department: "sales" });
db.getCollection('freelancers').insert({ name: "Doug", type: "freelancer", department: "warehouse"  });
db.getCollection('freelancers').insert({ name: "Brenda", type: "freelancer", department: "sales"  });

// Here we do a union of the employees and freelancers using a single aggregation query.
db.getCollection('freelancers').aggregate( // 1. Use any collection containing at least one document.
  [
    { $limit: 1 }, // 2. Keep only one document of the collection.
    { $project: { _id: '$$REMOVE' } }, // 3. Remove everything from the document.

    // 4. Lookup collections to union together.
    { $lookup: { from: 'employees', pipeline: [{ $match: { department: 'sales' } }], as: 'employees' } },
    { $lookup: { from: 'freelancers', pipeline: [{ $match: { department: 'sales' } }], as: 'freelancers' } },

    // 5. Union the collections together with a projection.
    { $project: { union: { $concatArrays: ["$employees", "$freelancers"] } } },

    // 6. Unwind and replace root so you end up with a result set.
    { $unwind: '$union' },
    { $replaceRoot: { newRoot: '$union' } }
  ]);

{ $project: { _id: '$$REMOVE' } }

{ $lookup: { from: 'collectionToUnion1', pipeline: [...], as: 'Collection1' } },
{ $lookup: { from: 'collectionToUnion2', pipeline: [...], as: 'Collection2' } },
{ $lookup: { from: 'collectionToUnion3', pipeline: [...], as: 'Collection3' } }

{
    Collection1: [...],
    Collection2: [...],
    Collection3: [...]
}

{
  "$project" :
  {
    "Union" : { $concatArrays: ["$Collection1", "$Collection2", "$Collection3"] }
  }
}

{ $unwind: "$Union" },
{ $replaceRoot: { newRoot: "$Union" } }

// > db.users.find()
//   [{ user: 1, name: "x" }, { user: 2, name: "y" }]
// > db.books.find()
//   [{ user: 1, book: "a" }, { user: 1, book: "b" }, { user: 2, book: "c" }]
// > db.movies.find()
//   [{ user: 1, movie: "g" }, { user: 2, movie: "h" }, { user: 2, movie: "i" }]
db.users.aggregate([
  { $unionWith: "books"  },
  { $unionWith: "movies" },
  { $group: {
    _id: "$user",
    user: {
      $accumulator: {
        accumulateArgs: ["$name", "$book", "$movie"],
        init: function() { return { books: [], movies: [] } },
        accumulate: function(user, name, book, movie) {
          if (name) user.name = name;
          if (book) user.books.push(book);
          if (movie) user.movies.push(movie);
          return user;
        },
        merge: function(userV1, userV2) {
          if (userV2.name) userV1.name = userV2.name;
          userV1.books.concat(userV2.books);
          userV1.movies.concat(userV2.movies);
          return userV1;
        },
        lang: "js"
      }
    }
  }}
])
// { _id: 1, user: { books: ["a", "b"], movies: ["g"], name: "x" } }
// { _id: 2, user: { books: ["c"], movies: ["h", "i"], name: "y" } }