MongoDB Aggregation Remove docs that have a duplicate field in a collection (dedupe)
Example
Note that the allowDiskUse: true option is optional but will help mitigate out of memory issues as this aggregation can be a memory intensive operation if your collection size is large - so i recommend to always use it.
var duplicates = [];
db.transactions.aggregate([
{ $group: {
_id: { cr_dr: "$cr_dr"},
dups: { "$addToSet": "$_id" },
count: { "$sum": 1 }
}
},
{ $match: {
count: { "$gt": 1 }
}}
],allowDiskUse: true}
)
.result
.forEach(function(doc) {
doc.dups.shift();
doc.dups.forEach( function(dupId){
duplicates.push(dupId);
}
)
})
// printjson(duplicates);
// Remove all duplicates in one go
db.transactions.remove({_id:{$in:duplicates}})
