MongoDB Aggregation Remove docs that have a duplicate field in a collection (dedupe)


Example

Note that the allowDiskUse: true option is optional but will help mitigate out of memory issues as this aggregation can be a memory intensive operation if your collection size is large - so i recommend to always use it.

var duplicates = [];

db.transactions.aggregate([
 { $group: { 
   _id: { cr_dr: "$cr_dr"},
   dups: { "$addToSet": "$_id" }, 
   count: { "$sum": 1 } 
 }
}, 
{ $match: { 
  count: { "$gt": 1 }   
}}
],allowDiskUse: true}
)              
.result          
.forEach(function(doc) {
  doc.dups.shift();   
  doc.dups.forEach( function(dupId){ 
    duplicates.push(dupId); 
  }
)    
})
// printjson(duplicates);     

// Remove all duplicates in one go    
db.transactions.remove({_id:{$in:duplicates}})