.populate() -> .batchSize() -> .cursor() populates docs in batch then again one-by-one

[sjlu]

Do you want to request a feature or report a bug?

bug

What is the current behavior?

.populate() -> .batchSize() -> .cursor() currently calls populate with respected batchSize then calls populate for each doc again, one-by-one.

If the current behavior is a bug, please provide the steps to reproduce.

  const cursor = models.Contact
    .find(query)
    .sort({
      last_messaged_at: -1,
      created_at: 1
    })
    .populate('_user')
    .limit(2)
    .batchSize(2)
    .cursor()

    for await (const doc of cursor) {
      console.log(doc)
    }

Mongoose: contacts.find({ _user: { '$in': [ new ObjectId("1"), new ObjectId("2") ] }, email: 'email@gmail.com'}, { sort: { last_messaged_at: -1, created_at: 1 }, limit: 2, batchSize: 2, projection: {}, cursor: { batchSize: 2 }, _populateBatchSize: 2})
Mongoose: users.find({ _id: { '$in': [ new ObjectId("1"), new ObjectId("2") ], [Symbol(mongoose#trustedSymbol)]: true }}, { skip: undefined, limit: undefined, perDocumentLimit: undefined, projection: {}})
Mongoose: users.find({ _id: { '$in': [ new ObjectId("1") ], [Symbol(mongoose#trustedSymbol)]: true }}, { skip: undefined, limit: undefined, perDocumentLimit: undefined, projection: {}})
Mongoose: users.find({ _id: { '$in': [ new ObjectId("2") ], [Symbol(mongoose#trustedSymbol)]: true }}, { skip: undefined, limit: undefined, perDocumentLimit: undefined, projection: {}})

What is the expected behavior?

Should populate 2 and not populate one by one after.

Related issues?
#9365

As far as I can tell pop was dropped in this refactor commit:
e49a1f6

What are the versions of Node.js, Mongoose and MongoDB you are using? Note that "latest" is not a version.

node 16.14.0
mongodb 4.4.12
mongoose 6.2.4

[sjlu]

Further investigation showing me that lib/model.js:L4467 passes in the following options but it doesn't look like it's using those PopulateOptions as cache.

[
  PopulateOptions {
    _docs: {
      '1': new ObjectId("1"),
      '2': new ObjectId("2")
    },
    _childDocs: [ [Object], [Object] ],
    path: '_user',
    _queryProjection: {},
    _localModel: Model { Contact }
  }
]

[IslandRhythms]

I think I am able to reproduce your issue but what should the mongodb calls look like then?

Mongoose: othermodels.insertOne({ citation: 'Stop', _id: new ObjectId("6228d9836f57a9ae4269eec2"), __v: 0}, { session: null })
Mongoose: othermodels.insertOne({ citation: 'Speeding', _id: new ObjectId("6228d9836f57a9ae4269eec4"), __v: 0}, { session: null })
Mongoose: tests.insertOne({ name: 'Test Testerson', citation: new ObjectId("6228d9836f57a9ae4269eec2"), _id: new ObjectId("6228d9836f57a9ae4269eec6"), __v: 0}, { session: null })
Mongoose: tests.insertOne({ name: 'Test Testerson', citation: new ObjectId("6228d9836f57a9ae4269eec4"), _id: new ObjectId("6228d9836f57a9ae4269eec8"), __v: 0}, { session: null })
Mongoose: tests.find({ name: 'Test Testerson' }, { batchSize: 2, projection: {}, cursor: { batchSize: 2 }, _populateBatchSize: 2})
Mongoose: othermodels.find({ _id: { '$in': [ new ObjectId("6228d9836f57a9ae4269eec2"), new ObjectId("6228d9836f57a9ae4269eec4") ], [Symbol(mongoose#trustedSymbol)]: true }}, { skip: undefined, limit: undefined, perDocumentLimit: undefined, projection: {}})
Mongoose: othermodels.find({ _id: { '$in': [ new ObjectId("6228d9836f57a9ae4269eec2") ], [Symbol(mongoose#trustedSymbol)]: true }}, { skip: undefined, limit: undefined, perDocumentLimit: undefined, projection: {}})
Mongoose: othermodels.find({ _id: { '$in': [ new ObjectId("6228d9836f57a9ae4269eec4") ], [Symbol(mongoose#trustedSymbol)]: true }}, { skip: undefined, limit: undefined, perDocumentLimit: undefined, projection: {}})

edit: I see it now, the first in operation has both object ids and then it follows up with two more, one each

[IslandRhythms]

const mongoose = require('mongoose');
mongoose.set('debug', true);

const testSchema = new mongoose.Schema({
    name: String,
    citation: {
        type: mongoose.Types.ObjectId,
        ref: 'OtherModel'
    }
});

const otherSchema = new mongoose.Schema({
    citation: String
});

const Test = mongoose.model('Test', testSchema);

const otherModel = mongoose.model('OtherModel', otherSchema);

async function run() {
await mongoose.connect('mongodb://localhost:27017');
await mongoose.connection.dropDatabase();

const entry = await otherModel.create({
    citation: 'Stop'
});

const otherEntry = await otherModel.create({
    citation: 'Speeding'
});

const test = await Test.create({
    name: 'Test Testerson',
    citation: entry._id
})

const otherTest = await Test.create({
    name: 'Test Testerson',
    citation: otherEntry._id
});

const cursor = Test.find({name: 'Test Testerson'}).populate('citation').batchSize(2).cursor();

for await (const doc of cursor) {
    // console.log('The doc', doc);
}

}

run();

[Uzlopak]

I think this means also that the typings are not correct. Like #11503