Merge pull request #14915 from Automattic/8.7

8.7

docs/populate.md

@@ -119,6 +119,33 @@ story.author = author;
 console.log(story.author.name); // prints "Ian Fleming"
 ```
 
+You can also push documents or POJOs onto a populated array, and Mongoose will add those documents if their `ref` matches.
+
+```javascript
+const fan1 = await Person.create({ name: 'Sean' });
+await Story.updateOne({ title: 'Casino Royale' }, { $push: { fans: { $each: [fan1._id] } } });
+
+const story = await Story.findOne({ title: 'Casino Royale' }).populate('fans');
+story.fans[0].name; // 'Sean'
+
+const fan2 = await Person.create({ name: 'George' });
+story.fans.push(fan2);
+story.fans[1].name; // 'George'
+
+story.fans.push({ name: 'Roger' });
+story.fans[2].name; // 'Roger'
+```
+
+If you push a non-POJO and non-document value, like an ObjectId, Mongoose `>= 8.7.0` will depopulate the entire array.
+
+```javascript
+const fan4 = await Person.create({ name: 'Timothy' });
+story.fans.push(fan4._id); // Push the `_id`, not the full document
+
+story.fans[0].name; // undefined, `fans[0]` is now an ObjectId
+story.fans[0].toString() === fan1._id.toString(); // true
+```
+
 ## Checking Whether a Field is Populated {#checking-populated}
 
 You can call the `populated()` function to check whether a field is populated.

lib/connection.js

@@ -106,6 +106,15 @@ Object.setPrototypeOf(Connection.prototype, EventEmitter.prototype);
 
 Object.defineProperty(Connection.prototype, 'readyState', {
   get: function() {
+    // If connection thinks it is connected, but we haven't received a heartbeat in 2 heartbeat intervals,
+    // that likely means the connection is stale (potentially due to frozen AWS Lambda container)
+    if (
+      this._readyState === STATES.connected &&
+      this._lastHeartbeatAt != null &&
+      typeof this.client?.topology?.s?.description?.heartbeatFrequencyMS === 'number' &&
+      Date.now() - this._lastHeartbeatAt >= this.client.topology.s.description.heartbeatFrequencyMS * 2) {
+      return STATES.disconnected;
+    }
     return this._readyState;
   },
   set: function(val) {

lib/drivers/node-mongodb-native/connection.js

@@ -23,6 +23,11 @@ const utils = require('../../utils');
 function NativeConnection() {
   MongooseConnection.apply(this, arguments);
   this._listening = false;
+  // Tracks the last time (as unix timestamp) the connection received a
+  // serverHeartbeatSucceeded or serverHeartbeatFailed event from the underlying MongoClient.
+  // If we haven't received one in a while (like due to a frozen AWS Lambda container) then
+  // `readyState` is likely stale.
+  this._lastHeartbeatAt = null;
 }
 
 /**
@@ -106,6 +111,7 @@ NativeConnection.prototype.useDb = function(name, options) {
       _opts.noListener = options.noListener;
     }
     newConn.db = _this.client.db(name, _opts);
+    newConn._lastHeartbeatAt = _this._lastHeartbeatAt;
     newConn.onOpen();
   }
 
@@ -409,6 +415,9 @@ function _setClient(conn, client, options, dbName) {
       }
     });
   }
+  client.on('serverHeartbeatSucceeded', () => {
+    conn._lastHeartbeatAt = Date.now();
+  });
 
   if (options.monitorCommands) {
     client.on('commandStarted', (data) => conn.emit('commandStarted', data));
@@ -417,6 +426,9 @@ function _setClient(conn, client, options, dbName) {
   }
 
   conn.onOpen();
+  if (client.topology?.s?.state === 'connected') {
+    conn._lastHeartbeatAt = Date.now();
+  }
 
   for (const i in conn.collections) {
     if (utils.object.hasOwnProperty(conn.collections, i)) {

lib/error/bulkSaveIncompleteError.js

@@ -0,0 +1,44 @@
+/*!
+ * Module dependencies.
+ */
+
+'use strict';
+
+const MongooseError = require('./mongooseError');
+
+
+/**
+ * If the underwriting `bulkWrite()` for `bulkSave()` succeeded, but wasn't able to update or
+ * insert all documents, we throw this error.
+ *
+ * @api private
+ */
+
+class MongooseBulkSaveIncompleteError extends MongooseError {
+  constructor(modelName, documents, bulkWriteResult) {
+    const matchedCount = bulkWriteResult?.matchedCount ?? 0;
+    const insertedCount = bulkWriteResult?.insertedCount ?? 0;
+    let preview = documents.map(doc => doc._id).join(', ');
+    if (preview.length > 100) {
+      preview = preview.slice(0, 100) + '...';
+    }
+
+    const numDocumentsNotUpdated = documents.length - matchedCount - insertedCount;
+    super(`${modelName}.bulkSave() was not able to update ${numDocumentsNotUpdated} of the given documents due to incorrect version or optimistic concurrency, document ids: ${preview}`);
+
+    this.modelName = modelName;
+    this.documents = documents;
+    this.bulkWriteResult = bulkWriteResult;
+    this.numDocumentsNotUpdated = numDocumentsNotUpdated;
+  }
+}
+
+Object.defineProperty(MongooseBulkSaveIncompleteError.prototype, 'name', {
+  value: 'MongooseBulkSaveIncompleteError'
+});
+
+/*!
+ * exports
+ */
+
+module.exports = MongooseBulkSaveIncompleteError;

lib/helpers/document/applyVirtuals.js

@@ -0,0 +1,146 @@
+'use strict';
+
+const mpath = require('mpath');
+
+module.exports = applyVirtuals;
+
+/**
+ * Apply a given schema's virtuals to a given POJO
+ *
+ * @param {Schema} schema
+ * @param {Object} obj
+ * @param {Array<string>} [virtuals] optional whitelist of virtuals to apply
+ * @returns
+ */
+
+function applyVirtuals(schema, obj, virtuals) {
+  if (obj == null) {
+    return obj;
+  }
+
+  let virtualsForChildren = virtuals;
+  let toApply = null;
+
+  if (Array.isArray(virtuals)) {
+    virtualsForChildren = [];
+    toApply = [];
+    for (const virtual of virtuals) {
+      if (virtual.length === 1) {
+        toApply.push(virtual[0]);
+      } else {
+        virtualsForChildren.push(virtual);
+      }
+    }
+  }
+
+  applyVirtualsToChildren(schema, obj, virtualsForChildren);
+  return applyVirtualsToDoc(schema, obj, toApply);
+}
+
+/**
+ * Apply virtuals to any subdocuments
+ *
+ * @param {Schema} schema subdocument schema
+ * @param {Object} res subdocument
+ * @param {Array<String>} [virtuals] optional whitelist of virtuals to apply
+ */
+
+function applyVirtualsToChildren(schema, res, virtuals) {
+  let attachedVirtuals = false;
+  for (const childSchema of schema.childSchemas) {
+    const _path = childSchema.model.path;
+    const _schema = childSchema.schema;
+    if (!_path) {
+      continue;
+    }
+    const _obj = mpath.get(_path, res);
+    if (_obj == null || (Array.isArray(_obj) && _obj.flat(Infinity).length === 0)) {
+      continue;
+    }
+
+    let virtualsForChild = null;
+    if (Array.isArray(virtuals)) {
+      virtualsForChild = [];
+      for (const virtual of virtuals) {
+        if (virtual[0] == _path) {
+          virtualsForChild.push(virtual.slice(1));
+        }
+      }
+
+      if (virtualsForChild.length === 0) {
+        continue;
+      }
+    }
+
+    applyVirtuals(_schema, _obj, virtualsForChild);
+    attachedVirtuals = true;
+  }
+
+  if (virtuals && virtuals.length && !attachedVirtuals) {
+    applyVirtualsToDoc(schema, res, virtuals);
+  }
+}
+
+/**
+ * Apply virtuals to a given document. Does not apply virtuals to subdocuments: use `applyVirtualsToChildren` instead
+ *
+ * @param {Schema} schema
+ * @param {Object} doc
+ * @param {Array<String>} [virtuals] optional whitelist of virtuals to apply
+ * @returns
+ */
+
+function applyVirtualsToDoc(schema, obj, virtuals) {
+  if (obj == null || typeof obj !== 'object') {
+    return;
+  }
+  if (Array.isArray(obj)) {
+    for (const el of obj) {
+      applyVirtualsToDoc(schema, el, virtuals);
+    }
+    return;
+  }
+
+  if (schema.discriminators && Object.keys(schema.discriminators).length > 0) {
+    for (const discriminatorKey of Object.keys(schema.discriminators)) {
+      const discriminator = schema.discriminators[discriminatorKey];
+      const key = discriminator.discriminatorMapping.key;
+      const value = discriminator.discriminatorMapping.value;
+      if (obj[key] == value) {
+        schema = discriminator;
+        break;
+      }
+    }
+  }
+
+  if (virtuals == null) {
+    virtuals = Object.keys(schema.virtuals);
+  }
+  for (const virtual of virtuals) {
+    if (schema.virtuals[virtual] == null) {
+      continue;
+    }
+    const virtualType = schema.virtuals[virtual];
+    const sp = Array.isArray(virtual)
+      ? virtual
+      : virtual.indexOf('.') === -1
+        ? [virtual]
+        : virtual.split('.');
+    let cur = obj;
+    for (let i = 0; i < sp.length - 1; ++i) {
+      cur[sp[i]] = sp[i] in cur ? cur[sp[i]] : {};
+      cur = cur[sp[i]];
+    }
+    let val = virtualType.applyGetters(cur[sp[sp.length - 1]], obj);
+    const isPopulateVirtual =
+      virtualType.options && (virtualType.options.ref || virtualType.options.refPath);
+    if (isPopulateVirtual && val === undefined) {
+      if (virtualType.options.justOne) {
+        val = null;
+      } else {
+        val = [];
+      }
+    }
+    cur[sp[sp.length - 1]] = val;
+  }
+}

lib/helpers/query/castUpdate.js

@@ -2,6 +2,7 @@
 
 const CastError = require('../../error/cast');
 const MongooseError = require('../../error/mongooseError');
+const SchemaString = require('../../schema/string');
 const StrictModeError = require('../../error/strict');
 const ValidationError = require('../../error/validation');
 const castNumber = require('../../cast/number');
@@ -307,6 +308,20 @@ function walkUpdatePath(schema, obj, op, options, context, filter, pref) {
           continue;
         }
 
+        hasKeys = true;
+      } else if (op === '$rename') {
+        const schematype = new SchemaString(`${prefix}${key}.$rename`);
+        try {
+          obj[key] = castUpdateVal(schematype, val, op, key, context, prefix + key);
+        } catch (error) {
+          aggregatedError = _appendError(error, context, key, aggregatedError);
+        }
+
+        if (obj[key] === void 0) {
+          delete obj[key];
+          continue;
+        }
+
         hasKeys = true;
       } else {
         const pathToCheck = (prefix + key);
@@ -372,10 +387,12 @@ function walkUpdatePath(schema, obj, op, options, context, filter, pref) {
           delete obj[key];
         }
       } else {
-        // gh-1845 temporary fix: ignore $rename. See gh-3027 for tracking
-        // improving this.
         if (op === '$rename') {
-          hasKeys = true;
+          if (obj[key] == null) {
+            throw new CastError('String', obj[key], `${prefix}${key}.$rename`);
+          }
+          const schematype = new SchemaString(`${prefix}${key}.$rename`);
+          obj[key] = schematype.castForQuery(null, obj[key], context);
           continue;
         }